IBO fix, HSRAC fix

This commit is contained in:
Prohurtz 2023-06-01 12:03:04 -05:00
parent ea33273b73
commit 1c7c6d964b
8 changed files with 164 additions and 87 deletions

View File

@ -368,4 +368,5 @@ class AlgoSettingsWidget:
if changed: if changed:
self.main_config.save() self.main_config.save()
#print(self.main_config)
self.osc_queue.put(EyeId.ALGOSETTINGS) self.osc_queue.put(EyeId.ALGOSETTINGS)

View File

@ -8,10 +8,13 @@ def BLINK(self):
intensity = np.sum(self.current_image_gray_clean) intensity = np.sum(self.current_image_gray_clean)
self.frames = self.frames + 1 self.frames = self.frames + 1
if len(str(intensity)) >= 8: # filter abnormally high values
print('filter, assume blink')
intensity = self.max_int + 1
if intensity > self.max_int: if intensity > self.max_int:
self.max_int = intensity self.max_int = intensity
if self.frames > 400: #TODO: test this number more (make it a setting??) if self.frames > 200: #TODO: test this number more (make it a setting??)
self.max_ints.append(self.max_int) self.max_ints.append(self.max_int)
if intensity < self.min_int: if intensity < self.min_int:
self.min_int = intensity self.min_int = intensity

View File

@ -352,9 +352,9 @@ class CameraWidget:
line_color="white", line_color="white",
) )
if not np.isnan(eye_info.blink): if not np.isnan(eye_info.blink):
graph.draw_line((-100,eye_info.blink * 100), (-100,100), color="black", width=10) graph.draw_line((-100,eye_info.blink * 100), (-100,100), color="#6f4ca1", width=10)
else: else:
graph.draw_line((-100, 0.0 * 100), (-100, 100), color="black", width=10) graph.draw_line((-100, 0.0 * 200), (-100, 100), color="black", width=10)
if eye_info.blink <= 0.0: if eye_info.blink <= 0.0:
graph.update(background_color="#6f4ca1") graph.update(background_color="#6f4ca1")

View File

@ -57,7 +57,7 @@ from haar_surround_feature import External_Run_HSF
from blob import * from blob import *
from ransac import * from ransac import *
from blink import * from blink import *
from utils.img_utils import circle_crop
from eye import EyeInfo, EyeInfoOrigin from eye import EyeInfo, EyeInfoOrigin
from intensity_based_openness import * from intensity_based_openness import *
@ -89,6 +89,7 @@ class EyeProcessor:
image_queue_outgoing: "queue.Queue", image_queue_outgoing: "queue.Queue",
eye_id, eye_id,
): ):
self.main_config = EyeTrackSettingsConfig
self.config = config self.config = config
self.settings = settings self.settings = settings
self.eye_id = eye_id self.eye_id = eye_id
@ -101,8 +102,9 @@ class EyeProcessor:
# Cross algo state # Cross algo state
self.lkg_projected_sphere = None self.lkg_projected_sphere = None
self.xc = None self.xc = 20
self.yc = None self.yc = 20
self.cc_radius = 40
# Image state # Image state
self.previous_image = None self.previous_image = None
@ -124,7 +126,7 @@ class EyeProcessor:
self.ymax = -69420 self.ymax = -69420
self.ymin = 69420 self.ymin = 69420
self.blink_clear = False self.blink_clear = False
self.cct = 300 self.cct = 200
self.cccs = False self.cccs = False
self.ts = 10 self.ts = 10
self.previous_rotation = self.config.rotation_angle self.previous_rotation = self.config.rotation_angle
@ -230,7 +232,14 @@ class EyeProcessor:
rotation_matrix, rotation_matrix,
(cols, rows), (cols, rows),
borderMode=cv2.BORDER_CONSTANT, borderMode=cv2.BORDER_CONSTANT,
borderValue=(ar + 10, ag + 10, ab + 10),#(255, 255, 255), borderValue=(ar, ag, ab),#(255, 255, 255),
)
self.current_image_white = cv2.warpAffine(
self.current_image,
rotation_matrix,
(cols, rows),
borderMode=cv2.BORDER_CONSTANT,
borderValue=(255, 255, 255),
) )
return True return True
except: except:
@ -243,7 +252,7 @@ class EyeProcessor:
self.eyeopen = BLINK(self) self.eyeopen = BLINK(self)
if self.settings.gui_IBO: if self.settings.gui_IBO:
self.eyeopen = self.ibo.intense(self.rawx, self.rawy, self.current_image) self.eyeopen = self.ibo.intense(self.rawx, self.rawy, self.current_image_white)
if self.eyeopen < 0.35: #threshold so the eye fully closes #todo: make this a setting? if self.eyeopen < 0.35: #threshold so the eye fully closes #todo: make this a setting?
self.eyeopen = 0.0 self.eyeopen = 0.0
if self.bd_blink == True: if self.bd_blink == True:
@ -251,7 +260,7 @@ class EyeProcessor:
# self.eyeopen = 0.0 # self.eyeopen = 0.0
if self.settings.gui_IBO and self.settings.gui_BLINK: if self.settings.gui_IBO and self.settings.gui_BLINK:
ibo = self.ibo.intense(self.rawx, self.rawy, self.current_image) ibo = self.ibo.intense(self.rawx, self.rawy, self.current_image_white)
blink = BLINK(self) blink = BLINK(self)
if blink == 0.0: if blink == 0.0:
@ -274,10 +283,19 @@ class EyeProcessor:
self.current_algorithm = EyeInfoOrigin.DADDY self.current_algorithm = EyeInfoOrigin.DADDY
def HSRACM(self): def HSRACM(self):
if self.eye_id in [EyeId.LEFT] and self.settings.gui_circular_crop_left:
self.current_image_gray, self.cct = circle_crop(self.current_image_gray, self.xc, self.yc, self.cc_radius, self.cct)
else:
pass
if self.eye_id in [EyeId.RIGHT] and self.settings.gui_circular_crop_right:
self.current_image_gray, self.cct = circle_crop(self.current_image_gray, self.xc, self.yc, self.cc_radius, self.cct)
else:
pass
self.hasrac_en = True self.hasrac_en = True
# todo: add process to initialise er_hsrac when resolution changes # todo: add process to initialise er_hsrac when resolution changes
self.rawx, self.rawy, self.thresh, self.radius = self.er_hsf.run(self.current_image_gray) self.rawx, self.rawy, self.thresh, self.radius = self.er_hsf.run(self.current_image_gray)
self.rawx, self.rawy, self.thresh = RANSAC3D(self) self.rawx, self.rawy, self.thresh = RANSAC3D(self, True)
#if self.prev_x is None: #if self.prev_x is None:
# self.prev_x = self.rawx # self.prev_x = self.rawx
@ -285,19 +303,38 @@ class EyeProcessor:
self.out_x, self.out_y = cal.cal_osc(self, self.rawx, self.rawy) self.out_x, self.out_y = cal.cal_osc(self, self.rawx, self.rawy)
self.current_algorithm = EyeInfoOrigin.HSRAC self.current_algorithm = EyeInfoOrigin.HSRAC
def HSFM(self): def HSFM(self):
if self.eye_id in [EyeId.LEFT] and self.settings.gui_circular_crop_left:
self.current_image_gray, self.cct = circle_crop(self.current_image_gray, self.xc, self.yc, self.cc_radius, self.cct)
else:
pass
if self.eye_id in [EyeId.RIGHT] and self.settings.gui_circular_crop_right:
self.current_image_gray, self.cct = circle_crop(self.current_image_gray, self.xc, self.yc, self.cc_radius, self.cct)
else:
pass
# todo: add process to initialise er_hsf when resolution changes # todo: add process to initialise er_hsf when resolution changes
self.rawx, self.rawy, self.thresh = self.er_hsf.run(self.current_image_gray) self.rawx, self.rawy, self.thresh = self.er_hsf.run(self.current_image_gray)
self.out_x, self.out_y = cal.cal_osc(self, self.rawx, self.rawy) self.out_x, self.out_y = cal.cal_osc(self, self.rawx, self.rawy)
self.current_algorithm = EyeInfoOrigin.HSF self.current_algorithm = EyeInfoOrigin.HSF
def RANSAC3DM(self): def RANSAC3DM(self):
if self.eye_id in [EyeId.LEFT] and self.settings.gui_circular_crop_left:
self.current_image_gray, self.cct = circle_crop(self.current_image_gray, self.xc, self.yc, self.cc_radius, self.cct)
else:
pass
if self.eye_id in [EyeId.RIGHT] and self.settings.gui_circular_crop_right:
self.current_image_gray, self.cct = circle_crop(self.current_image_gray, self.xc, self.yc, self.cc_radius, self.cct)
else:
pass
self.hasrac_en = False
current_image_gray_copy = self.current_image_gray.copy() # Duplicate before overwriting in RANSAC3D. current_image_gray_copy = self.current_image_gray.copy() # Duplicate before overwriting in RANSAC3D.
self.rawx, self.rawy, self.thresh = RANSAC3D(self) self.rawx, self.rawy, self.thresh = RANSAC3D(self, False)
self.out_x, self.out_y = cal.cal_osc(self, self.rawx, self.rawy) self.out_x, self.out_y = cal.cal_osc(self, self.rawx, self.rawy)
self.current_algorithm = EyeInfoOrigin.RANSAC self.current_algorithm = EyeInfoOrigin.RANSAC
def BLOBM(self): def BLOBM(self):
print("LSKDGFHL") print("LSKDGFHL")
self.rawx, self.rawy, self.thresh = BLOB(self) self.rawx, self.rawy, self.thresh = BLOB(self)

View File

@ -3,12 +3,17 @@ from functools import lru_cache
import cv2 import cv2
import numpy as np import numpy as np
from utils.misc_utils import clamp from utils.misc_utils import clamp
from utils.img_utils import safe_crop from utils.img_utils import safe_crop
from enum import IntEnum
class EyeId(IntEnum):
RIGHT = 0
LEFT = 1
BOTH = 2
SETTINGS = 3
# from line_profiler_pycharm import profile # from line_profiler_pycharm import profile
video_path = "ezgif.com-gif-maker.avi" video_path = "ezgif.com-gif-maker.avi"
@ -500,10 +505,15 @@ class HSF_cls(object):
return True return True
return False return False
cct = 300
ransac_lower_x = 100
ransac_lower_y = 100
cx = 0
cy = 0
def single_run(self): def single_run(self):
# Temporary implementation to run # Temporary implementation to run
## default_radius = 14 ## default_radius = 14
# cropbox=[] # debug code # cropbox=[] # debug code

View File

@ -122,6 +122,9 @@ class IntensityBasedOpeness:
self.color = [] self.color = []
self.x = [] self.x = []
self.fc = 0 self.fc = 0
self.filterlist = []
self.maxinten = 0
self.tri_filter = [] self.tri_filter = []
# try: # try:
@ -202,24 +205,38 @@ class IntensityBasedOpeness:
int_x, int_y = int(x), int(y) int_x, int_y = int(x), int(y)
if int_x < 0 or int_y < 0: if int_x < 0 or int_y < 0:
return self.prev_val return self.prev_val
upper_x = min(int_x + 15, frame.shape[1]-1) #TODO make this a setting upper_x = min(int_x + 25, frame.shape[1]-1) #TODO make this a setting
lower_x = max(int_x - 15, 0) lower_x = max(int_x - 25, 0)
upper_y = min(int_y + 15, frame.shape[0]-1) upper_y = min(int_y + 25, frame.shape[0]-1)
lower_y = max(int_y - 15, 0) lower_y = max(int_y - 25, 0)
# frame_crop = frame[lower_y:upper_y, lower_x:upper_x] # frame_crop = frame[lower_y:upper_y, lower_x:upper_x]
img = safe_crop(frame, lower_x, lower_y, upper_x, upper_y, 1) #frame = safe_crop(frame, lower_x, lower_y, upper_x, upper_y, False)
#ret_, th = cv2.threshold(frame_crop, 80, 1.0, cv2.THRESH_BINARY_INV, dst=frame_crop) #ret_, th = cv2.threshold(frame_crop, 80, 1.0, cv2.THRESH_BINARY_INV, dst=frame_crop)
frame_crop = frame frame_crop = frame
#ret, f = cv2.threshold(frame, 80, 255, cv2.THRESH_BINARY) #ret, f = cv2.threshold(frame, 80, 255, cv2.THRESH_BINARY)
# ret, frame_crop = cv2.threshold(frame_crop, 80, 255, cv2.THRESH_BINARY) # ret, frame_crop = cv2.threshold(frame_crop, 80, 255, cv2.THRESH_BINARY)
# The same can be done with cv2.integral, but since there is only one area of the rectangle for which we want to know the total value, there is no advantage in terms of computational complexity. # The same can be done with cv2.integral, but since there is only one area of the rectangle for which we want to know the total value, there is no advantage in terms of computational complexity.
intensity = frame_crop.sum() + 1 intensity = frame_crop.sum() + 1
self.tri_filter.append(intensity) #cv2.imshow('e', frame)
if len(self.tri_filter) > 3: # if cv2.waitKey(10) == 27:
self.tri_filter.pop(0) # exit()
intensity = sum(self.tri_filter) / 3 if len(self.filterlist) < 800:
self.filterlist.append(intensity)
else:
self.filterlist.pop()
self.filterlist.append(intensity)
print(len(str(intensity)))
if len(str(intensity)) >= 8: #filter abnormally high values
print('filter, assume blink')
intensity = self.maxval
#self.tri_filter.append(intensity)
#if len(self.tri_filter) > 3:
# self.tri_filter.pop(0)
# intensity = sum(self.tri_filter) / 3
#avg_color_per_row = np.average(frame_crop, axis=0) #avg_color_per_row = np.average(frame_crop, axis=0)
#avg_color = np.average(avg_color_per_row, axis=0) #avg_color = np.average(avg_color_per_row, axis=0)
# ar, ag, ab = avg_color # ar, ag, ab = avg_color
@ -229,11 +246,6 @@ class IntensityBasedOpeness:
#if cv2.waitKey(1) & 0xFF == ord("q"): #if cv2.waitKey(1) & 0xFF == ord("q"):
# pass # pass
#print(intensity)
# if our blob width/height are within suitable (yet arbitrary) boundaries, call that good.
#
# TODO This should be scaled based on camera resolution.
# numpy:np.sum(),ndarray.sum() # numpy:np.sum(),ndarray.sum()
# opencv:cv2.sumElems() # opencv:cv2.sumElems()
# I don't know which is faster. # I don't know which is faster.
@ -301,7 +313,8 @@ class IntensityBasedOpeness:
eyeopen = ((intensity - maxp) / (minp - maxp)) #for whatever reason when input and maxp are too close it outputs high eyeopen = ((intensity - maxp) / (minp - maxp)) #for whatever reason when input and maxp are too close it outputs high
# print(eyeopen, maxp, minp)
# eyeopen = ((eyeopen - 0.3) / (1.0 - 0.3))
eyeopen = 1 - eyeopen eyeopen = 1 - eyeopen
# print(eyeopen, intensity, maxp, minp, x, y) # print(eyeopen, intensity, maxp, minp, x, y)
@ -325,10 +338,10 @@ class IntensityBasedOpeness:
print('BLINK') print('BLINK')
#intensityold = img.sum() + 1 #intensityold = img.sum() + 1
avg_color_per_row = np.average(img, axis=0) # avg_color_per_row = np.average(img, axis=0)
avg_color = np.average(avg_color_per_row, axis=0) # avg_color = np.average(avg_color_per_row, axis=0)
ar, ag, ab = avg_color #ar, ag, ab = avg_color
intensity = int(ar * 8) # higher = closed # intensity = int(ar * 8) # higher = closed
#self.old.append(intensity) #self.old.append(intensity)
#self.color.append(intensity) #self.color.append(intensity)
# self.x.append(self.fc) # self.x.append(self.fc)

View File

@ -29,6 +29,7 @@ import cv2
import numpy as np import numpy as np
from enum import IntEnum from enum import IntEnum
from utils.img_utils import safe_crop from utils.img_utils import safe_crop
from utils.misc_utils import clamp
class EyeId(IntEnum): class EyeId(IntEnum):
RIGHT = 0 RIGHT = 0
LEFT = 1 LEFT = 1
@ -142,62 +143,28 @@ def fit_rotated_ellipse(data, P):
return (cx, cy, w, h, theta) return (cx, cy, w, h, theta)
cct = 300 cct = 300
def circle_crop(self): ransac_lower_x = 100
global cct ransac_lower_y = 100
avg_color_per_row = np.average(self.current_image, axis=0) cx = 0
avg_color = np.average(avg_color_per_row, axis=0) cy = 0
ar, ag, ab = avg_color
if cct == 0:
try:
ht, wd = self.current_image_gray.shape[:2]
radius = int(float(self.lkg_projected_sphere["axes"][0]))
self.xc = int(float(self.lkg_projected_sphere["center"][0]))
self.yc = int(float(self.lkg_projected_sphere["center"][1]))
if radius < 10: #minimum size
radius = 10
# draw filled circle in white on black background as mask
mask = np.zeros((ht, wd), dtype=np.uint8)
mask = cv2.circle(mask, (self.xc, self.yc), radius, 255, -1)
# create white colored background
color = np.full_like(self.current_image_gray, (ar))
# apply mask to image
masked_img = cv2.bitwise_and(self.current_image_gray, self.current_image_gray, mask=mask)
# apply inverse mask to colored image
masked_color = cv2.bitwise_and(color, color, mask=255 - mask)
# combine the two masked images
self.current_image_gray = cv2.add(masked_img, masked_color)
return self.current_image_gray
except:
return self.current_image_gray
pass
else:
cct = cct - 1
return self.current_image_gray
def RANSAC3D(self): def RANSAC3D(self, hsrac_en):
f = False f = False
if self.eye_id in [EyeId.LEFT] and self.settings.gui_circular_crop_left: global cct, ransac_lower_y, ransac_lower_x, cx, cy
self.current_image_gray = circle_crop(self)
else:
pass
if self.eye_id in [EyeId.RIGHT] and self.settings.gui_circular_crop_right:
self.current_image_gray = circle_crop(self)
else:
pass
if self.hasrac_en: if hsrac_en:
ransac_lower_x = self.rawx - self.radius ransac_upper_x = self.rawx + max(15, self.radius)
ransac_upper_x = self.rawx + self.radius ransac_lower_x = self.rawx - max(15, self.radius)
ransac_lower_y = self.rawy - self.radius ransac_upper_y = self.rawy + max(15, self.radius)
ransac_upper_y = self.rawy + self.radius ransac_lower_y = self.rawy - max(15, self.radius)
frame = safe_crop(self.current_image_gray_clean, ransac_lower_x, ransac_lower_y, ransac_upper_x, ransac_upper_y, 1) frame = safe_crop(self.current_image_gray_clean, ransac_lower_x, ransac_lower_y, ransac_upper_x, ransac_upper_y, 1)
else: else:
frame = self.current_image_gray_clean frame = self.current_image_gray_clean
global cct
kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (3, 3)) kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (3, 3))
thresh_add = 10 thresh_add = 10
rng = np.random.default_rng() rng = np.random.default_rng()
@ -315,10 +282,24 @@ def RANSAC3D(self):
eym = ellipse_3d["center"][1] eym = ellipse_3d["center"][1]
d = result_3d["diameter_3d"] d = result_3d["diameter_3d"]
self.cc_radius = int(float(self.lkg_projected_sphere["axes"][0]))
self.xc = int(float(self.lkg_projected_sphere["center"][0]))
self.yc = int(float(self.lkg_projected_sphere["center"][1]))
except: except:
f = True f = True
# Draw our image and stack it for visual output # Draw our image and stack it for visual output
if hsrac_en:
csy = newFrame2.shape[0]
csx = newFrame2.shape[1]
ransac_xy_offset = (ransac_lower_x, ransac_lower_y)
# cx = clamp((cx - 20) + center_x, 0, csx)
# cy = clamp((cy - 20) + center_y, 0, csy)
cx = int(clamp(cx + ransac_xy_offset[0], 0, csx))
cy = int(clamp(cy + ransac_xy_offset[1], 0, csy))
try: try:
cv2.drawContours(self.current_image_gray, contours, -1, (255, 0, 0), 1) # TODO: fix visualizations with HSRAC cv2.drawContours(self.current_image_gray, contours, -1, (255, 0, 0), 1) # TODO: fix visualizations with HSRAC
cv2.circle(self.current_image_gray, (int(cx), int(cy)), 2, (0, 0, 255), -1) cv2.circle(self.current_image_gray, (int(cx), int(cy)), 2, (0, 0, 255), -1)
@ -362,6 +343,9 @@ def RANSAC3D(self):
except: except:
pass pass
self.current_image_gray = newFrame2 self.current_image_gray = newFrame2
y, x = self.current_image_gray.shape y, x = self.current_image_gray.shape
thresh = cv2.resize(thresh, (x,y)) thresh = cv2.resize(thresh, (x,y))

View File

@ -1,7 +1,7 @@
import cv2 import cv2
import numpy as np
def safe_crop(img, x, y, x2, y2, keepsize=False):
def safe_crop(img, x, y, x2, y2, keepsize=True):
# The order of the arguments can be reconsidered. # The order of the arguments can be reconsidered.
img_h, img_w = img.shape[:2] img_h, img_w = img.shape[:2]
outimg = img[max(0, y) : min(img_h, y2), max(0, x) : min(img_w, x2)].copy() outimg = img[max(0, y) : min(img_h, y2), max(0, x) : min(img_w, x2)].copy()
@ -10,3 +10,32 @@ def safe_crop(img, x, y, x2, y2, keepsize=True):
# If the size is different from the expected size (smaller by the amount that is out of range) # If the size is different from the expected size (smaller by the amount that is out of range)
outimg = cv2.resize(outimg, (reqsize_x, reqsize_y)) outimg = cv2.resize(outimg, (reqsize_x, reqsize_y))
return outimg return outimg
def circle_crop(img, xc, yc, radius, cct):
avg_color_per_row = np.average(img, axis=0)
avg_color = np.average(avg_color_per_row, axis=0)
if cct <= 0:
try:
ht, wd = img.shape[:2]
if radius < 10: #minimum size
radius = 10
# draw filled circle in white on black background as mask
mask = np.zeros((ht, wd), dtype=np.uint8)
mask = cv2.circle(mask, (xc, yc), radius, 255, -1)
# create white colored background
color = np.full_like(img, (avg_color))
# apply mask to image
masked_img = cv2.bitwise_and(img, img, mask=mask)
# apply inverse mask to colored image
masked_color = cv2.bitwise_and(color, color, mask=255 - mask)
# combine the two masked images
outimg = cv2.add(masked_img, masked_color)
return outimg, cct
except:
return img, cct
pass
else:
cct = cct - 1
return img, cct