[IN TESTING] blink fixes, y axis lock, IBO bug fix

This commit is contained in:
Prohurtz 2023-07-14 18:46:58 -05:00
parent 0cd0ad6334
commit 09bf6737a2
6 changed files with 94 additions and 25 deletions

View File

@ -40,6 +40,7 @@ class AlgoSettingsWidget:
self.gui_BLOBP = f"-BLOBP{widget_id}-"
self.gui_thresh_add = f"-THRESHADD{widget_id}-"
self.gui_ROSC = f"-ROSC{widget_id}-"
self.gui_RANSACBLINK = f"-RANSACBLINK{widget_id}-"
self.gui_vrc_native = f"-VRCNATIVE{widget_id}-"
self.gui_circular_crop_left = f"-CIRCLECROPLEFT{widget_id}-"
self.gui_circular_crop_right = f"-CIRCLECROPRIGHT{widget_id}-"
@ -50,6 +51,7 @@ class AlgoSettingsWidget:
self.ibo_filter_samples = f"-IBOFILTERSAMPLE{widget_id}-"
self.calibration_samples = f"-CALIBRATIONSAMPLES{widget_id}-"
self.ibo_fully_close_eye_threshold = f"-CLOSETHRESH{widget_id}-"
self.main_config = main_config
self.config = main_config.settings
self.osc_queue = osc_queue
@ -180,7 +182,13 @@ class AlgoSettingsWidget:
background_color='#424042',
),
sg.Checkbox(
"Bianary Blink Algo",
"RANSAC Quick Blink Algo",
default=self.config.gui_RANSACBLINK,
key=self.gui_RANSACBLINK,
background_color='#424042',
),
sg.Checkbox(
"Binary Blink Algo",
default=self.config.gui_BLINK,
key=self.gui_BLINK,
background_color='#424042',
@ -390,6 +398,10 @@ class AlgoSettingsWidget:
self.config.gui_IBO = values[self.gui_IBO]
changed = True
if self.config.gui_RANSACBLINK != values[self.gui_RANSACBLINK]:
self.config.gui_RANSACBLINK = values[self.gui_RANSACBLINK]
changed = True
if self.config.gui_circular_crop_left != values[self.gui_circular_crop_left]:
self.config.gui_circular_crop_left = values[self.gui_circular_crop_left]
changed = True

View File

@ -74,6 +74,7 @@ class EyeTrackSettingsConfig(BaseModel):
osc_right_eye_close_address: str = "/avatar/parameters/RightEyeLidExpandedSqueeze"
osc_left_eye_close_address: str = "/avatar/parameters/LeftEyeLidExpandedSqueeze"
osc_invert_eye_close: bool = False
gui_RANSACBLINK: bool = True
class EyeTrackConfig(BaseModel):

View File

@ -160,9 +160,12 @@ class EyeProcessor:
self.blinkvalue = False
self.hasrac_en = False
self.radius = 10
self.past_blink = 0.7
self.prev_x = None
self.prev_y = None
self.prev_y = 0.1
self.prev_x_list = []
self.prev_y_list = []
self.blink_list = []
self.bd_blink = False
self.current_algo = EyeInfoOrigin.HSRAC
@ -282,11 +285,32 @@ class EyeProcessor:
self.eyeopen = 0.0
else:
self.eyeopen = ibo
if len(self.prev_y_list) >= 200: # "lock" eye when close/blink IN TESTING
self.prev_y_list.pop(0)
self.prev_y_list.append(self.out_y)
else:
self.prev_y_list.append(self.out_y)
# print(abs(self.eyeopen - self.past_blink))
blink_vec = min(abs(self.eyeopen - self.past_blink), 1) #clamp to 1
if blink_vec >= 0.1 or blink_vec == 0.0 and (self.out_y - self.prev_y) < 0.0:
# self.out_x = sum(self.prev_x_list) / len(self.prev_x_list)
self.out_y = sum(self.prev_y_list) / len(self.prev_y_list)
# print('AVG', self.out_y, len(self.prev_y_list))
self.past_blink = self.eyeopen
self.prev_x = self.out_x
self.prev_y = self.out_y
self.output_images_and_update(
self.thresh,
EyeInfo(self.current_algo, self.out_x, self.out_y, 0, self.eyeopen),
)
def BLINKM(self):
self.eyeopen = BLINK(self)
@ -327,7 +351,10 @@ class EyeProcessor:
self.rawx, self.rawy, self.thresh, self.radius = self.er_hsf.run(
self.current_image_gray
)
self.rawx, self.rawy, self.thresh, self.eyeopen = RANSAC3D(self, True)
self.rawx, self.rawy, self.thresh, ranblink = RANSAC3D(self, True)
if self.settings.gui_RANSACBLINK:
# print("yes", ranblink)
self.eyeopen = ranblink
# print(self.radius)
# if self.prev_x is None:
# self.prev_x = self.rawx
@ -370,7 +397,9 @@ class EyeProcessor:
current_image_gray_copy = (
self.current_image_gray.copy()
) # Duplicate before overwriting in RANSAC3D.
self.rawx, self.rawy, self.thresh, self.eyeopen = RANSAC3D(self, False)
self.rawx, self.rawy, self.thresh, ranblink = RANSAC3D(self, False)
if self.settings.gui_RANSACBLINK:
self.eyeopen = ranblink
self.out_x, self.out_y = cal.cal_osc(self, self.rawx, self.rawy)
self.current_algorithm = EyeInfoOrigin.RANSAC

View File

@ -259,18 +259,20 @@ class IntensityBasedOpeness:
self.filterlist.pop(0)
self.filterlist.append(intensity)
if intensity >= np.percentile(
self.filterlist, 98
): # filter abnormally high values
# print('filter, assume blink')
intensity = self.maxval
if intensity <= np.percentile( # TODO test this
self.filterlist, 0.3
): # filter abnormally low values
# print('filter, assume blink')
intensity = self.data[int_y, int_x]
try:
if intensity >= np.percentile(
self.filterlist, 98
): # filter abnormally high values
# print('filter, assume blink')
intensity = self.maxval
# if intensity <= np.percentile( # TODO test this
# self.filterlist, 0.3
# ): # filter abnormally low values
# print('filter, assume blink')
# intensity = self.data[int_y, int_x]
except:
pass
# self.tri_filter.append(intensity)
# if len(self.tri_filter) > 3:
# self.tri_filter.pop(0)

View File

@ -48,7 +48,7 @@ def output_osc(eye_x, eye_y, eye_blink, last_blink, self):
self.l_eye_blink = eye_blink
if self.l_eye_blink == 0.0:
if last_blink > 0.7: #when binary blink is on, blinks may be too fast for OSC so we repeat them.
if last_blink > 0.2: #when binary blink is on, blinks may be too fast for OSC so we repeat them.
for i in range(5):
self.client.send_message(self.config.osc_left_eye_close_address, eyelid_transformer(self,self.l_eye_blink))
last_blink = time.time() - last_blink
@ -69,7 +69,7 @@ def output_osc(eye_x, eye_y, eye_blink, last_blink, self):
self.r_eye_blink = eye_blink
if self.r_eye_blink == 0.0:
if last_blink > 0.7: #when binary blink is on, blinks may be too fast for OSC so we repeat them.
if last_blink > 0.2: #when binary blink is on, blinks may be too fast for OSC so we repeat them.
for i in range(5):
self.client.send_message(self.config.osc_right_eye_close_address, eyelid_transformer(self,self.r_eye_blink))
last_blink = time.time() - last_blink
@ -95,8 +95,16 @@ def output_osc(eye_x, eye_y, eye_blink, last_blink, self):
if self.main_config.eye_display_id in [EyeId.RIGHT, EyeId.LEFT]: # we are in single eye mode
se = True
self.client.send_message("/tracking/eye/EyesClosedAmount", float(1 - eye_blink))
if eye_blink == 0.0:
if last_blink > 0.2: # when binary blink is on, blinks may be too fast for OSC so we repeat them.
for i in range(5):
self.client.send_message("/tracking/eye/EyesClosedAmount",
float(1 - eye_blink))
eye_blink += 0.02 #TODO finish tuning value
print(eye_blink)
last_blink = time.time() - last_blink
else:
self.client.send_message("/tracking/eye/EyesClosedAmount", float(1 - eye_blink))
self.client.send_message("/tracking/eye/LeftRightVec", [float(eye_x), float(eye_y), 1.0, float(eye_x), float(eye_y), 1.0]) # vrc native ET
else:
@ -109,7 +117,7 @@ def output_osc(eye_x, eye_y, eye_blink, last_blink, self):
self.client.send_message(self.config.osc_left_eye_close_address,eyelid_transformer(self,eye_blink))
if self.l_eye_blink == 0.0:
if last_blink > 0.7: # when binary blink is on, blinks may be too fast for OSC so we repeat them.
if last_blink > 0.2: # when binary blink is on, blinks may be too fast for OSC so we repeat them.
for i in range(5):
self.client.send_message("/tracking/eye/EyesClosedAmount",
float(1 - eye_blink))
@ -131,7 +139,7 @@ def output_osc(eye_x, eye_y, eye_blink, last_blink, self):
self.client.send_message(self.config.osc_right_eye_close_address,eyelid_transformer(self,eye_blink))
if self.r_eye_blink == 0.0:
if last_blink > 0.7: # when binary blink is on, blinks may be too fast for OSC so we repeat them.
if last_blink > 0.2: # when binary blink is on, blinks may be too fast for OSC so we repeat them.
for i in range(5):
self.client.send_message("/tracking/eye/EyesClosedAmount",
float(1 - eye_blink))
@ -146,7 +154,7 @@ def output_osc(eye_x, eye_y, eye_blink, last_blink, self):
if self.main_config.eye_display_id in [EyeId.BOTH] and self.r_eye_blink != 621 and self.r_eye_blink != 621:
if self.r_eye_blink == 0.0 or self.l_eye_blink == 0.0:
if last_blink > 0.7: # when binary blink is on, blinks may be too fast for OSC so we repeat them.
if last_blink > 0.2: # when binary blink is on, blinks may be too fast for OSC so we repeat them.
for i in range(5):
self.client.send_message("/tracking/eye/EyesClosedAmount",
float(1))

View File

@ -166,6 +166,7 @@ def RANSAC3D(self, hsrac_en):
ranf = False
blink = 0.7
if hsrac_en:
center_x, center_y, upper_x, lower_x, upper_y, lower_y, ransac_lower_x, ransac_lower_y, ransac_upper_x, ransac_upper_y, ransac_xy_offset = get_center_noclamp(
(self.rawx, self.rawy), self.radius)
@ -318,9 +319,25 @@ def RANSAC3D(self, hsrac_en):
perscalarw = w / csx
perscalarh = h / csy
# print(abs(perscalarw-perscalarh))
if abs(perscalarw-perscalarh) >= 0.26: # TODO setting
blink = 0.0
# if abs(perscalarw-perscalarh) >= 0.2: # TODO setting
# blink = 0.0
if len(self.blink_list) >= 10000: # self calibrate ransac blink IN TESTING
self.blink_list.pop(0)
self.blink_list.append(abs(perscalarw-perscalarh))
else:
# print('app')
self.blink_list.append(abs(perscalarw-perscalarh))
# print(np.percentile(blink_list, 80), abs(perscalarw-perscalarh))
# print(len(self.blink_list))
if abs(perscalarw-perscalarh) >= np.percentile(
self.blink_list, 94
):
blink = 0.0
print('blink')
try:
cv2.drawContours(self.current_image_gray, contours, -1, (255, 0, 0), 1) # TODO: fix visualizations with HSRAC