mirror of
https://github.com/EyeTrackVR/EyeTrackVR.git
synced 2025-11-04 14:39:42 +08:00
[IN TESTING] blink fixes, y axis lock, IBO bug fix
This commit is contained in:
parent
0cd0ad6334
commit
09bf6737a2
@ -40,6 +40,7 @@ class AlgoSettingsWidget:
|
||||
self.gui_BLOBP = f"-BLOBP{widget_id}-"
|
||||
self.gui_thresh_add = f"-THRESHADD{widget_id}-"
|
||||
self.gui_ROSC = f"-ROSC{widget_id}-"
|
||||
self.gui_RANSACBLINK = f"-RANSACBLINK{widget_id}-"
|
||||
self.gui_vrc_native = f"-VRCNATIVE{widget_id}-"
|
||||
self.gui_circular_crop_left = f"-CIRCLECROPLEFT{widget_id}-"
|
||||
self.gui_circular_crop_right = f"-CIRCLECROPRIGHT{widget_id}-"
|
||||
@ -50,6 +51,7 @@ class AlgoSettingsWidget:
|
||||
self.ibo_filter_samples = f"-IBOFILTERSAMPLE{widget_id}-"
|
||||
self.calibration_samples = f"-CALIBRATIONSAMPLES{widget_id}-"
|
||||
self.ibo_fully_close_eye_threshold = f"-CLOSETHRESH{widget_id}-"
|
||||
|
||||
self.main_config = main_config
|
||||
self.config = main_config.settings
|
||||
self.osc_queue = osc_queue
|
||||
@ -180,7 +182,13 @@ class AlgoSettingsWidget:
|
||||
background_color='#424042',
|
||||
),
|
||||
sg.Checkbox(
|
||||
"Bianary Blink Algo",
|
||||
"RANSAC Quick Blink Algo",
|
||||
default=self.config.gui_RANSACBLINK,
|
||||
key=self.gui_RANSACBLINK,
|
||||
background_color='#424042',
|
||||
),
|
||||
sg.Checkbox(
|
||||
"Binary Blink Algo",
|
||||
default=self.config.gui_BLINK,
|
||||
key=self.gui_BLINK,
|
||||
background_color='#424042',
|
||||
@ -390,6 +398,10 @@ class AlgoSettingsWidget:
|
||||
self.config.gui_IBO = values[self.gui_IBO]
|
||||
changed = True
|
||||
|
||||
if self.config.gui_RANSACBLINK != values[self.gui_RANSACBLINK]:
|
||||
self.config.gui_RANSACBLINK = values[self.gui_RANSACBLINK]
|
||||
changed = True
|
||||
|
||||
if self.config.gui_circular_crop_left != values[self.gui_circular_crop_left]:
|
||||
self.config.gui_circular_crop_left = values[self.gui_circular_crop_left]
|
||||
changed = True
|
||||
|
||||
@ -74,6 +74,7 @@ class EyeTrackSettingsConfig(BaseModel):
|
||||
osc_right_eye_close_address: str = "/avatar/parameters/RightEyeLidExpandedSqueeze"
|
||||
osc_left_eye_close_address: str = "/avatar/parameters/LeftEyeLidExpandedSqueeze"
|
||||
osc_invert_eye_close: bool = False
|
||||
gui_RANSACBLINK: bool = True
|
||||
|
||||
|
||||
class EyeTrackConfig(BaseModel):
|
||||
|
||||
@ -160,9 +160,12 @@ class EyeProcessor:
|
||||
self.blinkvalue = False
|
||||
self.hasrac_en = False
|
||||
self.radius = 10
|
||||
|
||||
self.past_blink = 0.7
|
||||
self.prev_x = None
|
||||
self.prev_y = None
|
||||
self.prev_y = 0.1
|
||||
self.prev_x_list = []
|
||||
self.prev_y_list = []
|
||||
self.blink_list = []
|
||||
self.bd_blink = False
|
||||
self.current_algo = EyeInfoOrigin.HSRAC
|
||||
|
||||
@ -282,11 +285,32 @@ class EyeProcessor:
|
||||
self.eyeopen = 0.0
|
||||
else:
|
||||
self.eyeopen = ibo
|
||||
|
||||
|
||||
if len(self.prev_y_list) >= 200: # "lock" eye when close/blink IN TESTING
|
||||
self.prev_y_list.pop(0)
|
||||
self.prev_y_list.append(self.out_y)
|
||||
else:
|
||||
self.prev_y_list.append(self.out_y)
|
||||
|
||||
# print(abs(self.eyeopen - self.past_blink))
|
||||
blink_vec = min(abs(self.eyeopen - self.past_blink), 1) #clamp to 1
|
||||
|
||||
if blink_vec >= 0.1 or blink_vec == 0.0 and (self.out_y - self.prev_y) < 0.0:
|
||||
# self.out_x = sum(self.prev_x_list) / len(self.prev_x_list)
|
||||
self.out_y = sum(self.prev_y_list) / len(self.prev_y_list)
|
||||
# print('AVG', self.out_y, len(self.prev_y_list))
|
||||
self.past_blink = self.eyeopen
|
||||
self.prev_x = self.out_x
|
||||
self.prev_y = self.out_y
|
||||
|
||||
self.output_images_and_update(
|
||||
self.thresh,
|
||||
EyeInfo(self.current_algo, self.out_x, self.out_y, 0, self.eyeopen),
|
||||
)
|
||||
|
||||
|
||||
|
||||
def BLINKM(self):
|
||||
self.eyeopen = BLINK(self)
|
||||
|
||||
@ -327,7 +351,10 @@ class EyeProcessor:
|
||||
self.rawx, self.rawy, self.thresh, self.radius = self.er_hsf.run(
|
||||
self.current_image_gray
|
||||
)
|
||||
self.rawx, self.rawy, self.thresh, self.eyeopen = RANSAC3D(self, True)
|
||||
self.rawx, self.rawy, self.thresh, ranblink = RANSAC3D(self, True)
|
||||
if self.settings.gui_RANSACBLINK:
|
||||
# print("yes", ranblink)
|
||||
self.eyeopen = ranblink
|
||||
# print(self.radius)
|
||||
# if self.prev_x is None:
|
||||
# self.prev_x = self.rawx
|
||||
@ -370,7 +397,9 @@ class EyeProcessor:
|
||||
current_image_gray_copy = (
|
||||
self.current_image_gray.copy()
|
||||
) # Duplicate before overwriting in RANSAC3D.
|
||||
self.rawx, self.rawy, self.thresh, self.eyeopen = RANSAC3D(self, False)
|
||||
self.rawx, self.rawy, self.thresh, ranblink = RANSAC3D(self, False)
|
||||
if self.settings.gui_RANSACBLINK:
|
||||
self.eyeopen = ranblink
|
||||
self.out_x, self.out_y = cal.cal_osc(self, self.rawx, self.rawy)
|
||||
self.current_algorithm = EyeInfoOrigin.RANSAC
|
||||
|
||||
|
||||
@ -259,18 +259,20 @@ class IntensityBasedOpeness:
|
||||
self.filterlist.pop(0)
|
||||
self.filterlist.append(intensity)
|
||||
|
||||
if intensity >= np.percentile(
|
||||
self.filterlist, 98
|
||||
): # filter abnormally high values
|
||||
# print('filter, assume blink')
|
||||
intensity = self.maxval
|
||||
|
||||
if intensity <= np.percentile( # TODO test this
|
||||
self.filterlist, 0.3
|
||||
): # filter abnormally low values
|
||||
# print('filter, assume blink')
|
||||
intensity = self.data[int_y, int_x]
|
||||
try:
|
||||
if intensity >= np.percentile(
|
||||
self.filterlist, 98
|
||||
): # filter abnormally high values
|
||||
# print('filter, assume blink')
|
||||
intensity = self.maxval
|
||||
|
||||
# if intensity <= np.percentile( # TODO test this
|
||||
# self.filterlist, 0.3
|
||||
# ): # filter abnormally low values
|
||||
# print('filter, assume blink')
|
||||
# intensity = self.data[int_y, int_x]
|
||||
except:
|
||||
pass
|
||||
# self.tri_filter.append(intensity)
|
||||
# if len(self.tri_filter) > 3:
|
||||
# self.tri_filter.pop(0)
|
||||
|
||||
@ -48,7 +48,7 @@ def output_osc(eye_x, eye_y, eye_blink, last_blink, self):
|
||||
self.l_eye_blink = eye_blink
|
||||
|
||||
if self.l_eye_blink == 0.0:
|
||||
if last_blink > 0.7: #when binary blink is on, blinks may be too fast for OSC so we repeat them.
|
||||
if last_blink > 0.2: #when binary blink is on, blinks may be too fast for OSC so we repeat them.
|
||||
for i in range(5):
|
||||
self.client.send_message(self.config.osc_left_eye_close_address, eyelid_transformer(self,self.l_eye_blink))
|
||||
last_blink = time.time() - last_blink
|
||||
@ -69,7 +69,7 @@ def output_osc(eye_x, eye_y, eye_blink, last_blink, self):
|
||||
self.r_eye_blink = eye_blink
|
||||
|
||||
if self.r_eye_blink == 0.0:
|
||||
if last_blink > 0.7: #when binary blink is on, blinks may be too fast for OSC so we repeat them.
|
||||
if last_blink > 0.2: #when binary blink is on, blinks may be too fast for OSC so we repeat them.
|
||||
for i in range(5):
|
||||
self.client.send_message(self.config.osc_right_eye_close_address, eyelid_transformer(self,self.r_eye_blink))
|
||||
last_blink = time.time() - last_blink
|
||||
@ -95,8 +95,16 @@ def output_osc(eye_x, eye_y, eye_blink, last_blink, self):
|
||||
|
||||
if self.main_config.eye_display_id in [EyeId.RIGHT, EyeId.LEFT]: # we are in single eye mode
|
||||
se = True
|
||||
|
||||
self.client.send_message("/tracking/eye/EyesClosedAmount", float(1 - eye_blink))
|
||||
if eye_blink == 0.0:
|
||||
if last_blink > 0.2: # when binary blink is on, blinks may be too fast for OSC so we repeat them.
|
||||
for i in range(5):
|
||||
self.client.send_message("/tracking/eye/EyesClosedAmount",
|
||||
float(1 - eye_blink))
|
||||
eye_blink += 0.02 #TODO finish tuning value
|
||||
print(eye_blink)
|
||||
last_blink = time.time() - last_blink
|
||||
else:
|
||||
self.client.send_message("/tracking/eye/EyesClosedAmount", float(1 - eye_blink))
|
||||
self.client.send_message("/tracking/eye/LeftRightVec", [float(eye_x), float(eye_y), 1.0, float(eye_x), float(eye_y), 1.0]) # vrc native ET
|
||||
|
||||
else:
|
||||
@ -109,7 +117,7 @@ def output_osc(eye_x, eye_y, eye_blink, last_blink, self):
|
||||
self.client.send_message(self.config.osc_left_eye_close_address,eyelid_transformer(self,eye_blink))
|
||||
|
||||
if self.l_eye_blink == 0.0:
|
||||
if last_blink > 0.7: # when binary blink is on, blinks may be too fast for OSC so we repeat them.
|
||||
if last_blink > 0.2: # when binary blink is on, blinks may be too fast for OSC so we repeat them.
|
||||
for i in range(5):
|
||||
self.client.send_message("/tracking/eye/EyesClosedAmount",
|
||||
float(1 - eye_blink))
|
||||
@ -131,7 +139,7 @@ def output_osc(eye_x, eye_y, eye_blink, last_blink, self):
|
||||
self.client.send_message(self.config.osc_right_eye_close_address,eyelid_transformer(self,eye_blink))
|
||||
|
||||
if self.r_eye_blink == 0.0:
|
||||
if last_blink > 0.7: # when binary blink is on, blinks may be too fast for OSC so we repeat them.
|
||||
if last_blink > 0.2: # when binary blink is on, blinks may be too fast for OSC so we repeat them.
|
||||
for i in range(5):
|
||||
self.client.send_message("/tracking/eye/EyesClosedAmount",
|
||||
float(1 - eye_blink))
|
||||
@ -146,7 +154,7 @@ def output_osc(eye_x, eye_y, eye_blink, last_blink, self):
|
||||
|
||||
if self.main_config.eye_display_id in [EyeId.BOTH] and self.r_eye_blink != 621 and self.r_eye_blink != 621:
|
||||
if self.r_eye_blink == 0.0 or self.l_eye_blink == 0.0:
|
||||
if last_blink > 0.7: # when binary blink is on, blinks may be too fast for OSC so we repeat them.
|
||||
if last_blink > 0.2: # when binary blink is on, blinks may be too fast for OSC so we repeat them.
|
||||
for i in range(5):
|
||||
self.client.send_message("/tracking/eye/EyesClosedAmount",
|
||||
float(1))
|
||||
|
||||
@ -166,6 +166,7 @@ def RANSAC3D(self, hsrac_en):
|
||||
ranf = False
|
||||
blink = 0.7
|
||||
|
||||
|
||||
if hsrac_en:
|
||||
center_x, center_y, upper_x, lower_x, upper_y, lower_y, ransac_lower_x, ransac_lower_y, ransac_upper_x, ransac_upper_y, ransac_xy_offset = get_center_noclamp(
|
||||
(self.rawx, self.rawy), self.radius)
|
||||
@ -318,9 +319,25 @@ def RANSAC3D(self, hsrac_en):
|
||||
perscalarw = w / csx
|
||||
perscalarh = h / csy
|
||||
# print(abs(perscalarw-perscalarh))
|
||||
if abs(perscalarw-perscalarh) >= 0.26: # TODO setting
|
||||
blink = 0.0
|
||||
# if abs(perscalarw-perscalarh) >= 0.2: # TODO setting
|
||||
# blink = 0.0
|
||||
|
||||
if len(self.blink_list) >= 10000: # self calibrate ransac blink IN TESTING
|
||||
self.blink_list.pop(0)
|
||||
self.blink_list.append(abs(perscalarw-perscalarh))
|
||||
|
||||
else:
|
||||
# print('app')
|
||||
self.blink_list.append(abs(perscalarw-perscalarh))
|
||||
|
||||
# print(np.percentile(blink_list, 80), abs(perscalarw-perscalarh))
|
||||
# print(len(self.blink_list))
|
||||
|
||||
if abs(perscalarw-perscalarh) >= np.percentile(
|
||||
self.blink_list, 94
|
||||
):
|
||||
blink = 0.0
|
||||
print('blink')
|
||||
|
||||
try:
|
||||
cv2.drawContours(self.current_image_gray, contours, -1, (255, 0, 0), 1) # TODO: fix visualizations with HSRAC
|
||||
|
||||
Loading…
Reference in New Issue
Block a user