From 09bf6737a2e937c58d9e790c6c6aa1a23d1c75ae Mon Sep 17 00:00:00 2001 From: Prohurtz <48768484+RedHawk989@users.noreply.github.com> Date: Fri, 14 Jul 2023 18:46:58 -0500 Subject: [PATCH] [IN TESTING] blink fixes, y axis lock, IBO bug fix --- EyeTrackApp/algo_settings_widget.py | 14 +++++++++- EyeTrackApp/config.py | 1 + EyeTrackApp/eye_processor.py | 37 ++++++++++++++++++++++--- EyeTrackApp/intensity_based_openness.py | 24 ++++++++-------- EyeTrackApp/osc.py | 22 ++++++++++----- EyeTrackApp/ransac.py | 21 ++++++++++++-- 6 files changed, 94 insertions(+), 25 deletions(-) diff --git a/EyeTrackApp/algo_settings_widget.py b/EyeTrackApp/algo_settings_widget.py index 5c0f23a..eabb2e2 100644 --- a/EyeTrackApp/algo_settings_widget.py +++ b/EyeTrackApp/algo_settings_widget.py @@ -40,6 +40,7 @@ class AlgoSettingsWidget: self.gui_BLOBP = f"-BLOBP{widget_id}-" self.gui_thresh_add = f"-THRESHADD{widget_id}-" self.gui_ROSC = f"-ROSC{widget_id}-" + self.gui_RANSACBLINK = f"-RANSACBLINK{widget_id}-" self.gui_vrc_native = f"-VRCNATIVE{widget_id}-" self.gui_circular_crop_left = f"-CIRCLECROPLEFT{widget_id}-" self.gui_circular_crop_right = f"-CIRCLECROPRIGHT{widget_id}-" @@ -50,6 +51,7 @@ class AlgoSettingsWidget: self.ibo_filter_samples = f"-IBOFILTERSAMPLE{widget_id}-" self.calibration_samples = f"-CALIBRATIONSAMPLES{widget_id}-" self.ibo_fully_close_eye_threshold = f"-CLOSETHRESH{widget_id}-" + self.main_config = main_config self.config = main_config.settings self.osc_queue = osc_queue @@ -180,7 +182,13 @@ class AlgoSettingsWidget: background_color='#424042', ), sg.Checkbox( - "Bianary Blink Algo", + "RANSAC Quick Blink Algo", + default=self.config.gui_RANSACBLINK, + key=self.gui_RANSACBLINK, + background_color='#424042', + ), + sg.Checkbox( + "Binary Blink Algo", default=self.config.gui_BLINK, key=self.gui_BLINK, background_color='#424042', @@ -390,6 +398,10 @@ class AlgoSettingsWidget: self.config.gui_IBO = values[self.gui_IBO] changed = True + if self.config.gui_RANSACBLINK != values[self.gui_RANSACBLINK]: + self.config.gui_RANSACBLINK = values[self.gui_RANSACBLINK] + changed = True + if self.config.gui_circular_crop_left != values[self.gui_circular_crop_left]: self.config.gui_circular_crop_left = values[self.gui_circular_crop_left] changed = True diff --git a/EyeTrackApp/config.py b/EyeTrackApp/config.py index f6ca321..d0a0f3b 100644 --- a/EyeTrackApp/config.py +++ b/EyeTrackApp/config.py @@ -74,6 +74,7 @@ class EyeTrackSettingsConfig(BaseModel): osc_right_eye_close_address: str = "/avatar/parameters/RightEyeLidExpandedSqueeze" osc_left_eye_close_address: str = "/avatar/parameters/LeftEyeLidExpandedSqueeze" osc_invert_eye_close: bool = False + gui_RANSACBLINK: bool = True class EyeTrackConfig(BaseModel): diff --git a/EyeTrackApp/eye_processor.py b/EyeTrackApp/eye_processor.py index a8829f3..4919723 100644 --- a/EyeTrackApp/eye_processor.py +++ b/EyeTrackApp/eye_processor.py @@ -160,9 +160,12 @@ class EyeProcessor: self.blinkvalue = False self.hasrac_en = False self.radius = 10 - + self.past_blink = 0.7 self.prev_x = None - self.prev_y = None + self.prev_y = 0.1 + self.prev_x_list = [] + self.prev_y_list = [] + self.blink_list = [] self.bd_blink = False self.current_algo = EyeInfoOrigin.HSRAC @@ -282,11 +285,32 @@ class EyeProcessor: self.eyeopen = 0.0 else: self.eyeopen = ibo + + + if len(self.prev_y_list) >= 200: # "lock" eye when close/blink IN TESTING + self.prev_y_list.pop(0) + self.prev_y_list.append(self.out_y) + else: + self.prev_y_list.append(self.out_y) + + # print(abs(self.eyeopen - self.past_blink)) + blink_vec = min(abs(self.eyeopen - self.past_blink), 1) #clamp to 1 + + if blink_vec >= 0.1 or blink_vec == 0.0 and (self.out_y - self.prev_y) < 0.0: + # self.out_x = sum(self.prev_x_list) / len(self.prev_x_list) + self.out_y = sum(self.prev_y_list) / len(self.prev_y_list) + # print('AVG', self.out_y, len(self.prev_y_list)) + self.past_blink = self.eyeopen + self.prev_x = self.out_x + self.prev_y = self.out_y + self.output_images_and_update( self.thresh, EyeInfo(self.current_algo, self.out_x, self.out_y, 0, self.eyeopen), ) + + def BLINKM(self): self.eyeopen = BLINK(self) @@ -327,7 +351,10 @@ class EyeProcessor: self.rawx, self.rawy, self.thresh, self.radius = self.er_hsf.run( self.current_image_gray ) - self.rawx, self.rawy, self.thresh, self.eyeopen = RANSAC3D(self, True) + self.rawx, self.rawy, self.thresh, ranblink = RANSAC3D(self, True) + if self.settings.gui_RANSACBLINK: + # print("yes", ranblink) + self.eyeopen = ranblink # print(self.radius) # if self.prev_x is None: # self.prev_x = self.rawx @@ -370,7 +397,9 @@ class EyeProcessor: current_image_gray_copy = ( self.current_image_gray.copy() ) # Duplicate before overwriting in RANSAC3D. - self.rawx, self.rawy, self.thresh, self.eyeopen = RANSAC3D(self, False) + self.rawx, self.rawy, self.thresh, ranblink = RANSAC3D(self, False) + if self.settings.gui_RANSACBLINK: + self.eyeopen = ranblink self.out_x, self.out_y = cal.cal_osc(self, self.rawx, self.rawy) self.current_algorithm = EyeInfoOrigin.RANSAC diff --git a/EyeTrackApp/intensity_based_openness.py b/EyeTrackApp/intensity_based_openness.py index b1072c3..bc1fa67 100644 --- a/EyeTrackApp/intensity_based_openness.py +++ b/EyeTrackApp/intensity_based_openness.py @@ -259,18 +259,20 @@ class IntensityBasedOpeness: self.filterlist.pop(0) self.filterlist.append(intensity) - if intensity >= np.percentile( - self.filterlist, 98 - ): # filter abnormally high values - # print('filter, assume blink') - intensity = self.maxval - - if intensity <= np.percentile( # TODO test this - self.filterlist, 0.3 - ): # filter abnormally low values - # print('filter, assume blink') - intensity = self.data[int_y, int_x] + try: + if intensity >= np.percentile( + self.filterlist, 98 + ): # filter abnormally high values + # print('filter, assume blink') + intensity = self.maxval + # if intensity <= np.percentile( # TODO test this + # self.filterlist, 0.3 + # ): # filter abnormally low values + # print('filter, assume blink') + # intensity = self.data[int_y, int_x] + except: + pass # self.tri_filter.append(intensity) # if len(self.tri_filter) > 3: # self.tri_filter.pop(0) diff --git a/EyeTrackApp/osc.py b/EyeTrackApp/osc.py index a69079c..b43d016 100644 --- a/EyeTrackApp/osc.py +++ b/EyeTrackApp/osc.py @@ -48,7 +48,7 @@ def output_osc(eye_x, eye_y, eye_blink, last_blink, self): self.l_eye_blink = eye_blink if self.l_eye_blink == 0.0: - if last_blink > 0.7: #when binary blink is on, blinks may be too fast for OSC so we repeat them. + if last_blink > 0.2: #when binary blink is on, blinks may be too fast for OSC so we repeat them. for i in range(5): self.client.send_message(self.config.osc_left_eye_close_address, eyelid_transformer(self,self.l_eye_blink)) last_blink = time.time() - last_blink @@ -69,7 +69,7 @@ def output_osc(eye_x, eye_y, eye_blink, last_blink, self): self.r_eye_blink = eye_blink if self.r_eye_blink == 0.0: - if last_blink > 0.7: #when binary blink is on, blinks may be too fast for OSC so we repeat them. + if last_blink > 0.2: #when binary blink is on, blinks may be too fast for OSC so we repeat them. for i in range(5): self.client.send_message(self.config.osc_right_eye_close_address, eyelid_transformer(self,self.r_eye_blink)) last_blink = time.time() - last_blink @@ -95,8 +95,16 @@ def output_osc(eye_x, eye_y, eye_blink, last_blink, self): if self.main_config.eye_display_id in [EyeId.RIGHT, EyeId.LEFT]: # we are in single eye mode se = True - - self.client.send_message("/tracking/eye/EyesClosedAmount", float(1 - eye_blink)) + if eye_blink == 0.0: + if last_blink > 0.2: # when binary blink is on, blinks may be too fast for OSC so we repeat them. + for i in range(5): + self.client.send_message("/tracking/eye/EyesClosedAmount", + float(1 - eye_blink)) + eye_blink += 0.02 #TODO finish tuning value + print(eye_blink) + last_blink = time.time() - last_blink + else: + self.client.send_message("/tracking/eye/EyesClosedAmount", float(1 - eye_blink)) self.client.send_message("/tracking/eye/LeftRightVec", [float(eye_x), float(eye_y), 1.0, float(eye_x), float(eye_y), 1.0]) # vrc native ET else: @@ -109,7 +117,7 @@ def output_osc(eye_x, eye_y, eye_blink, last_blink, self): self.client.send_message(self.config.osc_left_eye_close_address,eyelid_transformer(self,eye_blink)) if self.l_eye_blink == 0.0: - if last_blink > 0.7: # when binary blink is on, blinks may be too fast for OSC so we repeat them. + if last_blink > 0.2: # when binary blink is on, blinks may be too fast for OSC so we repeat them. for i in range(5): self.client.send_message("/tracking/eye/EyesClosedAmount", float(1 - eye_blink)) @@ -131,7 +139,7 @@ def output_osc(eye_x, eye_y, eye_blink, last_blink, self): self.client.send_message(self.config.osc_right_eye_close_address,eyelid_transformer(self,eye_blink)) if self.r_eye_blink == 0.0: - if last_blink > 0.7: # when binary blink is on, blinks may be too fast for OSC so we repeat them. + if last_blink > 0.2: # when binary blink is on, blinks may be too fast for OSC so we repeat them. for i in range(5): self.client.send_message("/tracking/eye/EyesClosedAmount", float(1 - eye_blink)) @@ -146,7 +154,7 @@ def output_osc(eye_x, eye_y, eye_blink, last_blink, self): if self.main_config.eye_display_id in [EyeId.BOTH] and self.r_eye_blink != 621 and self.r_eye_blink != 621: if self.r_eye_blink == 0.0 or self.l_eye_blink == 0.0: - if last_blink > 0.7: # when binary blink is on, blinks may be too fast for OSC so we repeat them. + if last_blink > 0.2: # when binary blink is on, blinks may be too fast for OSC so we repeat them. for i in range(5): self.client.send_message("/tracking/eye/EyesClosedAmount", float(1)) diff --git a/EyeTrackApp/ransac.py b/EyeTrackApp/ransac.py index c76364c..fadbb23 100644 --- a/EyeTrackApp/ransac.py +++ b/EyeTrackApp/ransac.py @@ -166,6 +166,7 @@ def RANSAC3D(self, hsrac_en): ranf = False blink = 0.7 + if hsrac_en: center_x, center_y, upper_x, lower_x, upper_y, lower_y, ransac_lower_x, ransac_lower_y, ransac_upper_x, ransac_upper_y, ransac_xy_offset = get_center_noclamp( (self.rawx, self.rawy), self.radius) @@ -318,9 +319,25 @@ def RANSAC3D(self, hsrac_en): perscalarw = w / csx perscalarh = h / csy # print(abs(perscalarw-perscalarh)) - if abs(perscalarw-perscalarh) >= 0.26: # TODO setting - blink = 0.0 + # if abs(perscalarw-perscalarh) >= 0.2: # TODO setting + # blink = 0.0 + if len(self.blink_list) >= 10000: # self calibrate ransac blink IN TESTING + self.blink_list.pop(0) + self.blink_list.append(abs(perscalarw-perscalarh)) + + else: + # print('app') + self.blink_list.append(abs(perscalarw-perscalarh)) + + # print(np.percentile(blink_list, 80), abs(perscalarw-perscalarh)) + # print(len(self.blink_list)) + + if abs(perscalarw-perscalarh) >= np.percentile( + self.blink_list, 94 + ): + blink = 0.0 + print('blink') try: cv2.drawContours(self.current_image_gray, contours, -1, (255, 0, 0), 1) # TODO: fix visualizations with HSRAC