diff --git a/EyeTrackApp/camera.py b/EyeTrackApp/camera.py index 97eb062..e8d7b4e 100644 --- a/EyeTrackApp/camera.py +++ b/EyeTrackApp/camera.py @@ -83,7 +83,7 @@ class Camera: def done_callback(self, future): try: - time.sleep(0.1) # Very important wait + time.sleep(0.5) # Very important wait self.cv2_camera = future.result() except Exception: # Set cv2_camera to None if an exception occurred diff --git a/EyeTrackApp/camera_widget.py b/EyeTrackApp/camera_widget.py index ff9452b..1d5bd87 100644 --- a/EyeTrackApp/camera_widget.py +++ b/EyeTrackApp/camera_widget.py @@ -343,21 +343,23 @@ class CameraWidget: graph.draw_circle( (eye_info.x * -100, eye_info.y * -100), - 25, + 20, fill_color="black", line_color="white", ) else: graph.draw_circle( (0.0 * -100, 0.0 * -100), - 25, + 20, fill_color="black", line_color="white", ) + if not np.isnan(eye_info.blink): - graph.draw_line((-100,eye_info.blink * 200), (-100,100), color="#6f4ca1", width=10) + + graph.draw_line((-100, eye_info.blink * 200), (-100, 100), color="#6f4ca1", width=10) else: - graph.draw_line((-100, 0.0 * 200), (-100, 100), color="black", width=10) + graph.draw_line((-100, 0.5 * 200), (-100, 100), color="#6f4ca1", width=10) if eye_info.blink <= 0.0: graph.update(background_color="#6f4ca1") diff --git a/EyeTrackApp/eye_processor.py b/EyeTrackApp/eye_processor.py index 6689e7c..07bb17b 100644 --- a/EyeTrackApp/eye_processor.py +++ b/EyeTrackApp/eye_processor.py @@ -253,7 +253,7 @@ class EyeProcessor: if self.settings.gui_BLINK: self.eyeopen = BLINK(self) - if self.settings.gui_IBO: + if self.settings.gui_IBO and self.eyeopen != 0.0: self.eyeopen = self.ibo.intense( self.rawx, self.rawy, @@ -268,7 +268,7 @@ class EyeProcessor: if self.bd_blink == True: pass - if self.settings.gui_IBO and self.settings.gui_BLINK: + if self.settings.gui_IBO and self.settings.gui_BLINK and self.eyeopen != 0.0: ibo = self.ibo.intense( self.rawx, self.rawy, @@ -282,7 +282,7 @@ class EyeProcessor: self.eyeopen = 0.0 else: self.eyeopen = ibo - + print(self.eyeopen) self.output_images_and_update( self.thresh, EyeInfo(self.current_algo, self.out_x, self.out_y, 0, self.eyeopen), @@ -293,7 +293,7 @@ class EyeProcessor: def LEAPM(self): self.thresh = self.current_image_gray.copy() - self.current_image_gray, self.rawx, self.rawy, self.eyeopen = self.er_leap.run(self.current_image_gray) + self.current_image_gray, self.rawx, self.rawy, self.eyeopen = self.er_leap.run(self.current_image_gray, True) self.thresh = self.current_image_gray.copy() self.out_x, self.out_y = cal.cal_osc(self, self.rawx, self.rawy) self.current_algorithm = EyeInfoOrigin.LEAP @@ -328,7 +328,7 @@ class EyeProcessor: self.rawx, self.rawy, self.thresh, self.radius = self.er_hsf.run( self.current_image_gray ) - self.rawx, self.rawy, self.thresh = RANSAC3D(self, True) + self.rawx, self.rawy, self.thresh, self.eyeopen = RANSAC3D(self, True) # print(self.radius) # if self.prev_x is None: # self.prev_x = self.rawx @@ -371,7 +371,7 @@ class EyeProcessor: current_image_gray_copy = ( self.current_image_gray.copy() ) # Duplicate before overwriting in RANSAC3D. - self.rawx, self.rawy, self.thresh = RANSAC3D(self, False) + self.rawx, self.rawy, self.thresh, self.eyeopen = RANSAC3D(self, False) self.out_x, self.out_y = cal.cal_osc(self, self.rawx, self.rawy) self.current_algorithm = EyeInfoOrigin.RANSAC diff --git a/EyeTrackApp/intensity_based_openness.py b/EyeTrackApp/intensity_based_openness.py index d3b3a72..b1072c3 100644 --- a/EyeTrackApp/intensity_based_openness.py +++ b/EyeTrackApp/intensity_based_openness.py @@ -260,16 +260,16 @@ class IntensityBasedOpeness: self.filterlist.append(intensity) if intensity >= np.percentile( - self.filterlist, 99 + self.filterlist, 98 ): # filter abnormally high values # print('filter, assume blink') intensity = self.maxval - # if intensity <= np.percentile( # TODO test this - # self.filterlist, 1 - # ): # filter abnormally low values + if intensity <= np.percentile( # TODO test this + self.filterlist, 0.3 + ): # filter abnormally low values # print('filter, assume blink') - # intensity = self.maxval + intensity = self.data[int_y, int_x] # self.tri_filter.append(intensity) # if len(self.tri_filter) > 3: @@ -330,11 +330,12 @@ class IntensityBasedOpeness: changed = True else: intensitya = max( - data_val + 0.001, 1 - ) # if current intensity value is not less use - self.data[int_y, int_x] = intensitya # set value + data_val + 5000, 1 + ) # if current intensity value is not less use this is an agressive adjust, test + self.data[int_y, int_x] = intensitya # set value changed = True + # min pupil global if self.maxval == 0: # that value is not yet saved self.maxval = intensity # set value at 0 index @@ -353,8 +354,8 @@ class IntensityBasedOpeness: # Do the same thing as in the original version. eyeopen = self.prev_val # 0.9 else: - maxp = self.data[int_y, int_x] - minp = self.maxval + maxp = float(self.data[int_y, int_x]) + minp = float(self.maxval) eyeopen = (intensity - maxp) / ( minp - maxp @@ -362,23 +363,6 @@ class IntensityBasedOpeness: eyeopen = 1 - eyeopen - if len(self.openlist) < 1000: # TODO expose as setting? - self.openlist.append(eyeopen) - else: - if eyeopen >= np.percentile(self.openlist, 99) or eyeopen <= np.percentile(self.openlist, 1): - pass - else: - self.openlist.pop(0) - self.openlist.append(eyeopen) - - - try: - per = ((eyeopen - max(self.openlist)) / (min(self.openlist) - max(self.openlist))) - eyeopen = 1 - per - except: - pass - - if outputSamples > 0: if len(self.averageList) < outputSamples: self.averageList.append(eyeopen) @@ -387,7 +371,11 @@ class IntensityBasedOpeness: self.averageList.append(eyeopen) eyeopen = np.average(self.averageList) + if eyeopen > 1: # clamp values + eyeopen = 1.0 + if eyeopen < 0: + eyeopen = 0.0 diff --git a/EyeTrackApp/leap.py b/EyeTrackApp/leap.py index 25d678d..0279ae1 100644 --- a/EyeTrackApp/leap.py +++ b/EyeTrackApp/leap.py @@ -124,6 +124,8 @@ class LEAP_C(object): self.openlist = [] self.x = 0 self.y = 0 + self.frame_lim_count = 0 + self.last_lid = 0.7 self.ort_session1 = onnxruntime.InferenceSession( @@ -149,6 +151,14 @@ class LEAP_C(object): break def leap_run(self): + if self.lid_only == True: + if self.frame_lim_count > 0: + self.frame_lim_count = 0 + return self.current_image_gray, float(0), float(0), self.last_lid, + else: + self.frame_lim_count += 1 + pass + img = self.current_image_gray.copy() img = cv2.cvtColor(img, cv2.COLOR_GRAY2RGB) # img = imutils.rotate(img, angle=320) @@ -188,10 +198,14 @@ class LEAP_C(object): per = ((d - max(self.openlist)) / (min(self.openlist) - max(self.openlist))) per = 1 - per except: + per = 0.7 pass x = pre_landmark[17][0] y = pre_landmark[17][1] frame = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY) + + self.last_lid = per + return frame, float(x), float(y), per @@ -202,7 +216,8 @@ class External_Run_LEAP(object): def __init__(self): self.algo = LEAP_C() - def run(self, current_image_gray): + def run(self, current_image_gray, trackmode): + self.algo.lid_only = trackmode self.algo.current_image_gray = current_image_gray img, x, y, per = self.algo.leap_run() return img, x, y, per \ No newline at end of file diff --git a/EyeTrackApp/ransac.py b/EyeTrackApp/ransac.py index 9b2320c..c76364c 100644 --- a/EyeTrackApp/ransac.py +++ b/EyeTrackApp/ransac.py @@ -164,6 +164,7 @@ cct = 300 def RANSAC3D(self, hsrac_en): f = False ranf = False + blink = 0.7 if hsrac_en: center_x, center_y, upper_x, lower_x, upper_y, lower_y, ransac_lower_x, ransac_lower_y, ransac_upper_x, ransac_upper_y, ransac_xy_offset = get_center_noclamp( @@ -311,6 +312,15 @@ def RANSAC3D(self, hsrac_en): cy = int(clamp(cy + ransac_lower_y, 0, csy)) + #print(contours) + for cnt in contours: + (x, y, w, h) = cv2.boundingRect(cnt) + perscalarw = w / csx + perscalarh = h / csy + # print(abs(perscalarw-perscalarh)) + if abs(perscalarw-perscalarh) >= 0.26: # TODO setting + blink = 0.0 + try: cv2.drawContours(self.current_image_gray, contours, -1, (255, 0, 0), 1) # TODO: fix visualizations with HSRAC @@ -363,7 +373,7 @@ def RANSAC3D(self, hsrac_en): thresh = cv2.resize(thresh, (x,y)) try: self.failed = 0 # we have succeded, continue with this - return cx, cy, thresh + return cx, cy, thresh, blink except: self.failed = self.failed + 1 #we have failed, move onto next algo - return 0, 0, thresh \ No newline at end of file + return 0, 0, thresh, blink \ No newline at end of file