mirror of
https://github.com/EyeTrackVR/EyeTrackVR.git
synced 2025-11-04 14:39:42 +08:00
fix: LEAP Eyelid, tune RANSAC BLINK
This commit is contained in:
parent
03a1a965e5
commit
5a6248b897
@ -249,7 +249,7 @@ class EyeProcessor:
|
|||||||
pass
|
pass
|
||||||
|
|
||||||
def UPDATE(self):
|
def UPDATE(self):
|
||||||
|
# print(self.eyeopen)
|
||||||
if self.settings.gui_BLINK:
|
if self.settings.gui_BLINK:
|
||||||
self.eyeopen = BLINK(self)
|
self.eyeopen = BLINK(self)
|
||||||
|
|
||||||
@ -269,6 +269,7 @@ class EyeProcessor:
|
|||||||
self.eyeopen = 0.0
|
self.eyeopen = 0.0
|
||||||
|
|
||||||
if self.bd_blink == True:
|
if self.bd_blink == True:
|
||||||
|
prin("blinks")
|
||||||
pass
|
pass
|
||||||
|
|
||||||
if self.settings.gui_IBO and self.eyeopen != 0.0:
|
if self.settings.gui_IBO and self.eyeopen != 0.0:
|
||||||
@ -280,17 +281,17 @@ class EyeProcessor:
|
|||||||
self.settings.ibo_average_output_samples,
|
self.settings.ibo_average_output_samples,
|
||||||
)
|
)
|
||||||
|
|
||||||
if self.settings.gui_LEAP_lid:
|
if self.settings.gui_LEAP_lid and self.eyeopen != 0.0:
|
||||||
(
|
(
|
||||||
self.current_image_gray,
|
self.current_image_gray,
|
||||||
self.rawx,
|
self.rawx,
|
||||||
self.rawy,
|
self.rawy,
|
||||||
self.eyeopen,
|
self.eyeopen,
|
||||||
) = self.er_leap.run(self.current_image_gray)
|
) = self.er_leap.run(self.current_image_gray)
|
||||||
print(self.eyeopen)
|
# print(self.eyeopen)
|
||||||
|
|
||||||
if (
|
if (
|
||||||
len(self.prev_y_list) >= 200
|
len(self.prev_y_list) >= 100
|
||||||
): # "lock" eye when close/blink IN TESTING, kinda broke
|
): # "lock" eye when close/blink IN TESTING, kinda broke
|
||||||
self.prev_y_list.pop(0)
|
self.prev_y_list.pop(0)
|
||||||
self.prev_y_list.append(self.out_y)
|
self.prev_y_list.append(self.out_y)
|
||||||
@ -300,10 +301,10 @@ class EyeProcessor:
|
|||||||
# print(abs(self.eyeopen - self.past_blink))
|
# print(abs(self.eyeopen - self.past_blink))
|
||||||
blink_vec = min(abs(self.eyeopen - self.past_blink), 1) # clamp to 1
|
blink_vec = min(abs(self.eyeopen - self.past_blink), 1) # clamp to 1
|
||||||
|
|
||||||
# if blink_vec >= 0.17:
|
#if blink_vec >= 0.2:
|
||||||
# if blink_vec >= 0.1 or blink_vec == 0.0 and (self.out_y - self.prev_y) < 0.0:
|
if blink_vec >= 0.15 or blink_vec == 0.0 and (self.out_y - self.prev_y) < 0.0:
|
||||||
# self.out_x = sum(self.prev_x_list) / len(self.prev_x_list)
|
#self.out_x = sum(self.prev_x_list) / len(self.prev_x_list)
|
||||||
# self.out_y = sum(self.prev_y_list) / len(self.prev_y_list)
|
self.out_y = sum(self.prev_y_list) / len(self.prev_y_list)
|
||||||
# print('AVG', self.out_y, len(self.prev_y_list))
|
# print('AVG', self.out_y, len(self.prev_y_list))
|
||||||
|
|
||||||
self.past_blink = self.eyeopen
|
self.past_blink = self.eyeopen
|
||||||
@ -317,7 +318,7 @@ class EyeProcessor:
|
|||||||
if self.settings.gui_RANSACBLINK and self.eyeopen == 0.0:
|
if self.settings.gui_RANSACBLINK and self.eyeopen == 0.0:
|
||||||
pass
|
pass
|
||||||
else:
|
else:
|
||||||
self.eyeopen = 0.9
|
self.eyeopen = 0.8
|
||||||
|
|
||||||
def BLINKM(self):
|
def BLINKM(self):
|
||||||
self.eyeopen = BLINK(self)
|
self.eyeopen = BLINK(self)
|
||||||
@ -330,7 +331,7 @@ class EyeProcessor:
|
|||||||
self.thresh = self.current_image_gray.copy()
|
self.thresh = self.current_image_gray.copy()
|
||||||
self.out_x, self.out_y = cal.cal_osc(self, self.rawx, self.rawy)
|
self.out_x, self.out_y = cal.cal_osc(self, self.rawx, self.rawy)
|
||||||
self.current_algorithm = EyeInfoOrigin.LEAP
|
self.current_algorithm = EyeInfoOrigin.LEAP
|
||||||
print(self.eyeopen)
|
# print(self.eyeopen)
|
||||||
|
|
||||||
def DADDYM(self):
|
def DADDYM(self):
|
||||||
# todo: We should have a proper variable for drawing.
|
# todo: We should have a proper variable for drawing.
|
||||||
@ -363,6 +364,7 @@ class EyeProcessor:
|
|||||||
self.rawx, self.rawy, self.thresh, ranblink = RANSAC3D(self, True)
|
self.rawx, self.rawy, self.thresh, ranblink = RANSAC3D(self, True)
|
||||||
if self.settings.gui_RANSACBLINK: # might be redundant
|
if self.settings.gui_RANSACBLINK: # might be redundant
|
||||||
self.eyeopen = ranblink
|
self.eyeopen = ranblink
|
||||||
|
# print("RANBLINK", ranblink)
|
||||||
|
|
||||||
# print(self.radius)
|
# print(self.radius)
|
||||||
# if self.prev_x is None:
|
# if self.prev_x is None:
|
||||||
|
|||||||
@ -121,13 +121,16 @@ class LEAP_C(object):
|
|||||||
process.nice(psutil.BELOW_NORMAL_PRIORITY_CLASS) # Windows
|
process.nice(psutil.BELOW_NORMAL_PRIORITY_CLASS) # Windows
|
||||||
process.nice()
|
process.nice()
|
||||||
# See https://learn.microsoft.com/en-us/windows/win32/api/processthreadsapi/nf-processthreadsapi-getpriorityclass#return-value for values
|
# See https://learn.microsoft.com/en-us/windows/win32/api/processthreadsapi/nf-processthreadsapi-getpriorityclass#return-value for values
|
||||||
min_cutoff = 0.04
|
min_cutoff = 0.9
|
||||||
beta = 0.9
|
beta = 5.0
|
||||||
# print(np.random.rand(22, 2))
|
# print(np.random.rand(22, 2))
|
||||||
# noisy_point = np.array([1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1])
|
# noisy_point = np.array([1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1])
|
||||||
one_euro_filter = OneEuroFilter(
|
self.one_euro_filter = OneEuroFilter(
|
||||||
np.random.rand(7, 2), min_cutoff=min_cutoff, beta=beta
|
np.random.rand(7, 2), min_cutoff=min_cutoff, beta=beta
|
||||||
)
|
)
|
||||||
|
#self.one_euro_filter_open = OneEuroFilter(
|
||||||
|
# np.random.rand(1, 2), min_cutoff=0.01, beta=0.04
|
||||||
|
#)
|
||||||
self.dmax = 0
|
self.dmax = 0
|
||||||
self.dmin = 0
|
self.dmin = 0
|
||||||
self.openlist = []
|
self.openlist = []
|
||||||
@ -174,6 +177,7 @@ class LEAP_C(object):
|
|||||||
if not self.output_queue.empty():
|
if not self.output_queue.empty():
|
||||||
|
|
||||||
frame, pre_landmark = self.output_queue.get()
|
frame, pre_landmark = self.output_queue.get()
|
||||||
|
pre_landmark = self.one_euro_filter(pre_landmark)
|
||||||
# frame = cv2.resize(frame, (112, 112))
|
# frame = cv2.resize(frame, (112, 112))
|
||||||
|
|
||||||
for point in pre_landmark:
|
for point in pre_landmark:
|
||||||
@ -220,16 +224,16 @@ class LEAP_C(object):
|
|||||||
# d2 = math.dist(pre_landmark[2], pre_landmark[4])
|
# d2 = math.dist(pre_landmark[2], pre_landmark[4])
|
||||||
# d = d + d2
|
# d = d + d2
|
||||||
|
|
||||||
if len(self.openlist) < 2000: # TODO expose as setting?
|
if len(self.openlist) < 5000: # TODO expose as setting?
|
||||||
self.openlist.append(d)
|
self.openlist.append(d)
|
||||||
else:
|
else:
|
||||||
if d >= np.percentile(self.openlist, 99) or d <= np.percentile(
|
# if d >= np.percentile(self.openlist, 99) or d <= np.percentile(
|
||||||
self.openlist, 1
|
# self.openlist, 1
|
||||||
):
|
# ):
|
||||||
pass
|
# pass
|
||||||
else:
|
#else:
|
||||||
self.openlist.pop(0)
|
self.openlist.pop(0)
|
||||||
self.openlist.append(d)
|
self.openlist.append(d)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
per = (d - max(self.openlist)) / (
|
per = (d - max(self.openlist)) / (
|
||||||
@ -239,15 +243,18 @@ class LEAP_C(object):
|
|||||||
except:
|
except:
|
||||||
per = 0.7
|
per = 0.7
|
||||||
pass
|
pass
|
||||||
|
# print(d, per)
|
||||||
x = pre_landmark[6][0]
|
x = pre_landmark[6][0]
|
||||||
y = pre_landmark[6][1]
|
y = pre_landmark[6][1]
|
||||||
frame = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
|
frame = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
|
||||||
|
|
||||||
per = d - 0.1
|
# per = d - 0.1
|
||||||
self.last_lid = per
|
self.last_lid = per
|
||||||
if per <= 0.1:
|
# pera = np.array([per, per])
|
||||||
|
#self.one_euro_filter_open(pera)
|
||||||
|
if per <= 0.2: #TODO: EXPOSE AS SETTING
|
||||||
per == 0.0
|
per == 0.0
|
||||||
# print(per)
|
# print(per)
|
||||||
return frame, float(x), float(y), per
|
return frame, float(x), float(y), per
|
||||||
|
|
||||||
frame = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
|
frame = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
|
||||||
|
|||||||
@ -48,8 +48,8 @@ def output_osc(eye_x, eye_y, eye_blink, last_blink, self):
|
|||||||
self.l_eye_blink = eye_blink
|
self.l_eye_blink = eye_blink
|
||||||
|
|
||||||
if self.l_eye_blink == 0.0:
|
if self.l_eye_blink == 0.0:
|
||||||
if last_blink > 0.2: #when binary blink is on, blinks may be too fast for OSC so we repeat them.
|
if last_blink > 0.15: #when binary blink is on, blinks may be too fast for OSC so we repeat them.
|
||||||
for i in range(5):
|
for i in range(4):
|
||||||
self.client.send_message(self.config.osc_left_eye_close_address, eyelid_transformer(self,self.l_eye_blink))
|
self.client.send_message(self.config.osc_left_eye_close_address, eyelid_transformer(self,self.l_eye_blink))
|
||||||
last_blink = time.time() - last_blink
|
last_blink = time.time() - last_blink
|
||||||
if self.config.gui_eye_falloff:
|
if self.config.gui_eye_falloff:
|
||||||
@ -69,8 +69,9 @@ def output_osc(eye_x, eye_y, eye_blink, last_blink, self):
|
|||||||
self.r_eye_blink = eye_blink
|
self.r_eye_blink = eye_blink
|
||||||
|
|
||||||
if self.r_eye_blink == 0.0:
|
if self.r_eye_blink == 0.0:
|
||||||
if last_blink > 0.2: #when binary blink is on, blinks may be too fast for OSC so we repeat them.
|
if last_blink > 0.15: #when binary blink is on, blinks may be too fast for OSC so we repeat them.
|
||||||
for i in range(5):
|
print("REPEATING R BLINK")
|
||||||
|
for i in range(4):
|
||||||
self.client.send_message(self.config.osc_right_eye_close_address, eyelid_transformer(self,self.r_eye_blink))
|
self.client.send_message(self.config.osc_right_eye_close_address, eyelid_transformer(self,self.r_eye_blink))
|
||||||
last_blink = time.time() - last_blink
|
last_blink = time.time() - last_blink
|
||||||
if self.config.gui_eye_falloff:
|
if self.config.gui_eye_falloff:
|
||||||
|
|||||||
@ -435,8 +435,9 @@ def RANSAC3D(self, hsrac_en):
|
|||||||
elif len(self.blink_list) < 10000:
|
elif len(self.blink_list) < 10000:
|
||||||
self.blink_list.append(abs(perscalarw - perscalarh))
|
self.blink_list.append(abs(perscalarw - perscalarh))
|
||||||
|
|
||||||
if abs(perscalarw - perscalarh) >= np.percentile(self.blink_list, 94):
|
if abs(perscalarw - perscalarh) >= np.percentile(self.blink_list, 92):
|
||||||
blink = 0.0
|
blink = 0.0
|
||||||
|
print("RANSAC BLINK")
|
||||||
|
|
||||||
try:
|
try:
|
||||||
cv2.drawContours(
|
cv2.drawContours(
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user