From b11485f3fba155318e6564359dacc88c8b71ae38 Mon Sep 17 00:00:00 2001 From: Prohurtz <48768484+RedHawk989@users.noreply.github.com> Date: Mon, 22 Apr 2024 21:17:47 -0500 Subject: [PATCH] fix: bump binary blink open value --- EyeTrackApp/AHSF.py | 2 +- EyeTrackApp/blink.py | 23 +++++--- EyeTrackApp/daddy.py | 123 +++++++++++++++++++++--------------------- EyeTrackApp/ransac.py | 2 +- 4 files changed, 80 insertions(+), 70 deletions(-) diff --git a/EyeTrackApp/AHSF.py b/EyeTrackApp/AHSF.py index 096fb38..9ad50ea 100644 --- a/EyeTrackApp/AHSF.py +++ b/EyeTrackApp/AHSF.py @@ -965,7 +965,7 @@ def External_Run_AHSF(frame_gray): # (pupil_rect_coarse[0], pupil_rect_coarse[1]), # ( # pupil_rect_coarse[0] + pupil_rect_coarse[2], - # pupil_rect_coarse[1] + pupil_rect_coarse[3], + # pupil_rect_coarse[1] + pupil_rect_coarse[so 3], # ), # (0, 255, 0), # 2, diff --git a/EyeTrackApp/blink.py b/EyeTrackApp/blink.py index 1543103..095ab33 100644 --- a/EyeTrackApp/blink.py +++ b/EyeTrackApp/blink.py @@ -29,7 +29,8 @@ LICENSE: Summer Software Distribution License 1.0 import numpy as np -def BLINK(self): + +def BLINK(self): if self.blink_clear == True: self.max_ints = [] @@ -39,14 +40,18 @@ def BLINK(self): intensity = np.sum(self.current_image_gray_clean) if self.calibration_frame_counter == 300: - self.filterlist = [] #clear filter + self.filterlist = [] # clear filter if len(self.filterlist) < 300: self.filterlist.append(intensity) else: self.filterlist.pop(0) self.filterlist.append(intensity) - if intensity >= np.percentile(self.filterlist, 99) or intensity <= np.percentile(self.filterlist, 1) and len(self.max_ints) >= 1: # filter abnormally high values - try: # I don't want this here but I cant get python to stop crying when it's not + if ( + intensity >= np.percentile(self.filterlist, 99) + or intensity <= np.percentile(self.filterlist, 1) + and len(self.max_ints) >= 1 + ): # filter abnormally high values + try: # I don't want this here but I cant get python to stop crying when it's not intensity = min(self.max_ints) except: pass @@ -54,7 +59,7 @@ def BLINK(self): self.frames = self.frames + 1 if intensity > self.max_int: self.max_int = intensity - if self.frames > 300: #TODO: test this number more (make it a setting??) + if self.frames > 300: # TODO: test this number more (make it a setting??) self.max_ints.append(self.max_int) if intensity < self.min_int: self.min_int = intensity @@ -63,9 +68,11 @@ def BLINK(self): if intensity > min(self.max_ints): blinkvalue = 0.0 else: - blinkvalue = 0.7 + blinkvalue = 0.8 try: return blinkvalue except: - return 0.7 - # print(self.blinkvalue, self.max_int, self.min_int, self.frames, intensity) \ No newline at end of file + return 0.8 + + +# print(self.blinkvalue, self.max_int, self.min_int, self.frames, intensity) diff --git a/EyeTrackApp/daddy.py b/EyeTrackApp/daddy.py index 3d99ba5..27bc073 100644 --- a/EyeTrackApp/daddy.py +++ b/EyeTrackApp/daddy.py @@ -1,4 +1,4 @@ -''' +""" ------------------------------------------------------------------------------------------------------ ,@@@@@@ @@ -25,7 +25,7 @@ Algorithm App Implementations By: PallasNeko, Prohurtz Copyright (c) 2023 EyeTrackVR <3 LICENSE: GNU GPLv3 ------------------------------------------------------------------------------------------------------ -''' +""" import sys from typing import Tuple import math @@ -36,6 +36,7 @@ import onnxruntime from one_euro_filter import OneEuroFilter from utils.misc_utils import FastMedian, resource_path import os + os.environ["OMP_NUM_THREADS"] = "1" # DADDY # Please change the name of this script and the name of the method if you have something better. @@ -44,15 +45,16 @@ input_size = 192 # Do not change this number. heatmap_size = 48 # Do not change this number. kernel_size = 7 if platform.system() == "Darwin": - model_file = "EyeTrackApp/Models/daddy230210.onnx" # The model file name will be changed when performance stabilises. # funny MacOS files issues :P + model_file = "Models/daddy230210.onnx" # The model file name will be changed when performance stabilises. # funny MacOS files issues :P else: model_file = "Models/daddy230210.onnx" # The model file name will be changed when performance stabilises. + # SHA256 for model version verification # daddy230210.onnx = 59e59aa2a21024884200dd3acbd5e6a2e8d7209c46555fbdc727d4fe3adb68d3 imshow_enable = False save_video = False -save_filepath = 'output.mp4' +save_filepath = "output.mp4" def get_max_preds(batch_heatmaps): @@ -63,18 +65,18 @@ def get_max_preds(batch_heatmaps): heatmaps_reshaped = batch_heatmaps.reshape((batch_size, num_joints, -1)) idx = np.argmax(heatmaps_reshaped, 2) maxvals = np.amax(heatmaps_reshaped, 2) - + maxvals = maxvals.reshape((batch_size, num_joints, 1)) idx = idx.reshape((batch_size, num_joints, 1)) - + preds = np.tile(idx, (1, 1, 2)).astype(np.float32) - + preds[:, :, 0] = (preds[:, :, 0]) % width preds[:, :, 1] = np.floor((preds[:, :, 1]) / width) - + pred_mask = np.tile(np.greater(maxvals, 0.0), (1, 1, 2)) pred_mask = pred_mask.astype(np.float32) - + preds *= pred_mask return preds, maxvals @@ -89,12 +91,11 @@ def taylor(hm, coord): dx = 0.5 * (hm[py][px + 1] - hm[py][px - 1]) dy = 0.5 * (hm[py + 1][px] - hm[py - 1][px]) dxx = 0.25 * (hm[py][px + 2] - 2 * hm[py][px] + hm[py][px - 2]) - dxy = 0.25 * (hm[py + 1][px + 1] - hm[py - 1][px + 1] - hm[py + 1][px - 1] \ - + hm[py - 1][px - 1]) + dxy = 0.25 * (hm[py + 1][px + 1] - hm[py - 1][px + 1] - hm[py + 1][px - 1] + hm[py - 1][px - 1]) dyy = 0.25 * (hm[py + 2 * 1][px] - 2 * hm[py][px] + hm[py - 2 * 1][px]) derivative = np.matrix([[dx], [dy]]) hessian = np.matrix([[dxx, dxy], [dxy, dyy]]) - if dxx * dyy - dxy ** 2 != 0: + if dxx * dyy - dxy**2 != 0: hessianinv = hessian.I offset = -hessianinv * derivative offset = np.squeeze(np.array(offset.T), axis=0) @@ -113,9 +114,9 @@ def gaussian_blur(hm, kernel): for j in range(num_joints): origin_max = np.max(hm[i, j]) dr = np.zeros((height + 2 * border, width + 2 * border)) - dr[border: -border, border: -border] = hm[i, j].copy() + dr[border:-border, border:-border] = hm[i, j].copy() dr = cv2.GaussianBlur(dr, (kernel, kernel), 0) - hm[i, j] = dr[border: -border, border: -border].copy() + hm[i, j] = dr[border:-border, border:-border].copy() hm[i, j] *= origin_max / np.max(hm[i, j]) return hm @@ -123,7 +124,7 @@ def gaussian_blur(hm, kernel): def get_final_preds(hm, realsize): # base:https://github.com/ilovepose/DarkPose coords, maxvals = get_max_preds(hm) - + # post-processing hm = gaussian_blur(hm, kernel_size) hm = np.maximum(hm, 1e-10) @@ -131,22 +132,22 @@ def get_final_preds(hm, realsize): for n in range(coords.shape[0]): for p in range(coords.shape[1]): coords[n, p] = taylor(hm[n][p], coords[n][p]) - + preds = coords.copy() preds = (preds / heatmap_size) * realsize # input_size - + # Transform back # for i in range(coords.shape[0]): # preds[i] = transform_preds( # coords[i], center[i], scale[i], [heatmap_width, heatmap_height] # ) - + return preds, maxvals -def resize_with_pad(image: np.array, - new_shape: Tuple[int, int], - padding_color: Tuple[int] = (255, 255, 255)) -> np.array: +def resize_with_pad( + image: np.array, new_shape: Tuple[int, int], padding_color: Tuple[int] = (255, 255, 255) +) -> np.array: """ https://gist.github.com/IdeaKing/11cf5e146d23c5bb219ba3508cca89ec Maintains aspect ratio and resizes with padding. @@ -178,29 +179,32 @@ class BEER(object): self.p03_med = FastMedian(k=256) self.prev_ear = 0.5 # todo https://peerj.com/articles/cs-943/ - + def ear(self, pred): - p15 = np.linalg.norm(pred[1]-pred[5]) - p24 = np.linalg.norm(pred[2]-pred[4]) - p03 = np.linalg.norm(pred[0]-pred[3]) - self.p03_med+p03 - if p03 > self.p03_med.median()*1.5: + p15 = np.linalg.norm(pred[1] - pred[5]) + p24 = np.linalg.norm(pred[2] - pred[4]) + p03 = np.linalg.norm(pred[0] - pred[3]) + self.p03_med + p03 + if p03 > self.p03_med.median() * 1.5: return self.prev_ear - ear = (p15+p24)/(2*self.p03_med.median()) + ear = (p15 + p24) / (2 * self.p03_med.median()) self.ear_minmax(ear) norm_ear = self.ear_norm(ear) self.prev_ear = norm_ear.copy() return norm_ear - def ear_minmax(self,ear): + + def ear_minmax(self, ear): if ear < self.ear_min: self.ear_min = ear.copy() if ear > self.ear_max: self.ear_max = ear.copy() - - def ear_norm(self,ear): - return (ear-self.ear_min)/(self.ear_max-self.ear_min) # todo:It is better to add very small values to avoid zero division. - + + def ear_norm(self, ear): + return (ear - self.ear_min) / ( + self.ear_max - self.ear_min + ) # todo:It is better to add very small values to avoid zero division. + # # loopnum = 0 @@ -215,35 +219,33 @@ class DADDY_cls(object): options.intra_op_num_threads = 1 # This number should be changed accordingly options.execution_mode = onnxruntime.ExecutionMode.ORT_SEQUENTIAL options.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_ALL - - ort_session = onnxruntime.InferenceSession(resource_path(model_file), sess_options=options, providers=["CPUExecutionProvider"]) - ort_session.set_providers(['CPUExecutionProvider']) # only cpu mode - + + ort_session = onnxruntime.InferenceSession( + resource_path(model_file), sess_options=options, providers=["CPUExecutionProvider"] + ) + ort_session.set_providers(["CPUExecutionProvider"]) # only cpu mode + self.ort_session = ort_session self.input_name = ort_session.get_inputs()[0].name self.output_name = ort_session.get_outputs()[0].name - + min_cutoff = 0.0004 beta = 0.9 input_point = np.zeros((11, 2)) # np.array([1, 1]) - self.one_euro_filter = OneEuroFilter( - input_point, - min_cutoff=min_cutoff, - beta=beta - ) + self.one_euro_filter = OneEuroFilter(input_point, min_cutoff=min_cutoff, beta=beta) # self.ear_oef = OneEuroFilter( # np.zeros(1), # min_cutoff=min_cutoff, # beta=beta # ) # memo: Parameters need tuning - + self.beer = BEER() - + # filepath = 'test.mp4' # codec = cv2.VideoWriter_fourcc(*"mp4v") # video = cv2.VideoWriter(filepath, codec, 60.0, (200, 150), 0) # (60, 60)) # (150, 200)) # self.video = video - + def open_video(self, video_path): # Temporary implementation to run cap = cv2.VideoCapture(video_path) @@ -251,7 +253,7 @@ class DADDY_cls(object): raise IOError("Error opening video stream or file") self.cap = cap return True - + def read_frame(self): # Temporary implementation to run if not self.cap.isOpened(): @@ -263,26 +265,28 @@ class DADDY_cls(object): self.current_image_gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) return True return False - + def single_run(self): # Temporary implementation to run - + # todo: If it's the left hand eye, flip the image left to right. - + gray_frame = self.current_image_gray.copy() - + # frame_resize=resize_with_pad(gray_frame,(input_size,input_size)) # or frame_resize = cv2.resize(gray_frame, (input_size, input_size)) imgs = np.divide(frame_resize[np.newaxis, np.newaxis], 255, dtype=np.float32) # input/255.0 - + pred_heatmap = self.ort_session.run(None, {self.input_name: imgs})[0] # .reshape((-1, 2)) # if imshow_enable: # heatmap = pred_heatmap.reshape((-1, heatmap_size, heatmap_size)) # for i in range(heatmap.shape[0]): # cv2.imshow("heatmap_{}".format(i + 1), heatmap[i]) - - pred, max_val = get_final_preds(pred_heatmap, (self.current_image_gray.shape[1], self.current_image_gray.shape[0])) + + pred, max_val = get_final_preds( + pred_heatmap, (self.current_image_gray.shape[1], self.current_image_gray.shape[0]) + ) pred = pred.reshape((-1, 2)) # or # pred, max_val = get_final_preds(pred_heatmap, input_size) @@ -295,7 +299,7 @@ class DADDY_cls(object): pred = self.one_euro_filter(pred) kps = pred.astype(np.int32) - + # eyecenter = kps[:6].mean(axis=0).astype(int) ear = self.beer.ear(pred) # ear=self.ear_oef(ear[np.newaxis])#memo: Parameters need tuning @@ -316,7 +320,6 @@ class DADDY_cls(object): # cv2.putText(self.current_image_gray, str(i), (kps[i, 0] - 10, kps[i, 1] - 10), cv2.FONT_HERSHEY_SIMPLEX, 0.35, (0, 0, 255), 1) # cv2.putText(self.current_image_gray, "EAR: "+str(ear), (self.current_image_gray.shape[1]//10, self.current_image_gray.shape[0]//10), cv2.FONT_HERSHEY_SIMPLEX, 0.75, (255,0,0), 1) - # global loopnum # if loopnum < 1350*2: # # self.video.write(cv2.resize(gray_frame.copy(), (200, 150), None)) @@ -327,19 +330,19 @@ class DADDY_cls(object): # sys.exit() # if w_video: # video.release() - + # kps[i, :] = (x, y) # i == [0:6] = Inner and outer corners of eyes and eyelids # i == [6] = pupil # i == [7:] = iris - + return pupil_center_x, pupil_center_y, ear class External_Run_DADDY(object): def __init__(self): self.algo = DADDY_cls() - + def run(self, current_image_gray): self.algo.current_image_gray = current_image_gray pupil_x, pupil_y, ear = self.algo.single_run() @@ -350,4 +353,4 @@ if __name__ == "__main__": daddy = DADDY_cls() daddy.open_video(video_path) while daddy.read_frame(): - _ = daddy.single_run() \ No newline at end of file + _ = daddy.single_run() diff --git a/EyeTrackApp/ransac.py b/EyeTrackApp/ransac.py index aa320f6..3eb518a 100644 --- a/EyeTrackApp/ransac.py +++ b/EyeTrackApp/ransac.py @@ -213,7 +213,7 @@ cct = 300 def RANSAC3D(self, hsrac_en): f = False ranf = False - blink = 0.7 + blink = 0.8 angle = 0 if hsrac_en: