diff --git a/EyeTrackApp/daddy.py b/EyeTrackApp/daddy.py new file mode 100644 index 0000000..8b5740a --- /dev/null +++ b/EyeTrackApp/daddy.py @@ -0,0 +1,322 @@ +import sys +from typing import Tuple +import math + +import numpy as np +import cv2 +import onnxruntime + +from one_euro_filter import OneEuroFilter +from utils.misc_utils import FastMedian + +# DADDY +# Please change the name of this script and the name of the method if you have something better. +video_path = "ezgif.com-gif-maker.avi" +input_size = 192 # Do not change this number. +heatmap_size = 48 # Do not change this number. +kernel_size = 7 +model_file = "daddy230210.onnx" # The model file name will be changed when performance stabilises. +# SHA256 for model version verification +# daddy230210.onnx = 59e59aa2a21024884200dd3acbd5e6a2e8d7209c46555fbdc727d4fe3adb68d3 +imshow_enable = False +save_video = False +save_filepath = 'output.mp4' + + +def get_max_preds(batch_heatmaps): + # base:https://github.com/ilovepose/DarkPose + batch_size = batch_heatmaps.shape[0] + num_joints = batch_heatmaps.shape[1] + width = batch_heatmaps.shape[3] + heatmaps_reshaped = batch_heatmaps.reshape((batch_size, num_joints, -1)) + idx = np.argmax(heatmaps_reshaped, 2) + maxvals = np.amax(heatmaps_reshaped, 2) + + maxvals = maxvals.reshape((batch_size, num_joints, 1)) + idx = idx.reshape((batch_size, num_joints, 1)) + + preds = np.tile(idx, (1, 1, 2)).astype(np.float32) + + preds[:, :, 0] = (preds[:, :, 0]) % width + preds[:, :, 1] = np.floor((preds[:, :, 1]) / width) + + pred_mask = np.tile(np.greater(maxvals, 0.0), (1, 1, 2)) + pred_mask = pred_mask.astype(np.float32) + + preds *= pred_mask + return preds, maxvals + + +def taylor(hm, coord): + # base:https://github.com/ilovepose/DarkPose + heatmap_height = hm.shape[0] + heatmap_width = hm.shape[1] + px = int(coord[0]) + py = int(coord[1]) + if 1 < px < heatmap_width - 2 and 1 < py < heatmap_height - 2: + dx = 0.5 * (hm[py][px + 1] - hm[py][px - 1]) + dy = 0.5 * (hm[py + 1][px] - hm[py - 1][px]) + dxx = 0.25 * (hm[py][px + 2] - 2 * hm[py][px] + hm[py][px - 2]) + dxy = 0.25 * (hm[py + 1][px + 1] - hm[py - 1][px + 1] - hm[py + 1][px - 1] \ + + hm[py - 1][px - 1]) + dyy = 0.25 * (hm[py + 2 * 1][px] - 2 * hm[py][px] + hm[py - 2 * 1][px]) + derivative = np.matrix([[dx], [dy]]) + hessian = np.matrix([[dxx, dxy], [dxy, dyy]]) + if dxx * dyy - dxy ** 2 != 0: + hessianinv = hessian.I + offset = -hessianinv * derivative + offset = np.squeeze(np.array(offset.T), axis=0) + coord += offset + return coord + + +def gaussian_blur(hm, kernel): + # base:https://github.com/ilovepose/DarkPose + border = (kernel - 1) // 2 + batch_size = hm.shape[0] + num_joints = hm.shape[1] + height = hm.shape[2] + width = hm.shape[3] + for i in range(batch_size): + for j in range(num_joints): + origin_max = np.max(hm[i, j]) + dr = np.zeros((height + 2 * border, width + 2 * border)) + dr[border: -border, border: -border] = hm[i, j].copy() + dr = cv2.GaussianBlur(dr, (kernel, kernel), 0) + hm[i, j] = dr[border: -border, border: -border].copy() + hm[i, j] *= origin_max / np.max(hm[i, j]) + return hm + + +def get_final_preds(hm, realsize): + # base:https://github.com/ilovepose/DarkPose + coords, maxvals = get_max_preds(hm) + + # post-processing + hm = gaussian_blur(hm, kernel_size) + hm = np.maximum(hm, 1e-10) + hm = np.log(hm) + for n in range(coords.shape[0]): + for p in range(coords.shape[1]): + coords[n, p] = taylor(hm[n][p], coords[n][p]) + + preds = coords.copy() + preds = (preds / heatmap_size) * realsize # input_size + + # Transform back + # for i in range(coords.shape[0]): + # preds[i] = transform_preds( + # coords[i], center[i], scale[i], [heatmap_width, heatmap_height] + # ) + + return preds, maxvals + + +def resize_with_pad(image: np.array, + new_shape: Tuple[int, int], + padding_color: Tuple[int] = (255, 255, 255)) -> np.array: + """ + https://gist.github.com/IdeaKing/11cf5e146d23c5bb219ba3508cca89ec + Maintains aspect ratio and resizes with padding. + Params: + image: Image to be resized. + new_shape: Expected (width, height) of new image. + padding_color: Tuple in BGR of padding color + Returns: + image: Resized image with padding + """ + original_shape = (image.shape[1], image.shape[0]) + ratio = float(max(new_shape)) / max(original_shape) + new_size = tuple([int(x * ratio) for x in original_shape]) + image = cv2.resize(image, new_size) + delta_w = new_shape[0] - new_size[0] + delta_h = new_shape[1] - new_size[1] + top, bottom = delta_h // 2, delta_h - (delta_h // 2) + left, right = delta_w // 2, delta_w - (delta_w // 2) + image = cv2.copyMakeBorder(image, top, bottom, left, right, cv2.BORDER_CONSTANT, value=padding_color) + return image + + +# Better Eye aspEct Ratio +class BEER(object): + def __init__(self): + self.ear_th = 0.2 + self.ear_min = 0.05 + self.ear_max = 0.2 + self.p03_med = FastMedian(k=256) + self.prev_ear = 0.5 + # todo https://peerj.com/articles/cs-943/ + + def ear(self, pred): + p15 = np.linalg.norm(pred[1]-pred[5]) + p24 = np.linalg.norm(pred[2]-pred[4]) + p03 = np.linalg.norm(pred[0]-pred[3]) + self.p03_med+p03 + if p03 > self.p03_med.median()*1.5: + return self.prev_ear + ear = (p15+p24)/(2*self.p03_med.median()) + self.ear_minmax(ear) + norm_ear = self.ear_norm(ear) + self.prev_ear = norm_ear.copy() + return norm_ear + def ear_minmax(self,ear): + + if ear < self.ear_min: + self.ear_min = ear.copy() + if ear > self.ear_max: + self.ear_max = ear.copy() + + def ear_norm(self,ear): + return (ear-self.ear_min)/(self.ear_max-self.ear_min) # todo:It is better to add very small values to avoid zero division. + + +# +# loopnum = 0 +# + +# Deep leArning lanDmark Detection for eYes +class DADDY_cls(object): + def __init__(self): + onnxruntime.disable_telemetry_events() + options = onnxruntime.SessionOptions() + options.inter_op_num_threads = 1 # This number should be changed accordingly + options.intra_op_num_threads = 4 # This number should be changed accordingly + options.execution_mode = onnxruntime.ExecutionMode.ORT_SEQUENTIAL + options.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_ALL + + ort_session = onnxruntime.InferenceSession(model_file, sess_options=options, providers=["CPUExecutionProvider"]) + ort_session.set_providers(['CPUExecutionProvider']) # only cpu mode + + self.ort_session = ort_session + self.input_name = ort_session.get_inputs()[0].name + self.output_name = ort_session.get_outputs()[0].name + + min_cutoff = 0.0004 + beta = 0.9 + input_point = np.zeros((11, 2)) # np.array([1, 1]) + self.one_euro_filter = OneEuroFilter( + input_point, + min_cutoff=min_cutoff, + beta=beta + ) + # self.ear_oef = OneEuroFilter( + # np.zeros(1), + # min_cutoff=min_cutoff, + # beta=beta + # ) # memo: Parameters need tuning + + self.beer = BEER() + + # filepath = 'test.mp4' + # codec = cv2.VideoWriter_fourcc(*"mp4v") + # video = cv2.VideoWriter(filepath, codec, 60.0, (200, 150), 0) # (60, 60)) # (150, 200)) + # self.video = video + + def open_video(self, video_path): + # Temporary implementation to run + cap = cv2.VideoCapture(video_path) + if not cap.isOpened(): + raise IOError("Error opening video stream or file") + self.cap = cap + return True + + def read_frame(self): + # Temporary implementation to run + if not self.cap.isOpened(): + return False + ret, frame = self.cap.read() + if ret: + # I have set it to grayscale (1ch) just in case, but if the frame is 1ch, this line can be commented out. + # self.current_image=frame # debug code + self.current_image_gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) + return True + return False + + def single_run(self): + # Temporary implementation to run + + # todo: If it's the left hand eye, flip the image left to right. + + gray_frame = self.current_image_gray.copy() + + # frame_resize=resize_with_pad(gray_frame,(input_size,input_size)) + # or + frame_resize = cv2.resize(gray_frame, (input_size, input_size)) + imgs = np.divide(frame_resize[np.newaxis, np.newaxis], 255, dtype=np.float32) # input/255.0 + + pred_heatmap = self.ort_session.run(None, {self.input_name: imgs})[0] # .reshape((-1, 2)) + # if imshow_enable: + # heatmap = pred_heatmap.reshape((-1, heatmap_size, heatmap_size)) + # for i in range(heatmap.shape[0]): + # cv2.imshow("heatmap_{}".format(i + 1), heatmap[i]) + + pred, max_val = get_final_preds(pred_heatmap, (self.current_image_gray.shape[1], self.current_image_gray.shape[0])) + pred = pred.reshape((-1, 2)) + # or + # pred, max_val = get_final_preds(pred_heatmap, input_size) + # pred = pred.reshape((-1, 2)) + # height, width = self.current_image_gray.shape[:2] + # scale_x = input_size/ width + # scale_y = input_size / height + # pred[:, 0] *= scale_x + # pred[:, 1] *= scale_y + + pred = self.one_euro_filter(pred) + kps = pred.astype(np.int32) + + # eyecenter = kps[:6].mean(axis=0).astype(int) + # temp_eyecenter = pred[:6].mean(axis=0) + ear = self.beer.ear(pred) + # ear=self.ear_oef(ear[np.newaxis])#memo: Parameters need tuning + + if save_video or imshow_enable: + for i in range(kps.shape[0]): + if i < 6: + color = (0, 0, 255) + elif i == 6: + color = 128 # (0, 255, 0) + else: + color = (255, 0, 0) + cv2.circle(gray_frame, (kps[i, 0], kps[i, 1]), 1, color, 2) + cv2.putText(gray_frame, str(i), (kps[i, 0] - 10, kps[i, 1] - 10), cv2.FONT_HERSHEY_SIMPLEX, 0.35, (0, 0, 255), 1) + # cv2.circle(gray_frame, (eyecenter.tolist()), 1, (128, 128, 0), 2) + # cv2.circle(gray_frame, (iriscenter.tolist()), 1, (0, 128, 128), 2) + # cv2.circle(gray_frame,(eyecenter.tolist()),int(self.center_dist_med.median()),(0, 0, 255),1) + # cv2.ellipse(gray_frame, (eyecenter.tolist(), (int(self.iris_med_w.median()*2.5), int(self.iris_med_h.median()*2.5)), np.rad2deg(self.eye_med_rad.median())), (255, 0, 0), thickness=1) + cv2.putText(gray_frame, "EAR: "+str(ear), (self.current_image_gray.shape[1]//10, self.current_image_gray.shape[0]//10), cv2.FONT_HERSHEY_SIMPLEX, 0.75, (255,0,0), 1) + cv2.imshow("gray_frame", gray_frame) + if cv2.waitKey(1) & 0xFF == ord("q"): + pass + # global loopnum + # if loopnum < 1350*2: + # # self.video.write(cv2.resize(gray_frame.copy(), (200, 150), None)) + # loopnum += 1 + # else: + # # self.video.release() + # cv2.destroyAllWindows() + # sys.exit() + # if w_video: + # video.release() + + # kps[i, :] = (x, y) + # i == [0:6] = Inner and outer corners of eyes and eyelids + # i == [6] = pupil + # i == [7:] = iris + return ear + + +class External_Run_DADDY(object): + def __init__(self): + self.algo = DADDY_cls() + + def run(self, current_image_gray): + self.algo.current_image_gray = current_image_gray + pupil = self.algo.single_run() + return pupil + + +if __name__ == "__main__": + daddy = DADDY_cls() + daddy.open_video(video_path) + while daddy.read_frame(): + _ = daddy.single_run() \ No newline at end of file diff --git a/EyeTrackApp/daddy230210.onnx b/EyeTrackApp/daddy230210.onnx new file mode 100644 index 0000000..0b02392 Binary files /dev/null and b/EyeTrackApp/daddy230210.onnx differ diff --git a/EyeTrackApp/eye_processor.py b/EyeTrackApp/eye_processor.py index 95c935b..6ec540f 100644 --- a/EyeTrackApp/eye_processor.py +++ b/EyeTrackApp/eye_processor.py @@ -35,6 +35,8 @@ from dataclasses import dataclass import sys import asyncio + + sys.path.append(".") from config import EyeTrackCameraConfig from config import EyeTrackSettingsConfig @@ -50,6 +52,7 @@ from utils.misc_utils import PlaySound, SND_FILENAME, SND_ASYNC import importlib from osc import EyeId from osc_calibrate_filter import * +from daddy import External_Run_DADDY from haar_surround_feature import External_Run_HSF from blob import * from ransac import * @@ -167,6 +170,8 @@ class EyeProcessor: self.prev_x = None self.prev_y = None + + self.daddy = None @@ -244,7 +249,10 @@ class EyeProcessor: def BLINKM(self): self.eyeoffx = BLINK(self) - + + def DADDYM(self): + landmark = self.daddy.run(self.current_image_gray) + def HSRACM(self): # todo: added process to initialise er_hsrac when resolution changes @@ -301,8 +309,8 @@ class EyeProcessor: - def ALGOSELECT(self): - + def ALGOSELECT(self): + # self.DADDYM() if self.failed == 0 and self.firstalgo != None: self.firstalgo() else: @@ -322,11 +330,14 @@ class EyeProcessor: self.fourthalgo() else: self.failed = 0 # we have reached last possible algo and it is disabled, move to first algo - + + def run(self): + # Run the following somewhere + # self.daddy = External_Run_DADDY() self.firstalgo = None self.secondalgo = None diff --git a/EyeTrackApp/utils/misc_utils.py b/EyeTrackApp/utils/misc_utils.py index 0d99d66..c205fe7 100644 --- a/EyeTrackApp/utils/misc_utils.py +++ b/EyeTrackApp/utils/misc_utils.py @@ -1,4 +1,6 @@ import os +import typing + is_nt = True if os.name == "nt" else False def PlaySound(*args, **kwargs): pass @@ -12,3 +14,47 @@ if is_nt: def clamp(x, low, high): return max(low, min(x, high)) + + +def lst_median(lst, ordered=False): + # https://github.com/emilianavt/OpenSeeFace/blob/6f24efc4f58eb7cca47ec2146d934eabcc207e46/remedian.py + assert lst, "median needs a non-empty list" + n = len(lst) + p = q = n // 2 + if n < 3: + p, q = 0, n - 1 + else: + lst = lst if ordered else sorted(lst) + if not n % 2: # for even-length lists, use mean of mid 2 nums + q = p - 1 + return lst[p] if p == q else (lst[p] + lst[q]) / 2 + + +class FastMedian: + # https://github.com/emilianavt/OpenSeeFace/blob/6f24efc4f58eb7cca47ec2146d934eabcc207e46/remedian.py + # Initialization + def __init__(self, inits: typing.Optional[typing.Sequence] = [], k=64): # after some experimentation, 64 works ok + self.all, self.k = [], k + self.more, self.__median = None, None + if inits is not None: + [self + x for x in inits] + + # When full, push the median of current values to next list, then reset. + def __add__(self, x): + self.__median = None + self.all.append(x) # It would be faster to pre-allocate an array and assign it by index. + if len(self.all) == self.k: + self.more = self.more or FastMedian(k=self.k) + self.more + self.__medianPrim(self.all) + # It's going to be slower because of the re-allocation. + self.all = [] # reset + + # If there is a next list, ask its median. Else, work it out locally. + def median(self): + return self.more.median() if self.more else self.__medianPrim(self.all) + + # Only recompute median if we do not know it already. + def __medianPrim(self, all): + if self.__median is None: + self.__median = lst_median(all, ordered=False) + return self.__median \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 7796161..d69297e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,6 +17,7 @@ pydantic = "^1.10.2" win10toast_click = [ { version = "^0.1.2", platform = 'win32' } ] +onnxruntime = "^1.13.1" [tool.poetry.group.dev.dependencies] black = "^22.10.0"