Merge branch 'HSF-and-new-algos-feature-branch' into HSF-and-new-algos-feature-branch

This commit is contained in:
Prohurtz 2023-02-09 20:27:04 -06:00 committed by GitHub
commit 12304b5b3a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 715 additions and 1562 deletions

View File

@ -19,8 +19,8 @@
@@@@@@@@@@@@@@@@@
@@@@@@@@@@@@@(
HSR By: Sean.Denka (Optimization Wizard, Contributor), Summer#2406 (Main Algorithm Engineer)
RANSAC 3D By: Summer#2406 (Main Algorithm Engineer), Pupil Labs (pye3d), Sean.Denka (Optimization)
HSR By: PallasNeko (Optimization Wizard, Contributor), Summer#2406 (Main Algorithm Engineer)
RANSAC 3D By: Summer#2406 (Main Algorithm Engineer), Pupil Labs (pye3d), PallasNeko (Optimization)
BLOB By: Prohurtz#0001 (Main App Developer)
Algorithm App Implimentations By: Prohurtz#0001, qdot (Inital App Creator)
@ -51,10 +51,10 @@ if sys.platform.startswith("win"):
import importlib
from osc_calibrate_filter import *
from haar_surround_feature import *
from haar_surround_feature import External_Run_HSF
from blob import *
from ransac import *
from hsrac import *
from hsrac import External_Run_HSRACS
from blink import *
@ -147,7 +147,6 @@ class EyeProcessor:
self.cccs = False
self.ts = 10
self.previous_rotation = self.config.rotation_angle
self.calibration_frame_counter
self.camera_model = None
self.detector_3d = None
@ -242,13 +241,13 @@ class EyeProcessor:
pass
def BLINKM(self):
self.blinkvalue = BLINK(self)
self.eyeoffx = BLINK(self)
def HSRACM(self):
cx, cy, thresh, gray_frame = External_Run_HSRACS.HSRACS(self)
cx, cy, thresh, gray_frame, uncropframe = External_Run_HSRACS().run(self.current_image_gray)
self.current_image_gray = gray_frame
if self.prev_x == None:
if self.prev_x is None:
self.prev_x = cx
self.prev_y = cy
#print(self.prev_x, self.prev_y, cx, cy)
@ -256,43 +255,43 @@ class EyeProcessor:
# if (cx - self.prev_x) <= 45 and (cy - self.prev_y) <= 45 :
# self.prev_x = cx
# self.prev_y = cy
eyeopen = intense(cx, cy, self.current_image_gray)
self.eyeopen = intense(cx, cy, uncropframe)
out_x, out_y = cal_osc(self, cx, cy)
#print(self.eyeoffx, self.eyeopen)
if cx == 0:
self.output_images_and_update(thresh, EyeInformation(InformationOrigin.HSRAC, out_x, out_y, 0, eyeopen)) #update app
if self.eyeoffx:
self.output_images_and_update(thresh, EyeInformation(InformationOrigin.HSRAC, out_x, out_y, 0, 0.0)) #update app
else:
self.output_images_and_update(thresh, EyeInformation(InformationOrigin.HSRAC, out_x, out_y, 0, eyeopen))
self.output_images_and_update(thresh, EyeInformation(InformationOrigin.HSRAC, out_x, out_y, 0, self.eyeopen))
# else:
# print("EYE MOVED TOO FAST")
# self.output_images_and_update(thresh, EyeInformation(InformationOrigin.HSRAC, 0, 0, 0, False))
def HSFM(self):
cx, cy, frame = External_Run_HSF.HSFS(self)
eyeopen = intense(cx, cy, self.current_image_gray)
cx, cy, frame = External_Run_HSF().run(self.current_image_gray)
self.eyeopen = intense(cx, cy, self.current_image_gray)
out_x, out_y = cal_osc(self, cx, cy)
if cx == 0:
self.output_images_and_update(frame, EyeInformation(InformationOrigin.HSF, out_x, out_y, 0, eyeopen)) #update app
self.output_images_and_update(frame, EyeInformation(InformationOrigin.HSF, out_x, out_y, 0, self.eyeopen)) #update app
else:
self.output_images_and_update(frame, EyeInformation(InformationOrigin.HSF, out_x, out_y, 0, eyeopen))
self.output_images_and_update(frame, EyeInformation(InformationOrigin.HSF, out_x, out_y, 0, self.eyeopen))
def RANSAC3DM(self):
cx, cy, thresh = RANSAC3D(self)
eyeopen = intense(cx, cy, self.current_image_gray)
self.eyeopen = intense(cx, cy, self.current_image_gray)
out_x, out_y = cal_osc(self, cx, cy)
if cx == 0:
self.output_images_and_update(thresh, EyeInformation(InformationOrigin.RANSAC, out_x, out_y, 0, eyeopen)) #update app
self.output_images_and_update(thresh, EyeInformation(InformationOrigin.RANSAC, out_x, out_y, 0, self.eyeopen)) #update app
else:
self.output_images_and_update(thresh, EyeInformation(InformationOrigin.RANSAC, out_x, out_y, 0, eyeopen))
self.output_images_and_update(thresh, EyeInformation(InformationOrigin.RANSAC, out_x, out_y, 0, self.eyeopen))
def BLOBM(self):
cx, cy, thresh = BLOB(self)
eyeopen = intense(cx, cy, self.current_image_gray)
self.eyeopen = intense(cx, cy, self.current_image_gray)
out_x, out_y = cal_osc(self, cx, cy)
if cx == 0:
self.output_images_and_update(thresh, EyeInformation(InformationOrigin.HSRAC, out_x, out_y, 0, eyeopen)) #update app
self.output_images_and_update(thresh, EyeInformation(InformationOrigin.HSRAC, out_x, out_y, 0, self.eyeopen)) #update app
else:
self.output_images_and_update(thresh, EyeInformation(InformationOrigin.HSRAC, out_x, out_y, 0, eyeopen))
self.output_images_and_update(thresh, EyeInformation(InformationOrigin.HSRAC, out_x, out_y, 0, self.eyeopen))
@ -347,7 +346,7 @@ class EyeProcessor:
elif self.settings.gui_RANSAC3D and self.settings.gui_RANSAC3DP == 4:
self.fourthalgo = self.RANSAC3DM
if self.settings.gui_HSRAC == True and self.settings.gui_HSRACP == 1:
if self.settings.gui_HSRAC and self.settings.gui_HSRACP == 1:
self.firstalgo = self.HSRACM
elif self.settings.gui_HSRAC and self.settings.gui_HSRACP == 2:
self.secondalgo = self.HSRACM

View File

@ -7,8 +7,7 @@ import queue
import threading
import PySimpleGUI as sg
import sys
from urllib.request import urlopen
from bs4 import BeautifulSoup
import urllib.request
import webbrowser
@ -53,35 +52,33 @@ def main():
if config.settings.gui_update_check:
print("\033[95m[INFO] Checking for updates...\033[0m")
url = "https://raw.githubusercontent.com/RedHawk989/EyeTrackVR-Installer/master/Version-Data/Version_Num.txt"
html = urlopen(url).read()
soup = BeautifulSoup(html, features="html.parser")
for script in soup(["script", "style"]):
script.extract()
text = soup.get_text()
# break into lines and remove leading and trailing space on each
lines = (line.strip() for line in text.splitlines())
# break multi-headlines into a line each
chunks = (phrase.strip() for line in lines for phrase in line.split(" "))
# drop blank lines
latestversion = '\n'.join(chunk for chunk in chunks if chunk)
if appversion == latestversion: # If what we scraped and hardcoded versions are same, assume we are up to date.
print(f"\033[92m[INFO] App is up to date! [{latestversion}]\033[0m")
else:
print(f"\033[93m[INFO] You have app version [{appversion}] installed. Please update to [{latestversion}] for the newest features.\033[0m")
if sys.platform.startswith("win"):
toaster = ToastNotifier()
toaster.show_toast( #show windows toast
"EyeTrackVR has an update.",
"Click to go to the latest version.",
icon_path= "Images/logo.ico",
duration=5,
threaded=True,
callback_on_click=open_url
req = urllib.request.Request(url)
try:
with urllib.request.urlopen(req, timeout=10) as res:
latestversion = res.read().decode("utf-8").strip()
except urllib.error.HTTPError as err:
print("Failed to check latest version.")
print("{} : {}".format(err.code,err.reason))
except urllib.error.URLError as err:
print("Failed to check latest version.")
print(err.reason)
else:
if appversion == latestversion: # If what we scraped and hardcoded versions are same, assume we are up to date.
print(f"\033[92m[INFO] App is up to date! [{latestversion}]\033[0m")
else:
print(
f"\033[93m[INFO] You have app version [{appversion}] installed. Please update to [{latestversion}] for the newest features.\033[0m")
if sys.platform.startswith("win"):
toaster = ToastNotifier()
toaster.show_toast( # show windows toast
"EyeTrackVR has an update.",
"Click to go to the latest version.",
icon_path="Images/logo.ico",
duration=5,
threaded=True,
callback_on_click=open_url
)
# Check to see if we have an ROI. If not, bring up ROI finder GUI.
# Spawn worker threads

View File

@ -1,16 +1,18 @@
import functools
import math
import sys
import timeit
from functools import lru_cache
import cv2
import numpy as np
from utils.misc_utils import clamp
from utils.img_utils import safe_crop
# from line_profiler_pycharm import profile
video_path = "ezgif.com-gif-maker.avi"
imshow_enable = True
imshow_enable = False
calc_print_enable = True
save_video = False
skip_autoradius = False
@ -27,181 +29,6 @@ blink_init_frames = 60 * 3 # 60fps*3sec,Number of blink statistical frames
# step==(x,y)
default_step = (5, 5) # bigger the steps,lower the processing time! ofc acc also takes an impact
"""
Attention.
If using cv2.filter2D in this code, be careful with the kernel
https://stackoverflow.com/questions/39457468/convolution-without-any-padding-opencv-python
"""
def TimeitWrapper(*args, **kwargs):
"""
This decorator @TimeitWrapper() prints the function name and execution time in seconds.
:param args:
:param kwargs:
:return:
"""
def decorator(function):
@functools.wraps(function)
def wrapper(*args, **kwargs):
start = timeit.default_timer()
results = function(*args, **kwargs)
end = timeit.default_timer()
print('{} execution time: {:.10f} s'.format(function.__name__, end - start))
return results
return wrapper
return decorator
class TimeitResult(object):
"""
from https://github.com/ipython/ipython/blob/339c0d510a1f3cb2158dd8c6e7f4ac89aa4c89d8/IPython/core/magics/execution.py#L55
Object returned by the timeit magic with info about the run.
Contains the following attributes :
loops: (int) number of loops done per measurement
repeat: (int) number of times the measurement has been repeated
best: (float) best execution time / number
all_runs: (list of float) execution time of each run (in s)
"""
def __init__(self, loops, repeat, best, worst, all_runs, precision):
self.loops = loops
self.repeat = repeat
self.best = best
self.worst = worst
self.all_runs = all_runs
self._precision = precision
self.timings = [dt / self.loops for dt in all_runs]
@property
def average(self):
return math.fsum(self.timings) / len(self.timings)
@property
def stdev(self):
mean = self.average
return (math.fsum([(x - mean) ** 2 for x in self.timings]) / len(self.timings)) ** 0.5
def __str__(self):
pm = '+-'
if hasattr(sys.stdout, 'encoding') and sys.stdout.encoding:
try:
u'\xb1'.encode(sys.stdout.encoding)
pm = u'\xb1'
except:
pass
return "min:{best} max:{worst} mean:{mean} {pm} {std} per loop (mean {pm} std. dev. of {runs} run{run_plural}, {loops:,} loop{loop_plural} each)".format(
pm=pm,
runs=self.repeat,
loops=self.loops,
loop_plural="" if self.loops == 1 else "s",
run_plural="" if self.repeat == 1 else "s",
mean=format_time(self.average, self._precision),
std=format_time(self.stdev, self._precision),
best=format_time(self.best, self._precision),
worst=format_time(self.worst, self._precision),
)
def _repr_pretty_(self, p, cycle):
unic = self.__str__()
p.text(u'<TimeitResult : ' + unic + u'>')
class FPSResult(object):
"""
base https://github.com/ipython/ipython/blob/339c0d510a1f3cb2158dd8c6e7f4ac89aa4c89d8/IPython/core/magics/execution.py#L55
"""
def __init__(self, loops, repeat, best, worst, all_runs, precision):
self.loops = loops
self.repeat = repeat
self.best = 1 / best
self.worst = 1 / worst
self.all_runs = all_runs
self._precision = precision
self.fps = [1 / dt for dt in all_runs]
self.unit = "fps"
@property
def average(self):
return math.fsum(self.fps) / len(self.fps)
@property
def stdev(self):
mean = self.average
return (math.fsum([(x - mean) ** 2 for x in self.fps]) / len(self.fps)) ** 0.5
def __str__(self):
pm = '+-'
if hasattr(sys.stdout, 'encoding') and sys.stdout.encoding:
try:
u'\xb1'.encode(sys.stdout.encoding)
pm = u'\xb1'
except:
pass
return "min:{best} max:{worst} mean:{mean} {pm} {std} per loop (mean {pm} std. dev. of {runs} run{run_plural}, {loops:,} loop{loop_plural} each)".format(
pm=pm,
runs=self.repeat,
loops=self.loops,
loop_plural="" if self.loops == 1 else "s",
run_plural="" if self.repeat == 1 else "s",
mean="%.*g%s" % (self._precision, self.average, self.unit),
std="%.*g%s" % (self._precision, self.stdev, self.unit),
best="%.*g%s" % (self._precision, self.best, self.unit),
worst="%.*g%s" % (self._precision, self.worst, self.unit),
)
def _repr_pretty_(self, p, cycle):
unic = self.__str__()
p.text(u'<FPSResult : ' + unic + u'>')
def format_time(timespan, precision=3):
"""
https://github.com/ipython/ipython/blob/339c0d510a1f3cb2158dd8c6e7f4ac89aa4c89d8/IPython/core/magics/execution.py#L1473
Formats the timespan in a human readable form
"""
if timespan >= 60.0:
# we have more than a minute, format that in a human readable form
# Idea from http://snipplr.com/view/5713/
parts = [("d", 60 * 60 * 24), ("h", 60 * 60), ("min", 60), ("s", 1)]
time = []
leftover = timespan
for suffix, length in parts:
value = int(leftover / length)
if value > 0:
leftover = leftover % length
time.append(u'%s%s' % (str(value), suffix))
if leftover < 1:
break
return " ".join(time)
# Unfortunately the unicode 'micro' symbol can cause problems in
# certain terminals.
# See bug: https://bugs.launchpad.net/ipython/+bug/348466
# Try to prevent crashes by being more secure than it needs to
# E.g. eclipse is able to print a µ, but has no sys.stdout.encoding set.
units = [u"s", u"ms", u'us', "ns"] # the save value
if hasattr(sys.stdout, 'encoding') and sys.stdout.encoding:
try:
u'\xb5'.encode(sys.stdout.encoding)
units = [u"s", u"ms", u'\xb5s', "ns"]
except:
pass
scaling = [1, 1e3, 1e6, 1e9]
if timespan > 0.0:
order = min(-int(math.floor(math.log10(timespan)) // 3), 3)
else:
order = 3
return u"%.*g %s" % (precision, timespan * scaling[order], units[order])
class CvParameters:
# It may be a little slower because a dict named "self" is read for each function call.
def __init__(self, radius, step):
@ -419,7 +246,7 @@ def conv_int(frame_int, kernel, xy_step, padding, xy_steps_list):
return frame_conv, min_response, center
class Auto_Radius_Calc(object):
class AutoRadiusCalc(object):
def __init__(self):
self.response_list = []
self.radius_cand_list = []
@ -522,7 +349,7 @@ class Auto_Radius_Calc(object):
return None
class Blink_Detector(object):
class BlinkDetector(object):
def __init__(self):
self.response_list = []
self.response_max = None
@ -665,7 +492,7 @@ class CenterCorrection(object):
return out_x, out_y
class HSRAC_cls(object):
class HSF_cls(object):
def __init__(self):
# I'd like to take into account things like print, end_time - start_time processing time, etc., but it's too much trouble.
@ -679,9 +506,9 @@ class HSRAC_cls(object):
self.cv_modeo = ["first_frame", "radius_adjust", "blink_adjust", "normal"]
self.now_modeo = self.cv_modeo[0]
self.auto_radius_calc = Auto_Radius_Calc()
self.blink_detector = Blink_Detector()
self.center_q1 = Blink_Detector()
self.auto_radius_calc = AutoRadiusCalc()
self.blink_detector = BlinkDetector()
self.center_q1 = BlinkDetector()
self.center_correct = CenterCorrection()
self.cap = None
@ -709,9 +536,13 @@ class HSRAC_cls(object):
def single_run(self):
# Temporary implementation to run
## default_radius = 14
# cropbox=[] # debug code
frame = self.current_image_gray
if self.now_modeo == self.cv_modeo[1]:
# adjustment of radius
@ -756,7 +587,12 @@ class HSRAC_cls(object):
lower_y = center_y - radius
# Crop the image using the calculated bounds
cropped_image = gray_frame[lower_y:upper_y, lower_x:upper_x]
cropped_image = safe_crop(gray_frame, lower_x, lower_y, upper_x, upper_y)
# cropbox = [clamp(val, 0, gray_frame.shape[i]) for i, val in
# zip([1, 0, 1, 0], [lower_x, lower_y, upper_x, upper_y])] # debug code
if self.now_modeo == self.cv_modeo[0] or self.now_modeo == self.cv_modeo[1]:
# If mode is first_frame or radius_adjust, record current radius and response
@ -770,8 +606,14 @@ class HSRAC_cls(object):
lower_x = center_x - self.center_correct.center_q1_radius
upper_y = center_y + self.center_correct.center_q1_radius
lower_y = center_y - self.center_correct.center_q1_radius
self.center_q1.add_response(cv2.mean(gray_frame[lower_y:upper_y, lower_x:upper_x])[0])
self.center_q1.add_response(
cv2.mean(safe_crop(gray_frame, lower_x, lower_y, upper_x, upper_y,keepsize=False))[
0
]
)
else:
self.blink_detector.calc_thresh()
@ -792,9 +634,19 @@ class HSRAC_cls(object):
else:
# pass
if not self.center_correct.setup_comp:
self.center_correct.init_array(gray_frame.shape, self.center_q1.quartile_1, radius)
center_x, center_y = self.center_correct.correction(gray_frame, center_x, center_y)
self.center_correct.init_array(
gray_frame.shape, self.center_q1.quartile_1, radius
)
elif self.center_correct.frame_shape != gray_frame.shape:
"""The resolution should have changed and the statistics should have changed, so essentially the statistics
need to be reworked, but implementation will be postponed as viability is the highest priority. """
self.center_correct.init_array(
gray_frame.shape, self.center_q1.quartile_1, radius
)
center_x, center_y = self.center_correct.correction(
gray_frame, center_x, center_y
)
# Define the center point and radius
center_xy = (center_x, center_y)
upper_x = center_x + radius
@ -802,10 +654,16 @@ class HSRAC_cls(object):
upper_y = center_y + radius
lower_y = center_y - radius
# Crop the image using the calculated bounds
cropped_image = gray_frame[lower_y:upper_y, lower_x:upper_x]
# if imshow_enable or save_video:
# cv2.circle(frame, (orig_x, orig_y), 6, (0, 0, 255), -1)
# cv2.circle(frame, (center_x, center_y), 3, (255, 0, 0), -1)
cropped_image = safe_crop(
gray_frame, lower_x, lower_y, upper_x, upper_y
)
# cropbox = [clamp(val, 0, gray_frame.shape[i]) for i, val in
# zip([1, 0, 1, 0], [lower_x, lower_y, upper_x, upper_y])] # debug code
# if imshow_enable or save_video:
# cv2.circle(frame, (orig_x, orig_y), 6, (0, 0, 255), -1)
# cv2.circle(frame, (center_x, center_y), 3, (255, 0, 0), -1)
# If you want to update response_max. it may be more cost-effective to rewrite response_list in the following way
# https://stackoverflow.com/questions/42771110/fastest-way-to-left-cycle-a-numpy-array-like-pop-push-for-a-queue
@ -837,20 +695,28 @@ class HSRAC_cls(object):
self.now_modeo = self.cv_modeo[2]
else:
self.now_modeo = self.cv_modeo[1]
# debug code
# return center_x,center_y,cropbox,frame
return center_x, center_y, frame
class External_Run_HSF:
class External_Run_HSF(object):
def __init__(self):
self.algo = HSF_cls()
hsrac = HSRAC_cls()
def HSFS(self):
External_Run_HSF.hsrac.current_image_gray = self.current_image_gray
center_x, center_y, frame = External_Run_HSF.hsrac.single_run()
def run(self, current_image_gray):
self.algo.current_image_gray = current_image_gray
# debug code
# center_x, center_y,cropbox, frame = self.algo.single_run()
# return center_x, center_y,cropbox, frame
center_x, center_y, frame = self.algo.single_run()
return center_x, center_y, frame
if __name__ == '__main__':
hsrac = HSRAC_cls()
hsrac.open_video(video_path)
while hsrac.read_frame():
_ = hsrac.single_run()
if __name__ == "__main__":
hsf = HSF_cls()
hsf.open_video(video_path)
while hsf.read_frame():
_ = hsf.single_run()

File diff suppressed because it is too large Load Diff

View File

@ -1,13 +1,18 @@
import pandas as pd
import numpy as np
import time
import os
import cv2
from pythonosc import udp_client
from enum import IntEnum
#higher intensity means more closed/ more white/less pupil
#Hm I need an acronym for this, any ideas?
#IBO Intensity Based Openess
#IBO Intensity Based Openess
class EyeId(IntEnum):
RIGHT = 0
LEFT = 1
BOTH = 2
SETTINGS = 3
# HOW THIS WORKS:
# we get the intensity of pupil area from HSF crop, When the eyelid starts to close, the pupil starts being obstructed by skin which is generally lighter than the pupil.
@ -15,79 +20,183 @@ from pythonosc import udp_client
# ex. when you look up there is less pupil visible, which results in an uncalculated change in intensity even though the eyelid has not moved in a meaningful way.
# We compare the darkest intensity of that area, to the lightest (global) intensity to find the appropriate openness state via a float.
fname = "IBO.csv" #TODO Expose as setting
try:
data = pd.read_csv(fname, sep=",")
except:
cf = open(fname, "w")
cf.write("xy,intensity")
cf.close()
data = pd.read_csv(fname, sep=",")
# Note.
# OpenCV on Windows will generate an error if the file path contains non-ASCII characters when using cv2.imread(), cv2.imwrite(), etc.
# https://stackoverflow.com/questions/43185605/how-do-i-read-an-image-from-a-path-with-unicode-characters
# https://github.com/opencv/opencv/issues/18305
if EyeId.RIGHT:
fname = "IBO_RIGHT.png"
if EyeId.LEFT:
fname = "IBO_LEFT.png"
lct = time.time()
data = None
def csv2data(frameshape, filepath):
# For data checking
frameshape = (frameshape[0], frameshape[1]+1)
out = np.zeros(frameshape, dtype=np.uint32)
xy_list = []
val_list = []
with open(filepath, mode="r", encoding="utf-8") as in_f:
# Skip header.
_ = in_f.readline()
for s in in_f:
xyval = [int(val) for val in s.strip().split(',')]
xy_list.append((xyval[0], xyval[1]))
val_list.append(xyval[2])
xy_list = np.array(xy_list)
val_list = np.array(val_list)
out[xy_list[:, 1], xy_list[:, 0]] = val_list[:]
return out
def data2csv(data_u32, filepath):
# For data checking
nonzero_index = np.nonzero(data_u32) #(row,col)
data_list = data_u32[nonzero_index].tolist()
datalines = ["{},{},{}\n".format(x, y, val) for y, x, val in zip(*nonzero_index, data_list)]
with open(filepath, 'w', encoding="utf-8") as out_f:
out_f.write("x,y,intensity\n")
out_f.writelines(datalines)
return
def u32_u16_1ch3ch(img):
img_copy = img.copy()
# In the case of bit operations. (img>>32)&0xffff,(img>>16)&0xffff,img&0xffff
out = np.zeros((*img.shape[:2], 3), dtype=np.uint16)
for i in range(3):
out[:, :, i] = img_copy % 0xffff
img_copy //= 0xffff
return out
def u16_u32_3ch_1ch(img):
# In the case of bit operations. ((img >> 32) & 0xffff) << 32 |((img >> 16) & 0xffff) << 16 | (img & 0xffff)
out = np.zeros(img.shape[:2], dtype=np.uint32)
for i in range(3):
out += img[:, :, i] if i == 0 else img[:, :, i] * (i * 0xffff)
return out
def newdata(frameshape):
print("Initialise data for blinking.")
return np.zeros(frameshape, dtype=np.uint32)
def check_and_load(frameshape, now_data):
# In the future, both eyes may be processed at the same time. Therefore, data should be passed as arguments.
req_newdata = False
# Not very clever, but increase the width by 1px to save the maximum value.
frameshape = (frameshape[0], frameshape[1]+1)
if now_data is None:
print("Load data for blinking: {}".format(fname))
if os.path.isfile(fname):
img = cv2.imread(fname, flags=cv2.IMREAD_UNCHANGED)
if img.shape[:2] != frameshape:
print("size does not match the input frame.")
req_newdata = True
else:
now_data = u16_u32_3ch_1ch(img)
else:
print("File does not exist.")
req_newdata = True
else:
if now_data.shape != frameshape:
# Using the previous and current frame sizes and centre positions from the original, etc., the data can be ported to some extent, but there may be many areas where code changes are required.
print("Frame size changed.")
req_newdata = True
if req_newdata:
now_data = newdata(frameshape)
# data2csv(now_data, "a.csv")
# csv2data(frameshape,"a.csv")
return now_data
#TODO we need more pixel points for smooth operation, lets get this setup in hsrac
def intense(x, y, frame):
# upper_x = x + 25
# lower_x = x - 25
# upper_y = y + 25
# lower_y = y - 25
# frame = frame[lower_y:upper_y, lower_x:upper_x]
try:
xy = int(str(int(x)) + str(int(y)) + str(int(x)+int(y)))
intensity = np.sum(frame)
except:
return 0.0 #TODO find how on earth a hyphen gets thrown into this
global lct, data
# 0 in data is used as the initial value.
# When assigning a value, +1 is added to the value to be assigned.
data = check_and_load(frame.shape[:2], data)
int_x, int_y = int(x), int(y)
# upper_x = min(int_x + 25, frame.shape[1]) #TODO make this a setting
#lower_x = max(int_x - 25, 0)
#upper_y = min(int_y + 25, frame.shape[0])
#lower_y = max(int_y - 25, 0)
#frame_crop = frame[lower_y:upper_y, lower_x:upper_x]
frame_crop = frame
# The same can be done with cv2.integral, but since there is only one area of the rectangle for which we want to know the total value, there is no advantage in terms of computational complexity.
intensity = frame_crop.sum()+1
# numpy:np.sum(),ndarray.sum()
# opencv:cv2.sumElems()
# I don't know which is faster.
print(frame.shape[1], frame.shape[0], int_x, int_y)
changed = False
try: #max pupil per cord
dfb = data[data['xy']==xy].index.values.astype(int)[0] # find pandas index of line with matching xy value
newval_flg = False
if int_y >= frame.shape[1]:
data_val = 0
print('CAUGHT Y OUT OF BOUNDS')
if intensity < data.at[dfb, 'intensity']: #if current intensity value is less (more pupil), save that
data.at[dfb, 'intensity'] = intensity # set value
changed = True
print("var adjusted")
else:
intensitya = data.at[dfb, 'intensity'] - 3 #if current intensity value is less (more pupil), save that
data.at[dfb, 'intensity'] = intensitya # set value
changed = True
else:
except: # that value is not yet saved
data.loc[len(data.index)] = [xy, intensity] #create new data on last line of csv with current intesity
data_val = data[int_y, int_x]
# max pupil per cord
if data_val == 0:
# The value of the specified coordinates has not yet been recorded.
data[int_y, int_x] = intensity
changed = True
newval_flg = True
elif intensity < data_val: # if current intensity value is less (more pupil), save that
data[int_y, int_x] = intensity # set value
changed = True
print("var adjusted")
else:
intensitya = max(data_val - 3, 1) # if current intensity value is less (more pupil), save that
data[int_y, int_x] = intensitya # set value
changed = True
try: # min pupil global
if intensity > data.at[0, 'intensity']: #if current intensity value is more (less pupil), save that NOTE: we have the
data.at[0, 'intensity'] = intensity # set value at 0 index
changed = True
print("new max", intensity)
else:
intensityd = data.at[0, 'intensity'] - 10 #continuously adjust closed intensity, will be set when user blink, used to allow eyes to close when lighting changes
data.at[0, 'intensity'] = intensityd # set value at 0 index
changed = True
except: # there is no max intensity yet, create
data.at[0, 'intensity'] = intensity # set value at 0 index
# min pupil global
if data[0, -1] == 0: # that value is not yet saved
data[0, -1] = intensity # set value at 0 index
changed = True
print("create max", intensity)
try:
maxp = data.at[dfb, 'intensity']
minp = data.at[0, 'intensity']
#eyeopen = (intensity - minp) / (maxp - minp)
eyeopen = (intensity - maxp) / (minp - maxp)
eyeopen = 1 - eyeopen
print(intensity, maxp, minp, x, y)
# print(f"EYEOPEN: {eyeopen}")
except:
elif intensity > data[0, -1]: # if current intensity value is more (less pupil), save that NOTE: we have the
data[0, -1] = intensity # set value at 0 index
changed = True
print("new max", intensity)
else:
intensityd = max(data[0, -1] - 10, 1) #continuously adjust closed intensity, will be set when user blink, used to allow eyes to close when lighting changes
data[0, -1] = intensityd # set value at 0 index
changed = True
if newval_flg:
# Do the same thing as in the original version.
print('[INFO] Something went wrong, assuming blink.')
eyeopen = 0.0
eyeopen = 0.7
else:
maxp = data[int_y, int_x]
minp = data[0, -1]
diffp = minp - maxp if (minp - maxp) != 0 else 1
eyeopen = (intensity - maxp) / diffp
eyeopen = 1 - eyeopen
#eyeopen = eyeopen - 0.2
# print(intensity, maxp, minp, x, y)
# print(f"EYEOPEN: {eyeopen}")
# print(int(x), int(y), eyeopen, maxp, minp)
if changed == True:
data.to_csv(fname, encoding='utf-8', index=False) #save file since we made a change
if changed and ((time.time() - lct) > 4): # save every 4 seconds if something changed to save disk usage
cv2.imwrite(fname, u32_u16_1ch3ch(data))
lct = time.time()
print("SAVED")
return eyeopen

View File

@ -126,11 +126,9 @@ class VRChatOSC:
if eye_id in [EyeId.RIGHT]:
rb = True
if last_blink > 0.7:
for i in range(5):
self.client.send_message("/avatar/parameters/RightEyeLid", float(1))
self.client.send_message("/avatar/parameters/RightEyeLidExpandedSqueeze", float(eye_info.blink)) # close eye
last_blink = time.time() - last_blink
self.client.send_message("/avatar/parameters/RightEyeLid", float(1))
self.client.send_message("/avatar/parameters/RightEyeLidExpandedSqueeze", float(eye_info.blink)) # close eye
else:
if eye_id in [EyeId.LEFT]:
@ -146,13 +144,10 @@ class VRChatOSC:
self.client.send_message("/avatar/parameters/LeftEyeLid", float(0))# old param open left
self.client.send_message("/avatar/parameters/LeftEyeLidExpandedSqueeze", float(eye_info.blink)) # open left eye
if rb and lb: # If both eyes are closed, blink
if last_blink > 0.5:
for i in range(4):
self.client.send_message("/avatar/parameters/RightEyeLid", float(1)) #close eye
self.client.send_message("/avatar/parameters/LeftEyeLid", float(1))
self.client.send_message("/avatar/parameters/RightEyeLidExpandedSqueeze", float(eye_info.blink)) # close eye
self.client.send_message("/avatar/parameters/LeftEyeLidExpandedSqueeze", float(eye_info.blink))
last_blink = time.time() - last_blink
self.client.send_message("/avatar/parameters/RightEyeLid", float(1)) #close eye
self.client.send_message("/avatar/parameters/LeftEyeLid", float(1))
self.client.send_message("/avatar/parameters/RightEyeLidExpandedSqueeze", float(eye_info.blink)) # close eye
self.client.send_message("/avatar/parameters/LeftEyeLidExpandedSqueeze", float(eye_info.blink))
class VRChatOSCReceiver:

View File

@ -19,7 +19,7 @@
@@@@@@@@@@@@@@@@@
@@@@@@@@@@@@@(
RANSAC 3D By: Summer#2406 (Main Algorithm Engineer), Pupil Labs (pye3d), Sean.Denka (Optimization)
RANSAC 3D By: Summer#2406 (Main Algorithm Engineer), Pupil Labs (pye3d), PallasNeko (Optimization)
Algorithm App Implimentations By: Prohurtz#0001, qdot (Inital App Creator)
Copyright (c) 2022 EyeTrackVR <3

View File

@ -1,308 +0,0 @@
xy,intensity
473582.0,357639.0
473583.0,250971.0
463582.0,259772.0
463581.0,250001.0
463481.0,262307.0
473482.0,259063.0
503485.0,253050.0
513586.0,242916.0
523486.0,245595.0
523587.0,239752.0
523588.0,235965.0
533589.0,234207.0
553792.0,227890.0
573997.0,219112.0
594099.0,211998.0
6040100.0,97336.0
6040101.0,204647.0
6041101.0,206041.0
553692.0,222899.0
533689.0,234080.0
523689.0,235235.0
533588.0,234591.0
513587.0,240166.0
503586.0,242147.0
483584.0,248543.0
453681.0,252077.0
453581.0,252706.0
453580.0,238681.0
443580.0,251563.0
0.0,-282.0
483583.0,246862.0
513687.0,239926.0
513486.0,244214.0
543691.0,229066.0
553691.0,224848.0
563793.0,222710.0
563693.0,221737.0
563692.0,221532.0
543589.0,233991.0
523688.0,237634.0
513688.0,240610.0
503686.0,241587.0
493686.0,243116.0
483685.0,241468.0
483785.0,242252.0
473784.0,243209.0
463784.0,244225.0
453884.0,245899.0
453883.0,245353.0
443883.0,244192.0
443882.0,245493.0
433982.0,246043.0
433882.0,246397.0
423982.0,247620.0
423981.0,248308.0
433881.0,250237.0
443782.0,255290.0
453782.0,251893.0
463783.0,247845.0
443781.0,256185.0
453783.0,252276.0
463682.0,257657.0
473683.0,248249.0
473684.0,254666.0
483684.0,253290.0
493685.0,253745.0
553590.0,160147.0
563591.0,228955.0
573592.0,227065.0
583594.0,229045.0
573593.0,229591.0
563592.0,230609.0
553490.0,235088.0
543489.0,235599.0
503585.0,235551.0
493585.0,252256.0
463683.0,249050.0
453682.0,256510.0
433780.0,257233.0
423780.0,257321.0
433781.0,254818.0
423881.0,255737.0
463884.0,245749.0
473785.0,242568.0
483786.0,249030.0
493786.0,247564.0
503787.0,244608.0
543690.0,234709.0
543590.0,233992.0
553591.0,232013.0
583593.0,228357.0
583493.0,226658.0
593594.0,228134.0
583492.0,227921.0
573391.0,231334.0
573390.0,231444.0
563390.0,234623.0
563389.0,234593.0
553388.0,237340.0
543387.0,235194.0
533386.0,239102.0
523386.0,241225.0
523385.0,241552.0
513384.0,250384.0
503383.0,253299.0
493383.0,254572.0
483382.0,258460.0
473481.0,259955.0
463480.0,258556.0
453480.0,259863.0
443680.0,261992.0
433579.0,261555.0
433679.0,259909.0
433680.0,256797.0
423880.0,258517.0
424082.0,253054.0
434083.0,250342.0
434084.0,248101.0
444084.0,246720.0
444085.0,250595.0
444488.0,265335.0
464995.0,289119.0
4954104.0,295429.0
4953103.0,294742.0
454893.0,283001.0
464591.0,271508.0
474491.0,264746.0
484290.0,258544.0
484190.0,255352.0
484089.0,251344.0
494089.0,245657.0
493989.0,232298.0
503990.0,230283.0
513890.0,238264.0
513990.0,226967.0
523890.0,235722.0
523891.0,234865.0
533891.0,233355.0
533791.0,232810.0
543791.0,232428.0
523790.0,237078.0
513789.0,242689.0
513788.0,242871.0
533488.0,237010.0
533487.0,238427.0
533387.0,237412.0
513385.0,244320.0
503384.0,248585.0
493382.0,255212.0
483381.0,254841.0
473381.0,258108.0
473380.0,257087.0
453479.0,260130.0
443579.0,261332.0
473885.0,242285.0
483886.0,240662.0
483887.0,238495.0
493887.0,236504.0
493988.0,234020.0
503989.0,232407.0
513991.0,225947.0
523991.0,224778.0
504090.0,229604.0
494090.0,233456.0
483988.0,236861.0
484088.0,237088.0
483987.0,239272.0
473987.0,238618.0
473986.0,240348.0
473886.0,241339.0
493484.0,249358.0
503484.0,247653.0
543388.0,233512.0
523284.0,256409.0
543084.0,269285.0
542983.0,280529.0
542883.0,291685.0
9738135.0,213939.0
9638134.0,219657.0
9738136.0,213451.0
9839137.0,204007.0
9839138.0,204217.0
9440134.0,228059.0
9241134.0,240587.0
9141133.0,246331.0
9539134.0,224047.0
9739136.0,211537.0
533084.0,279261.0
533184.0,262679.0
523285.0,255018.0
532983.0,274461.0
532781.0,302302.0
9637133.0,222471.0
9537132.0,229626.0
9737135.0,215710.0
9637134.0,221660.0
9537133.0,229847.0
9737134.0,215498.0
523184.0,257570.0
494291.0,298100.0
5250102.0,307350.0
514192.0,302543.0
523487.0,269115.0
493483.0,259705.0
483482.0,264756.0
483483.0,263669.0
542782.0,288667.0
543185.0,253212.0
533185.0,257473.0
533285.0,252283.0
513284.0,256989.0
8552138.0,258478.0
5151103.0,312095.0
514799.0,313296.0
544296.0,302653.0
533892.0,289151.0
543488.0,256997.0
514395.0,299664.0
5351104.0,304022.0
485098.0,307963.0
514294.0,304829.0
503687.0,251585.0
503788.0,248328.0
493888.0,246838.0
464086.0,252127.0
454085.0,239326.0
443984.0,255446.0
433983.0,256770.0
443681.0,264741.0
543286.0,253677.0
543186.0,251910.0
513283.0,258668.0
503283.0,259877.0
493282.0,264590.0
483281.0,266513.0
463380.0,269924.0
483280.0,270615.0
493079.0,271830.0
492979.0,271781.0
492978.0,272371.0
493080.0,271169.0
513082.0,261750.0
513183.0,260039.0
523083.0,271107.0
532881.0,278287.0
9635131.0,228473.0
9535130.0,233599.0
9535131.0,233232.0
9436130.0,236829.0
9736134.0,218341.0
8850138.0,238086.0
5249101.0,310885.0
494695.0,315451.0
533286.0,252608.0
504394.0,302886.0
504293.0,306282.0
522982.0,272159.0
9635132.0,229019.0
9736133.0,218560.0
523082.0,298688.0
493181.0,280463.0
504696.0,307991.0
9149141.0,221327.0
504091.0,304619.0
493787.0,253151.0
513182.0,261244.0
513081.0,263879.0
503181.0,264893.0
493584.0,256296.0
503182.0,264151.0
6050110.0,143710.0
554095.0,79411.0
10050150.0,83752.0
6045105.0,146315.0
6540105.0,48067.0
6535100.0,137496.0
653095.0,86445.0
7535110.0,153158.0
4040.0,-6.0
7030100.0,163605.0
652085.0,0.0
65065.0,-15.0
6565130.0,235622.0
6555120.0,292006.0
6560125.0,234630.0
5560115.0,372229.0
5055105.0,272624.0
5550105.0,241914.0
5050100.0,318630.0
5555110.0,252782.0
6055115.0,217787.0
5545100.0,225673.0
504595.0,236617.0
405090.0,262150.0
455095.0,272500.0
454590.0,267193.0
405595.0,267416.0
404080.0,215254.0
4060100.0,273388.0
603595.0,269172.0
404585.0,257809.0
403575.0,253865.0
403070.0,264744.0
453075.0,265099.0
503080.0,261448.0
6550115.0,158537.0
4065105.0,67964.0

View File

View File

@ -0,0 +1,12 @@
import cv2
def safe_crop(img, x, y, x2, y2, keepsize=True):
# The order of the arguments can be reconsidered.
img_h, img_w = img.shape[:2]
outimg = img[max(0, y) : min(img_h, y2), max(0, x) : min(img_w, x2)].copy()
reqsize_x, reqsize_y = abs(x2 - x), abs(y2 - y)
if keepsize and outimg.shape[:2] != (reqsize_y, reqsize_x):
# If the size is different from the expected size (smaller by the amount that is out of range)
outimg = cv2.resize(outimg, (reqsize_x, reqsize_y))
return outimg

View File

@ -0,0 +1,2 @@
def clamp(x, low, high):
return max(low, min(x, high))

View File

@ -0,0 +1,171 @@
import functools
import math
import sys
import timeit
def TimeitWrapper(*args, **kwargs):
"""
This decorator @TimeitWrapper() prints the function name and execution time in seconds.
:param args:
:param kwargs:
:return:
"""
def decorator(function):
@functools.wraps(function)
def wrapper(*args, **kwargs):
start = timeit.default_timer()
results = function(*args, **kwargs)
end = timeit.default_timer()
print('{} execution time: {:.10f} s'.format(function.__name__, end - start))
return results
return wrapper
return decorator
class TimeitResult(object):
"""
from https://github.com/ipython/ipython/blob/339c0d510a1f3cb2158dd8c6e7f4ac89aa4c89d8/IPython/core/magics/execution.py#L55
Object returned by the timeit magic with info about the run.
Contains the following attributes :
loops: (int) number of loops done per measurement
repeat: (int) number of times the measurement has been repeated
best: (float) best execution time / number
all_runs: (list of float) execution time of each run (in s)
"""
def __init__(self, loops, repeat, best, worst, all_runs, precision):
self.loops = loops
self.repeat = repeat
self.best = best
self.worst = worst
self.all_runs = all_runs
self._precision = precision
self.timings = [dt / self.loops for dt in all_runs]
@property
def average(self):
return math.fsum(self.timings) / len(self.timings)
@property
def stdev(self):
mean = self.average
return (math.fsum([(x - mean) ** 2 for x in self.timings]) / len(self.timings)) ** 0.5
def __str__(self):
pm = '+-'
if hasattr(sys.stdout, 'encoding') and sys.stdout.encoding:
try:
u'\xb1'.encode(sys.stdout.encoding)
pm = u'\xb1'
except:
pass
return "min:{best} max:{worst} mean:{mean} {pm} {std} per loop (mean {pm} std. dev. of {runs} run{run_plural}, {loops:,} loop{loop_plural} each)".format(
pm=pm,
runs=self.repeat,
loops=self.loops,
loop_plural="" if self.loops == 1 else "s",
run_plural="" if self.repeat == 1 else "s",
mean=format_time(self.average, self._precision),
std=format_time(self.stdev, self._precision),
best=format_time(self.best, self._precision),
worst=format_time(self.worst, self._precision),
)
def _repr_pretty_(self, p, cycle):
unic = self.__str__()
p.text(u'<TimeitResult : ' + unic + u'>')
class FPSResult(object):
"""
base https://github.com/ipython/ipython/blob/339c0d510a1f3cb2158dd8c6e7f4ac89aa4c89d8/IPython/core/magics/execution.py#L55
"""
def __init__(self, loops, repeat, best, worst, all_runs, precision):
self.loops = loops
self.repeat = repeat
self.best = 1 / best
self.worst = 1 / worst
self.all_runs = all_runs
self._precision = precision
self.fps = [1 / dt for dt in all_runs]
self.unit = "fps"
@property
def average(self):
return math.fsum(self.fps) / len(self.fps)
@property
def stdev(self):
mean = self.average
return (math.fsum([(x - mean) ** 2 for x in self.fps]) / len(self.fps)) ** 0.5
def __str__(self):
pm = '+-'
if hasattr(sys.stdout, 'encoding') and sys.stdout.encoding:
try:
u'\xb1'.encode(sys.stdout.encoding)
pm = u'\xb1'
except:
pass
return "min:{best} max:{worst} mean:{mean} {pm} {std} per loop (mean {pm} std. dev. of {runs} run{run_plural}, {loops:,} loop{loop_plural} each)".format(
pm=pm,
runs=self.repeat,
loops=self.loops,
loop_plural="" if self.loops == 1 else "s",
run_plural="" if self.repeat == 1 else "s",
mean="%.*g%s" % (self._precision, self.average, self.unit),
std="%.*g%s" % (self._precision, self.stdev, self.unit),
best="%.*g%s" % (self._precision, self.best, self.unit),
worst="%.*g%s" % (self._precision, self.worst, self.unit),
)
def _repr_pretty_(self, p, cycle):
unic = self.__str__()
p.text(u'<FPSResult : ' + unic + u'>')
def format_time(timespan, precision=3):
"""
https://github.com/ipython/ipython/blob/339c0d510a1f3cb2158dd8c6e7f4ac89aa4c89d8/IPython/core/magics/execution.py#L1473
Formats the timespan in a human readable form
"""
if timespan >= 60.0:
# we have more than a minute, format that in a human readable form
# Idea from http://snipplr.com/view/5713/
parts = [("d", 60 * 60 * 24), ("h", 60 * 60), ("min", 60), ("s", 1)]
time = []
leftover = timespan
for suffix, length in parts:
value = int(leftover / length)
if value > 0:
leftover = leftover % length
time.append(u'%s%s' % (str(value), suffix))
if leftover < 1:
break
return " ".join(time)
# Unfortunately the unicode 'micro' symbol can cause problems in
# certain terminals.
# See bug: https://bugs.launchpad.net/ipython/+bug/348466
# Try to prevent crashes by being more secure than it needs to
# E.g. eclipse is able to print a µ, but has no sys.stdout.encoding set.
units = [u"s", u"ms", u'us', "ns"] # the save value
if hasattr(sys.stdout, 'encoding') and sys.stdout.encoding:
try:
u'\xb5'.encode(sys.stdout.encoding)
units = [u"s", u"ms", u'\xb5s', "ns"]
except:
pass
scaling = [1, 1e3, 1e6, 1e9]
if timespan > 0.0:
order = min(-int(math.floor(math.log10(timespan)) // 3), 3)
else:
order = 3
return u"%.*g %s" % (precision, timespan * scaling[order], units[order])

12
poetry.lock generated
View File

@ -12,18 +12,6 @@ files = [
{file = "altgraph-0.17.3.tar.gz", hash = "sha256:ad33358114df7c9416cdb8fa1eaa5852166c505118717021c6a8c7c7abbd03dd"},
]
[[package]]
name = "beautifulsoup4"
version = "4.11.1"
description = "Screen-scraping library"
category = "main"
optional = false
python-versions = ">=3.6.0"
files = [
{file = "beautifulsoup4-4.11.1-py3-none-any.whl", hash = "sha256:58d5c3d29f5a36ffeb94f02f0d786cd53014cf9b3b3951d42e0080d8a9498d30"},
{file = "beautifulsoup4-4.11.1.tar.gz", hash = "sha256:ad9aa55b65ef2808eb405f46cf74df7fcb7044d5cbc26487f96eb2ef2e436693"},
]
[package.dependencies]
soupsieve = ">1.2"

View File

@ -18,7 +18,6 @@ pydantic = "^1.10.2"
win10toast_click = [
{ version = "^0.1.2", platform = 'win32' }
]
beautifulsoup4 = "^4.11.1"
[tool.poetry.group.dev.dependencies]
black = "^22.10.0"