From 2f1d3d9275310db87e4bda041c072dd9572eb07e Mon Sep 17 00:00:00 2001 From: Sebastian Fitt Date: Sat, 29 Apr 2023 14:32:53 +0200 Subject: [PATCH 01/11] feat: add taskipy to run tasks via poetry --- pyproject.toml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 354d688..5d680cc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,11 +20,15 @@ winotify = [ { version = "^1.1.0", platform = 'win32' } ] onnxruntime = "^1.13.1" +taskipy = "^1.10.4" [tool.poetry.group.dev.dependencies] black = "^22.10.0" pyinstaller = "^5.6.2" flake8 = "^5.0.4" +[tool.taskipy.tasks] +dev = "python eyetrackapp.py" + [build-system] requires = ["poetry-core>=1.0.0"] build-backend = "poetry.core.masonry.api" From 1d9dfeae19cd3ab8fd8252cb466b47576428c1e5 Mon Sep 17 00:00:00 2001 From: Sebastian Fitt Date: Sat, 29 Apr 2023 14:32:54 +0200 Subject: [PATCH 02/11] feat: add colorama instead of using escape codes --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 5d680cc..092795d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,6 +20,7 @@ winotify = [ { version = "^1.1.0", platform = 'win32' } ] onnxruntime = "^1.13.1" +colorama = "^0.4.6" taskipy = "^1.10.4" [tool.poetry.group.dev.dependencies] black = "^22.10.0" From 632ec2a8384db57e7a70099d26accbe3f97db774 Mon Sep 17 00:00:00 2001 From: Sebastian Fitt Date: Sat, 29 Apr 2023 14:32:54 +0200 Subject: [PATCH 03/11] feat: updated poetry.lock --- poetry.lock | 81 +++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 69 insertions(+), 12 deletions(-) diff --git a/poetry.lock b/poetry.lock index d013d62..095e6d7 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.4.2 and should not be changed by hand. +# This file is automatically @generated by Poetry and should not be changed by hand. [[package]] name = "altgraph" @@ -163,7 +163,7 @@ colorama = {version = "*", markers = "platform_system == \"Windows\""} name = "colorama" version = "0.4.6" description = "Cross-platform colored terminal text." -category = "dev" +category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" files = [ @@ -363,6 +363,18 @@ files = [ {file = "msgpack-1.0.5.tar.gz", hash = "sha256:c075544284eadc5cddc70f4757331d99dcbc16b2bbd4849d15f8aae4cf36d31c"}, ] +[[package]] +name = "mslex" +version = "0.3.0" +description = "shlex for windows" +category = "main" +optional = false +python-versions = ">=3.5" +files = [ + {file = "mslex-0.3.0-py2.py3-none-any.whl", hash = "sha256:380cb14abf8fabf40e56df5c8b21a6d533dc5cbdcfe42406bbf08dda8f42e42a"}, + {file = "mslex-0.3.0.tar.gz", hash = "sha256:4a1ac3f25025cad78ad2fe499dd16d42759f7a3801645399cce5c404415daa97"}, +] + [[package]] name = "mypy-extensions" version = "1.0.0" @@ -526,19 +538,19 @@ files = [ [[package]] name = "platformdirs" -version = "3.2.0" +version = "3.3.0" description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "platformdirs-3.2.0-py3-none-any.whl", hash = "sha256:ebe11c0d7a805086e99506aa331612429a72ca7cd52a1f0d277dc4adc20cb10e"}, - {file = "platformdirs-3.2.0.tar.gz", hash = "sha256:d5b638ca397f25f979350ff789db335903d7ea010ab28903f57b27e1b16c2b08"}, + {file = "platformdirs-3.3.0-py3-none-any.whl", hash = "sha256:ea61fd7b85554beecbbd3e9b37fb26689b227ffae38f73353cbcc1cf8bd01878"}, + {file = "platformdirs-3.3.0.tar.gz", hash = "sha256:64370d47dc3fca65b4879f89bdead8197e93e05d696d6d1816243ebae8595da5"}, ] [package.extras] -docs = ["furo (>=2022.12.7)", "proselint (>=0.13)", "sphinx (>=6.1.3)", "sphinx-autodoc-typehints (>=1.22,!=1.23.4)"] -test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.2.2)", "pytest-cov (>=4)", "pytest-mock (>=3.10)"] +docs = ["furo (>=2023.3.27)", "proselint (>=0.13)", "sphinx (>=6.1.3)", "sphinx-autodoc-typehints (>=1.23,!=1.23.4)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.3.1)", "pytest-cov (>=4)", "pytest-mock (>=3.10)"] [[package]] name = "protobuf" @@ -563,6 +575,33 @@ files = [ {file = "protobuf-4.22.3.tar.gz", hash = "sha256:23452f2fdea754a8251d0fc88c0317735ae47217e0d27bf330a30eec2848811a"}, ] +[[package]] +name = "psutil" +version = "5.9.5" +description = "Cross-platform lib for process and system monitoring in Python." +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "psutil-5.9.5-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:be8929ce4313f9f8146caad4272f6abb8bf99fc6cf59344a3167ecd74f4f203f"}, + {file = "psutil-5.9.5-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:ab8ed1a1d77c95453db1ae00a3f9c50227ebd955437bcf2a574ba8adbf6a74d5"}, + {file = "psutil-5.9.5-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:4aef137f3345082a3d3232187aeb4ac4ef959ba3d7c10c33dd73763fbc063da4"}, + {file = "psutil-5.9.5-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:ea8518d152174e1249c4f2a1c89e3e6065941df2fa13a1ab45327716a23c2b48"}, + {file = "psutil-5.9.5-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:acf2aef9391710afded549ff602b5887d7a2349831ae4c26be7c807c0a39fac4"}, + {file = "psutil-5.9.5-cp27-none-win32.whl", hash = "sha256:5b9b8cb93f507e8dbaf22af6a2fd0ccbe8244bf30b1baad6b3954e935157ae3f"}, + {file = "psutil-5.9.5-cp27-none-win_amd64.whl", hash = "sha256:8c5f7c5a052d1d567db4ddd231a9d27a74e8e4a9c3f44b1032762bd7b9fdcd42"}, + {file = "psutil-5.9.5-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:3c6f686f4225553615612f6d9bc21f1c0e305f75d7d8454f9b46e901778e7217"}, + {file = "psutil-5.9.5-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7a7dd9997128a0d928ed4fb2c2d57e5102bb6089027939f3b722f3a210f9a8da"}, + {file = "psutil-5.9.5-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:89518112647f1276b03ca97b65cc7f64ca587b1eb0278383017c2a0dcc26cbe4"}, + {file = "psutil-5.9.5-cp36-abi3-win32.whl", hash = "sha256:104a5cc0e31baa2bcf67900be36acde157756b9c44017b86b2c049f11957887d"}, + {file = "psutil-5.9.5-cp36-abi3-win_amd64.whl", hash = "sha256:b258c0c1c9d145a1d5ceffab1134441c4c5113b2417fafff7315a917a026c3c9"}, + {file = "psutil-5.9.5-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:c607bb3b57dc779d55e1554846352b4e358c10fff3abf3514a7a6601beebdb30"}, + {file = "psutil-5.9.5.tar.gz", hash = "sha256:5410638e4df39c54d957fc51ce03048acd8e6d60abc0f5107af51e5fb566eb3c"}, +] + +[package.extras] +test = ["enum34", "ipaddress", "mock", "pywin32", "wmi"] + [[package]] name = "pycodestyle" version = "2.9.1" @@ -814,14 +853,14 @@ use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] [[package]] name = "setuptools" -version = "67.6.1" +version = "67.7.2" description = "Easily download, build, install, upgrade, and uninstall Python packages" category = "dev" optional = false python-versions = ">=3.7" files = [ - {file = "setuptools-67.6.1-py3-none-any.whl", hash = "sha256:e728ca814a823bf7bf60162daf9db95b93d532948c4c0bea762ce62f60189078"}, - {file = "setuptools-67.6.1.tar.gz", hash = "sha256:257de92a9d50a60b8e22abfcbb771571fde0dbf3ec234463212027a4eeecbe9a"}, + {file = "setuptools-67.7.2-py3-none-any.whl", hash = "sha256:23aaf86b85ca52ceb801d32703f12d77517b2556af839621c641fca11287952b"}, + {file = "setuptools-67.7.2.tar.gz", hash = "sha256:f104fa03692a2602fa0fec6c6a9e63b6c8a968de13e17c026957dd1f53d80990"}, ] [package.extras] @@ -856,11 +895,29 @@ files = [ [package.dependencies] mpmath = ">=0.19" +[[package]] +name = "taskipy" +version = "1.10.4" +description = "tasks runner for python projects" +category = "main" +optional = false +python-versions = ">=3.6,<4.0" +files = [ + {file = "taskipy-1.10.4-py3-none-any.whl", hash = "sha256:b96245d7f2956d36821435acaa822143ef6d2ff6cfecc48cf0a48c4b95456c14"}, + {file = "taskipy-1.10.4.tar.gz", hash = "sha256:0006429f708f530fc7add28ca51fd41f6c6e1fbe86763ff05125e1af3e8bdc7e"}, +] + +[package.dependencies] +colorama = ">=0.4.4,<0.5.0" +mslex = {version = ">=0.3.0,<0.4.0", markers = "sys_platform == \"win32\""} +psutil = ">=5.7.2,<6.0.0" +tomli = {version = ">=2.0.1,<3.0.0", markers = "python_version >= \"3.7\" and python_version < \"4.0\""} + [[package]] name = "tomli" version = "2.0.1" description = "A lil' TOML parser" -category = "dev" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -912,4 +969,4 @@ files = [ [metadata] lock-version = "2.0" python-versions = "~3.10.0" -content-hash = "f712f691b445b82d4d43ff66867f0142eedf33d4f024bb44168235de1e137206" +content-hash = "e7ba1f268906e96f3e40127b34c43e8eaa2a2367bc3a2b471d7f6917a1edcc76" From d6d19383bd0dba3e8af39d2101658eb2dbbaf847 Mon Sep 17 00:00:00 2001 From: Sebastian Fitt Date: Sat, 29 Apr 2023 14:32:54 +0200 Subject: [PATCH 04/11] refactor: extract EyeInfo enums and dataclasses --- EyeTrackApp/camera_widget.py | 8 +++--- EyeTrackApp/eye.py | 26 ++++++++++++++++++ EyeTrackApp/eye_processor.py | 51 +++++++++++------------------------- EyeTrackApp/ransac.py | 4 +-- 4 files changed, 48 insertions(+), 41 deletions(-) create mode 100644 EyeTrackApp/eye.py diff --git a/EyeTrackApp/camera_widget.py b/EyeTrackApp/camera_widget.py index 9043d70..361757d 100644 --- a/EyeTrackApp/camera_widget.py +++ b/EyeTrackApp/camera_widget.py @@ -2,7 +2,7 @@ import PySimpleGUI as sg from config import EyeTrackConfig from config import EyeTrackSettingsConfig from threading import Event, Thread -from eye_processor import EyeProcessor, InformationOrigin +from eye_processor import EyeProcessor, EyeInfoOrigin from enum import Enum from queue import Queue, Empty from camera import Camera, CameraState @@ -312,7 +312,7 @@ class CameraWidget: graph = window[self.gui_output_graph] graph.erase() - if eye_info.info_type != InformationOrigin.FAILURE: #and not eye_info.blink: + if eye_info.info_type != EyeInfoOrigin.FAILURE: #and not eye_info.blink: graph.update(background_color="white") if not np.isnan(eye_info.x) and not np.isnan(eye_info.y): @@ -332,10 +332,10 @@ class CameraWidget: # elif eye_info.blink: # graph.update(background_color="#6f4ca1") - elif eye_info.info_type == InformationOrigin.FAILURE: + elif eye_info.info_type == EyeInfoOrigin.FAILURE: graph.update(background_color="red") # Relay information to OSC - if eye_info.info_type != InformationOrigin.FAILURE: + if eye_info.info_type != EyeInfoOrigin.FAILURE: self.osc_queue.put((self.eye_id, eye_info)) except Empty: pass diff --git a/EyeTrackApp/eye.py b/EyeTrackApp/eye.py new file mode 100644 index 0000000..04eb6ef --- /dev/null +++ b/EyeTrackApp/eye.py @@ -0,0 +1,26 @@ +from dataclasses import dataclass +from enum import Enum, IntEnum + +class EyeId(IntEnum): + RIGHT = 0 + LEFT = 1 + BOTH = 2 + SETTINGS = 3 + + +class EyeInfoOrigin(Enum): + RANSAC = 1 + BLOB = 2 + FAILURE = 3 + HSF = 4 + HSRAC = 5 + DADDY = 6 + + +@dataclass +class EyeInfo: + info_type: EyeInfoOrigin + x: float + y: float + pupil_dialation: float + blink: float diff --git a/EyeTrackApp/eye_processor.py b/EyeTrackApp/eye_processor.py index 5b3c471..7abd9f1 100644 --- a/EyeTrackApp/eye_processor.py +++ b/EyeTrackApp/eye_processor.py @@ -59,29 +59,9 @@ from ransac import * from hsrac import External_Run_HSRACS from blink import * - +from eye import EyeInfo, EyeInfoOrigin from intensity_eye_open import * -class InformationOrigin(Enum): - RANSAC = 1 - BLOB = 2 - FAILURE = 3 - HSF = 4 - HSRAC = 5 - DADDY = 6 - -bbb = 0 -@dataclass -class EyeInformation: - info_type: InformationOrigin - x: float - y: float - pupil_dialation: float - blink: float - - -lowb = np.array(0) - def run_once(f): def wrapper(*args, **kwargs): @@ -179,8 +159,8 @@ class EyeProcessor: self.prev_x = None self.prev_y = None self.bd_blink = False - self.current_algo = InformationOrigin.HSRAC - + self.current_algo = EyeInfoOrigin.HSRAC + try: min_cutoff = float(self.settings.gui_min_cutoff) # 0.0004 @@ -196,7 +176,7 @@ class EyeProcessor: beta=beta ) - def output_images_and_update(self, threshold_image, output_information: EyeInformation): + def output_images_and_update(self, threshold_image, output_information: EyeInfo): try: image_stack = np.concatenate( ( @@ -280,7 +260,7 @@ class EyeProcessor: else: self.eyeopen = ibo - self.output_images_and_update(self.thresh, EyeInformation(self.current_algo, self.out_x, self.out_y, 0, self.eyeopen)) + self.output_images_and_update(self.thresh, EyeInfo(self.current_algo, self.out_x, self.out_y, 0, self.eyeopen)) @@ -293,7 +273,7 @@ class EyeProcessor: self.rawx, self.rawy, self.eyeopen = self.er_daddy.run(self.current_image_gray) # Daddy also uses a one euro filter, so I'll have to use it twice, but I'm not going to think too much about it. self.out_x, self.out_y = cal.cal_osc(self, self.rawx, self.rawy) - self.current_algorithm = InformationOrigin.DADDY + self.current_algorithm = EyeInfoOrigin.DADDY def HSRACM(self): # todo: added process to initialise er_hsrac when resolution changes @@ -302,26 +282,26 @@ class EyeProcessor: self.prev_x = self.rawx self.prev_y = self.rawy self.out_x, self.out_y = cal.cal_osc(self, self.rawx, self.rawy) - self.current_algorithm = InformationOrigin.HSRAC + self.current_algorithm = EyeInfoOrigin.HSRAC def HSFM(self): # todo: added process to initialise er_hsf when resolution changes self.rawx, self.rawy, self.thresh = self.er_hsf.run(self.current_image_gray) self.eyeopen = self.ibo.intense(self.rawx, self.rawy, self.current_image_gray) self.out_x, self.out_y = cal.cal_osc(self, self.rawx, self.rawy) - self.current_algorithm = InformationOrigin.HSF + self.current_algorithm = EyeInfoOrigin.HSF def RANSAC3DM(self): current_image_gray_copy = self.current_image_gray.copy() # Duplicate before overwriting in RANSAC3D. self.rawx, self.rawy, self.thresh = RANSAC3D(self) self.out_x, self.out_y = cal.cal_osc(self, self.rawx, self.rawy) - self.current_algorithm = InformationOrigin.RANSAC + self.current_algorithm = EyeInfoOrigin.RANSAC def BLOBM(self): print("calling") self.rawx, self.rawy, self.thresh = BLOB(self) self.out_x, self.out_y = cal.cal_osc(self, self.rawx, self.rawy) - self.current_algorithm = InformationOrigin.BLOB + self.current_algorithm = EyeInfoOrigin.BLOB @@ -359,6 +339,7 @@ class EyeProcessor: # Run the following somewhere # self.daddy = External_Run_DADDY() + self.firstalgo = None self.secondalgo = None self.thirdalgo = None @@ -473,23 +454,23 @@ class EyeProcessor: # cx, cy, thresh = HSRAC(self) # out_x, out_y = cal_osc(self, cx, cy) # if cx == 0: - # self.output_images_and_update(thresh, EyeInformation(InformationOrigin.HSRAC, out_x, out_y, 0, True)) #update app + # self.output_images_and_update(thresh, EyeInfo(EyeInfoOrigin.HSRAC, out_x, out_y, 0, True)) #update app # else: - # self.output_images_and_update(thresh, EyeInformation(InformationOrigin.HSRAC, out_x, out_y, 0, self.blinkvalue)) + # self.output_images_and_update(thresh, EyeInfo(EyeInfoOrigin.HSRAC, out_x, out_y, 0, self.blinkvalue)) # cx, cy, thresh = RANSAC3D(self) # out_x, out_y = cal_osc(self, cx, cy) - # self.output_images_and_update(thresh, EyeInformation(InformationOrigin.RANSAC, out_x, out_y, 0, False)) #update app + # self.output_images_and_update(thresh, EyeInfo(EyeInfoOrigin.RANSAC, out_x, out_y, 0, False)) #update app # cx, cy, larger_threshold = BLOB(self) # out_x, out_y = cal_osc(self, cx, cy) - # self.output_images_and_update(larger_threshold, EyeInformation(InformationOrigin.BLOB, out_x, out_y, 0, False)) #update app + # self.output_images_and_update(larger_threshold, EyeInfo(EyeInfoOrigin.BLOB, out_x, out_y, 0, False)) #update app #center_x, center_y, frame = HSF(self) #run algo #out_x, out_y = cal_osc(self, center_x, center_y) #filter and calibrate - #self.output_images_and_update(frame, EyeInformation(InformationOrigin.HSF, out_x, out_y, 0, False)) #update app + #self.output_images_and_update(frame, EyeInfo(EyeInfoOrigin.HSF, out_x, out_y, 0, False)) #update app self.ALGOSELECT() #run our algos in priority order set in settings #self.BLOBM() diff --git a/EyeTrackApp/ransac.py b/EyeTrackApp/ransac.py index 1800a40..d50766d 100644 --- a/EyeTrackApp/ransac.py +++ b/EyeTrackApp/ransac.py @@ -335,7 +335,7 @@ def RANSAC3D(self): self.failed = self.failed + 1 #we have failed, move onto next algo return 0, 0, thresh # Shove a concatenated image out to the main GUI thread for rendering - #self.output_images_and_update(thresh, EyeInformation(InformationOrigin.FAILURE, 0 ,0, 0, False)) + #self.output_images_and_update(thresh, EyeInfo(EyeInfoOrigin.FAILURE, 0 ,0, 0, False)) #self.output_images_and_update(thresh, output_info) #except: - # self.output_images_and_update(thresh, EyeInformation(InformationOrigin.RANSAC, out_x, out_y, 0, self.blinkvalue)) + # self.output_images_and_update(thresh, EyeInfo(EyeInfoOrigin.RANSAC, out_x, out_y, 0, self.blinkvalue)) From 446a20cea822189f1f5adf3d10b9d9e252b5186e Mon Sep 17 00:00:00 2001 From: Sebastian Fitt Date: Sat, 29 Apr 2023 14:32:54 +0200 Subject: [PATCH 05/11] refactor: cleanup imports --- EyeTrackApp/config.py | 9 +++++---- EyeTrackApp/eyetrackapp.py | 20 +++++++++++--------- EyeTrackApp/osc_calibrate_filter.py | 6 ++---- EyeTrackApp/settings_widget.py | 9 +++------ 4 files changed, 21 insertions(+), 23 deletions(-) diff --git a/EyeTrackApp/config.py b/EyeTrackApp/config.py index c0f1c40..fac07a6 100644 --- a/EyeTrackApp/config.py +++ b/EyeTrackApp/config.py @@ -1,10 +1,11 @@ -from typing import Union, Dict -from osc import EyeId -import os.path import json -from pydantic import BaseModel +import os.path import shutil +from eye import EyeId +from pydantic import BaseModel +from typing import Union + CONFIG_FILE_NAME: str = "eyetrack_settings.json" BACKUP_CONFIG_FILE_NAME: str = "eyetrack_settings.backup" diff --git a/EyeTrackApp/eyetrackapp.py b/EyeTrackApp/eyetrackapp.py index af80d13..ecc8817 100644 --- a/EyeTrackApp/eyetrackapp.py +++ b/EyeTrackApp/eyetrackapp.py @@ -1,15 +1,17 @@ import os -from utils.misc_utils import is_nt -from osc import VRChatOSCReceiver, VRChatOSC, EyeId -from config import EyeTrackConfig -from camera_widget import CameraWidget -from settings_widget import SettingsWidget - -import queue -import threading +import os import PySimpleGUI as sg -import os +import queue import requests +import threading + +from camera_widget import CameraWidget +from config import EyeTrackConfig +from eye import EyeId +from osc import VRChatOSCReceiver, VRChatOSC +from settings_widget import SettingsWidget +from utils.misc_utils import is_nt + if is_nt: from winotify import Notification diff --git a/EyeTrackApp/osc_calibrate_filter.py b/EyeTrackApp/osc_calibrate_filter.py index b9a315d..5321105 100644 --- a/EyeTrackApp/osc_calibrate_filter.py +++ b/EyeTrackApp/osc_calibrate_filter.py @@ -1,9 +1,7 @@ -import sys -from utils.misc_utils import PlaySound, SND_FILENAME, SND_ASYNC import numpy as np + from enum import IntEnum -from config import EyeTrackConfig -from config import EyeTrackSettingsConfig +from utils.misc_utils import PlaySound, SND_FILENAME, SND_ASYNC class EyeId(IntEnum): RIGHT = 0 diff --git a/EyeTrackApp/settings_widget.py b/EyeTrackApp/settings_widget.py index 378836d..312d35e 100644 --- a/EyeTrackApp/settings_widget.py +++ b/EyeTrackApp/settings_widget.py @@ -1,12 +1,9 @@ import PySimpleGUI as sg + from config import EyeTrackSettingsConfig -from threading import Event, Thread -from eye_processor import EyeProcessor, InformationOrigin -from enum import Enum -from queue import Queue, Empty -from camera import Camera, CameraState -import cv2 from osc import EyeId +from queue import Queue +from threading import Event class SettingsWidget: def __init__(self, widget_id: EyeId, main_config: EyeTrackSettingsConfig, osc_queue: Queue): From ca5622244528e60ad40e2cc994a80b39afe4bff0 Mon Sep 17 00:00:00 2001 From: Sebastian Fitt Date: Sat, 29 Apr 2023 14:32:54 +0200 Subject: [PATCH 06/11] fix: pinv for pseudo inverse when singular matrix --- EyeTrackApp/hsrac.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/EyeTrackApp/hsrac.py b/EyeTrackApp/hsrac.py index 8f9af11..525e362 100644 --- a/EyeTrackApp/hsrac.py +++ b/EyeTrackApp/hsrac.py @@ -112,9 +112,9 @@ def fit_rotated_ellipse_ransac(data: np.ndarray, sfc: np.random.Generator, iter_ np.matmul(dm_rng_swap_trans, dm_rng_swap, out=dm_rng_5x5) # np.linalg.solve(np.matmul(dm_rng_swap_trans, dm_rng_swap), dm_rng_swap_trans) # solve is slow https://github.com/bogovicj/JaneliaMLCourse/issues/1 - #_umath_linalg.inv(dm_rng_5x5, signature='d->d', - # extobj=inv_ext, out=dm_rng_5x5) dm_rng_5x5 = np.linalg.pinv(dm_rng_5x5) + # _umath_linalg.inv(dm_rng_5x5, signature='d->d', + # extobj=inv_ext, out=dm_rng_5x5) np.matmul(dm_rng_5x5, dm_rng_swap_trans, out=dm_rng_p5smp) np.matmul(dm_rng_p5smp, dm_rng_six, out=dm_rng_p_npaxis) From 4def96575969e3f22a4a915c9e5e71576403c2c8 Mon Sep 17 00:00:00 2001 From: Sebastian Fitt Date: Sat, 29 Apr 2023 14:32:54 +0200 Subject: [PATCH 07/11] fix: ensure serial is closed when thread crashed --- EyeTrackApp/camera.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/EyeTrackApp/camera.py b/EyeTrackApp/camera.py index 29e31e2..1039208 100644 --- a/EyeTrackApp/camera.py +++ b/EyeTrackApp/camera.py @@ -56,9 +56,11 @@ class Camera: self.buffer = b'' self.error_message = "\033[93m[WARN] Capture source {} not found, retrying...\033[0m" + def __del__(self): if self.serial_connection is not None: self.serial_connection.close() + def set_output_queue(self, camera_output_outgoing: "queue.Queue"): self.camera_output_outgoing = camera_output_outgoing From 3b5582de6eab3b3e65eb7d76c631cd089a06ee40 Mon Sep 17 00:00:00 2001 From: Sebastian Fitt Date: Sat, 29 Apr 2023 14:32:54 +0200 Subject: [PATCH 08/11] feat: slightly reduce dropped frame count --- EyeTrackApp/camera.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/EyeTrackApp/camera.py b/EyeTrackApp/camera.py index 1039208..8b93721 100644 --- a/EyeTrackApp/camera.py +++ b/EyeTrackApp/camera.py @@ -162,24 +162,24 @@ class Camera: # Create jpeg frame from byte string image = cv2.imdecode(np.fromstring(jpeg, dtype=np.uint8), cv2.IMREAD_UNCHANGED) if image is None: - print("image not found") + print("\033[93m[WARN] Frame drop. Corrupted JPEG.\033[0m") return # Discard the serial buffer. This is due to the fact that it # may build up some outdated frames. A bit of a workaround here tbh. - self.serial_connection.reset_input_buffer() - self.buffer = b'' + if self.serial_connection.in_waiting > 32768: + print(f"\033[94m[INFO] Discarding the serial buffer ({self.serial_connection.in_waiting} bytes\033[0m") + self.serial_connection.reset_input_buffer() + self.buffer = b'' # Calculate the fps. current_frame_time = time.time() delta_time = current_frame_time - self.last_frame_time self.last_frame_time = current_frame_time if delta_time > 0: self.fps = 1 / delta_time + # print(f'FPS: {int(self.fps)}') self.frame_number = self.frame_number + 1 if should_push: self.push_image_to_queue(image, self.frame_number, self.fps) - - except UnboundLocalError as ex: - print(ex) except Exception: print("\033[93m[WARN]Serial capture source problem, assuming camera disconnected, waiting for reconnect.\033[0m") self.serial_connection.close() @@ -204,13 +204,14 @@ class Camera: xonxoff=False, dsrdtr=False, rtscts=False) + # Set explicit buffer size for serial. + conn.set_buffer_size(rx_size = 65536, tx_size = 65536) - conn.reset_input_buffer() - - print(f"\033[94m[INFO] Serial Tracker successfully connected on {port}\033[0m") + print(f"\033[94m[INFO] ETVR Serial Tracker device connected on {port}\033[0m") self.serial_connection = conn self.camera_status = CameraState.CONNECTED except Exception: + print(f"\033[94m[INFO] Failed to connect on {port}\033[0m") self.camera_status = CameraState.DISCONNECTED def push_image_to_queue(self, image, frame_number, fps): From 6d7630cf6029e2de78aa41bdd468ed4d59898951 Mon Sep 17 00:00:00 2001 From: Sebastian Fitt Date: Sat, 29 Apr 2023 14:32:54 +0200 Subject: [PATCH 09/11] fix: when address is set but no devices connected --- EyeTrackApp/camera.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/EyeTrackApp/camera.py b/EyeTrackApp/camera.py index 8b93721..e0945d4 100644 --- a/EyeTrackApp/camera.py +++ b/EyeTrackApp/camera.py @@ -155,6 +155,8 @@ class Camera: return jpeg def get_serial_camera_picture(self, should_push): + if self.serial_connection is None: + return try: if self.serial_connection.in_waiting: jpeg = self.get_next_jpeg_frame() From 2efa3c35896d1b10e037afa8c12fc9b780c977be Mon Sep 17 00:00:00 2001 From: Sebastian Fitt Date: Sat, 29 Apr 2023 14:40:44 +0200 Subject: [PATCH 10/11] feat: more readable logging with colorama --- EyeTrackApp/camera.py | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/EyeTrackApp/camera.py b/EyeTrackApp/camera.py index e0945d4..8f179b8 100644 --- a/EyeTrackApp/camera.py +++ b/EyeTrackApp/camera.py @@ -1,13 +1,14 @@ -from config import EyeTrackConfig -from enum import Enum import cv2 +import numpy as np import queue import serial import serial.tools.list_ports import threading import time -import numpy as np +from colorama import Fore +from config import EyeTrackConfig +from enum import Enum WAIT_TIME = 0.1 @@ -55,7 +56,7 @@ class Camera: self.start = True self.buffer = b'' - self.error_message = "\033[93m[WARN] Capture source {} not found, retrying...\033[0m" + self.error_message = f"{Fore.YELLOW}[WARN] Capture source {{}} not found, retrying...{Fore.RESET}" def __del__(self): if self.serial_connection is not None: @@ -67,7 +68,7 @@ class Camera: def run(self): while True: if self.cancellation_event.is_set(): - print("\033[94m[INFO] Exiting Capture thread\033[0m") + print(f"{Fore.CYAN}[INFO] Exiting Capture thread{Fore.RESET}") return should_push = True # If things aren't open, retry until they are. Don't let read requests come in any earlier @@ -130,7 +131,7 @@ class Camera: if should_push: self.push_image_to_queue(image, frame_number, self.fps) except: - print("\033[93m[WARN] Capture source problem, assuming camera disconnected, waiting for reconnect.\033[0m") + print(f"{Fore.YELLOW}[WARN] Capture source problem, assuming camera disconnected, waiting for reconnect.{Fore.RESET}") self.camera_status = CameraState.DISCONNECTED pass @@ -164,12 +165,12 @@ class Camera: # Create jpeg frame from byte string image = cv2.imdecode(np.fromstring(jpeg, dtype=np.uint8), cv2.IMREAD_UNCHANGED) if image is None: - print("\033[93m[WARN] Frame drop. Corrupted JPEG.\033[0m") + print(f"{Fore.YELLOW}[WARN] Frame drop. Corrupted JPEG.{Fore.RESET}") return # Discard the serial buffer. This is due to the fact that it # may build up some outdated frames. A bit of a workaround here tbh. if self.serial_connection.in_waiting > 32768: - print(f"\033[94m[INFO] Discarding the serial buffer ({self.serial_connection.in_waiting} bytes\033[0m") + print(f"{Fore.CYAN}[INFO] Discarding the serial buffer ({self.serial_connection.in_waiting} bytes{Fore.RESET}") self.serial_connection.reset_input_buffer() self.buffer = b'' # Calculate the fps. @@ -183,7 +184,7 @@ class Camera: if should_push: self.push_image_to_queue(image, self.frame_number, self.fps) except Exception: - print("\033[93m[WARN]Serial capture source problem, assuming camera disconnected, waiting for reconnect.\033[0m") + print(f"{Fore.YELLOW}[WARN] Serial capture source problem, assuming camera disconnected, waiting for reconnect.{Fore.RESET}") self.serial_connection.close() self.camera_status = CameraState.DISCONNECTED pass @@ -209,11 +210,11 @@ class Camera: # Set explicit buffer size for serial. conn.set_buffer_size(rx_size = 65536, tx_size = 65536) - print(f"\033[94m[INFO] ETVR Serial Tracker device connected on {port}\033[0m") + print(f"{Fore.CYAN}[INFO] ETVR Serial Tracker device connected on {port}{Fore.RESET}") self.serial_connection = conn self.camera_status = CameraState.CONNECTED except Exception: - print(f"\033[94m[INFO] Failed to connect on {port}\033[0m") + print(f"{Fore.CYAN}[INFO] Failed to connect on {port}{Fore.RESET}") self.camera_status = CameraState.DISCONNECTED def push_image_to_queue(self, image, frame_number, fps): @@ -222,6 +223,6 @@ class Camera: qsize = self.camera_output_outgoing.qsize() if qsize > 1: print( - f"\033[91m[WARN] CAPTURE QUEUE BACKPRESSURE OF {qsize}. CHECK FOR CRASH OR TIMING ISSUES IN ALGORITHM.\033[0m") + f"{Fore.YELLOW}[WARN] CAPTURE QUEUE BACKPRESSURE OF {qsize}. CHECK FOR CRASH OR TIMING ISSUES IN ALGORITHM.{Fore.RESET}") self.camera_output_outgoing.put((image, frame_number, fps)) self.capture_event.clear() From 61637442c61a8c82e2b38c491d3a409080ccf2a9 Mon Sep 17 00:00:00 2001 From: Sebastian Fitt Date: Sat, 29 Apr 2023 15:48:29 +0200 Subject: [PATCH 11/11] feat: show bitrate, fps, latency in tracking mode --- EyeTrackApp/camera.py | 19 +++++---- EyeTrackApp/camera_widget.py | 81 +++++++++++++++++++++++------------- 2 files changed, 63 insertions(+), 37 deletions(-) diff --git a/EyeTrackApp/camera.py b/EyeTrackApp/camera.py index 8f179b8..f6e9e78 100644 --- a/EyeTrackApp/camera.py +++ b/EyeTrackApp/camera.py @@ -53,6 +53,7 @@ class Camera: self.last_frame_time = time.time() self.frame_number = 0 self.fps = 0 + self.bps = 0 self.start = True self.buffer = b'' @@ -128,6 +129,7 @@ class Camera: raise RuntimeError("Problem while getting frame") frame_number = self.wired_camera.get(cv2.CAP_PROP_POS_FRAMES) self.fps = self.wired_camera.get(cv2.CAP_PROP_FPS) + self.bps = image.nbytes if should_push: self.push_image_to_queue(image, frame_number, self.fps) except: @@ -156,10 +158,11 @@ class Camera: return jpeg def get_serial_camera_picture(self, should_push): - if self.serial_connection is None: + conn = self.serial_connection + if conn is None: return try: - if self.serial_connection.in_waiting: + if conn.in_waiting: jpeg = self.get_next_jpeg_frame() if jpeg: # Create jpeg frame from byte string @@ -169,9 +172,9 @@ class Camera: return # Discard the serial buffer. This is due to the fact that it # may build up some outdated frames. A bit of a workaround here tbh. - if self.serial_connection.in_waiting > 32768: - print(f"{Fore.CYAN}[INFO] Discarding the serial buffer ({self.serial_connection.in_waiting} bytes{Fore.RESET}") - self.serial_connection.reset_input_buffer() + if conn.in_waiting >= 32768: + print(f"{Fore.CYAN}[INFO] Discarding the serial buffer ({conn.in_waiting} bytes){Fore.RESET}") + conn.reset_input_buffer() self.buffer = b'' # Calculate the fps. current_frame_time = time.time() @@ -179,13 +182,13 @@ class Camera: self.last_frame_time = current_frame_time if delta_time > 0: self.fps = 1 / delta_time - # print(f'FPS: {int(self.fps)}') + self.bps = len(jpeg) / delta_time self.frame_number = self.frame_number + 1 if should_push: self.push_image_to_queue(image, self.frame_number, self.fps) except Exception: print(f"{Fore.YELLOW}[WARN] Serial capture source problem, assuming camera disconnected, waiting for reconnect.{Fore.RESET}") - self.serial_connection.close() + conn.close() self.camera_status = CameraState.DISCONNECTED pass @@ -208,7 +211,7 @@ class Camera: dsrdtr=False, rtscts=False) # Set explicit buffer size for serial. - conn.set_buffer_size(rx_size = 65536, tx_size = 65536) + conn.set_buffer_size(rx_size = 32768, tx_size = 32768) print(f"{Fore.CYAN}[INFO] ETVR Serial Tracker device connected on {port}{Fore.RESET}") self.serial_connection = conn diff --git a/EyeTrackApp/camera_widget.py b/EyeTrackApp/camera_widget.py index 361757d..d61b23f 100644 --- a/EyeTrackApp/camera_widget.py +++ b/EyeTrackApp/camera_widget.py @@ -1,6 +1,7 @@ import PySimpleGUI as sg from config import EyeTrackConfig from config import EyeTrackSettingsConfig +from collections import deque from threading import Event, Thread from eye_processor import EyeProcessor, EyeInfoOrigin from enum import Enum @@ -24,6 +25,8 @@ class CameraWidget: self.gui_save_tracking_button = f"-SAVETRACKINGBUTTON{widget_id}-" self.gui_tracking_layout = f"-TRACKINGLAYOUT{widget_id}-" self.gui_tracking_image = f"-IMAGE{widget_id}-" + self.gui_tracking_fps = f"-TRACKINGFPS{widget_id}-" + self.gui_tracking_bps = f"-TRACKINGBPS{widget_id}-" self.gui_output_graph = f"-OUTPUTGRAPH{widget_id}-" self.gui_restart_calibration = f"-RESTARTCALIBRATION{widget_id}-" self.gui_stop_calibration = f"-STOPCALIBRATION{widget_id}-" @@ -46,6 +49,35 @@ class CameraWidget: else: raise RuntimeError("\033[91m[WARN] Cannot have a camera widget represent both eyes!\033[0m") + self.cancellation_event = Event() + # Set the event until start is called, otherwise we can block if shutdown is called. + self.cancellation_event.set() + self.capture_event = Event() + self.capture_queue = Queue() + self.roi_queue = Queue() + + self.image_queue = Queue() + + self.ransac = EyeProcessor( + self.config, + self.settings_config, + self.cancellation_event, + self.capture_event, + self.capture_queue, + self.image_queue, + self.eye_id, + ) + + self.camera_status_queue = Queue() + self.camera = Camera( + self.config, + 0, + self.cancellation_event, + self.capture_event, + self.camera_status_queue, + self.capture_queue, + ) + self.roi_layout = [ [ sg.Graph( @@ -82,6 +114,8 @@ class CameraWidget: [ sg.Text("Mode:", background_color='#424042'), sg.Text("Calibrating", key=self.gui_mode_readout, background_color='#424042'), + sg.Text("", key=self.gui_tracking_fps, background_color='#424042'), + sg.Text("", key=self.gui_tracking_bps, background_color='#424042'), # sg.Checkbox( # "Circle crop:", # default=self.config.gui_circular_crop, @@ -123,40 +157,23 @@ class CameraWidget: ], ] - self.cancellation_event = Event() - # Set the event until start is called, otherwise we can block if shutdown is called. - self.cancellation_event.set() - self.capture_event = Event() - self.capture_queue = Queue() - self.roi_queue = Queue() - - self.image_queue = Queue() - - self.ransac = EyeProcessor( - self.config, - self.settings_config, - self.cancellation_event, - self.capture_event, - self.capture_queue, - self.image_queue, - self.eye_id, - ) - - self.camera_status_queue = Queue() - self.camera = Camera( - self.config, - 0, - self.cancellation_event, - self.capture_event, - self.camera_status_queue, - self.capture_queue, - ) - self.x0, self.y0 = None, None self.x1, self.y1 = None, None self.figure = None self.is_mouse_up = True self.in_roi_mode = False + self.movavg_fps_queue = deque(maxlen=120) + self.movavg_bps_queue = deque(maxlen=120) + + def _movavg_fps(self, next_fps): + self.movavg_fps_queue.append(next_fps) + fps = round(sum(self.movavg_fps_queue) / len(self.movavg_fps_queue)) + millisec = round((1 / fps if fps else 0) * 1000) + return f"{fps} Fps {millisec} ms" + + def _movavg_bps(self, next_bps): + self.movavg_bps_queue.append(next_bps) + return f"{sum(self.movavg_bps_queue) / len(self.movavg_bps_queue) * 0.001 * 0.001 * 8:.3f} Mbps" def started(self): return not self.cancellation_event.is_set() @@ -261,6 +278,10 @@ class CameraWidget: needs_roi_set = self.config.roi_window_h <= 0 or self.config.roi_window_w <= 0 + # TODO: Refactor if statements below... + window[self.gui_tracking_fps].update('') + window[self.gui_tracking_bps].update('') + if self.config.capture_source is None or self.config.capture_source == "": window[self.gui_mode_readout].update("Waiting for camera address") window[self.gui_roi_message].update(visible=False) @@ -275,6 +296,8 @@ class CameraWidget: window[self.gui_mode_readout].update("Calibration") else: window[self.gui_mode_readout].update("Tracking") + window[self.gui_tracking_fps].update(self._movavg_fps(self.camera.fps)) + window[self.gui_tracking_bps].update(self._movavg_bps(self.camera.bps)) if self.in_roi_mode: try: