Feature/etvr module support (#106)

* initial changes

* Mostly clean up, refactor registering listeners to make sense, backport tests

* Add initial implementation of VRCFTModuleSender

* Add basic GUI for the modules settings

* Fix tooltip descriptions

# TODO:
# - there's ghosts in the machine - vrc osc is not working properly
# - min/maxing will require field combinators in the modules lmao

* Fix type validation bugs, fix typos

# TODO:
# - there's ghosts in the machine - vrc osc is not working properly
# - min/maxing will require field combinators in the modules lmao

* Add checkbox to switch to ETVR Module

# TODO:
# - there's ghosts in the machine - vrc osc is not working properly
# - min/maxing will require field combinators in the modules lmao

* Black stuff

# TODO:
# - there's ghosts in the machine - vrc osc is not working properly
# - min/maxing will require field combinators in the modules lmao

* Remove coverage by default

# TODO:
# - there's ghosts in the machine - vrc osc is not working properly
# - min/maxing will require field combinators in the modules lmao

* Fix timeout in tests

# TODO:
# - there's ghosts in the machine - vrc osc is not working properly
# - min/maxing will require field combinators in the modules lmao

* HEAVY WIP: Refactor native output,

NOTE:

I brought back the entire old OSC implementation as a live reference, this will be removed once I'm done.

This also lays ground for other modes as they're pretty similar

# TODO:
# - there's ghosts in the machine - vrc osc is not working properly
# - min/maxing will require field combinators in the modules lmao

* HEAVY WIP: Refactor v1 params output,

# TODO:
# - min/maxing will require field combinators in the modules lmao

* HEAVY WIP: Refactor v2 params output

# TODO:
# - min/maxing will require field combinators in the modules lmao

* Finish refactoring v2 and v1, fixup tests, refactor native

# TODO:
# - min/maxing will require field combinators in the modules lmao

* Add tests for v1 params

# TODO:
# - min/maxing will require field combinators in the modules lmao

* Add tests for native params

# TODO:
# - min/maxing will require field combinators in the modules lmao

* Fix OSC not getting up after config reset. Remove reset command, config sends everything changed anyway, sunset the idea of using single client and thus simplify the code a bit

# TODO:
# - min/maxing will require field combinators in the modules lmao

* Rename gui_PortNumber to gui_VRCFTModulePort for readability

# TODO:
# - min/maxing will require field combinators in the modules lmao

* Cleanup EyeID usage

# TODO:
# - min/maxing will require field combinators in the modules lmao

* Cleanup osc after rebase

# TODO:
# - min/maxing will require field combinators in the modules lmao

* Make VRChatOSCSender a bit more readable

# TODO:
# - min/maxing will require field combinators in the modules lmao

* Remove unsued VRChatOSCReceiver, this is taken care of by generic OSCReceiver

# TODO:
# - min/maxing will require field combinators in the modules lmao

* Commit crimes with try_convert_to_float to make osc, pysimplegui and pydantic happy

* Cleanup after merge

* Disable emulation by default

* Fix OSCReceiver crashing on unknown addresses

* Adjust VRCFT Module settings to look better in game

* Fix recalibrate and recenter for OSC only working for the right eye

* Fix save and restart button not restarting the tracking

* Fix broken tracking on v1 params for eye_x, clean up implementation

* Fix regular value being passed to OSC listeners instead of OSCMessage

* Add a TODO, probably to be ignored

* Add support for custom ETVR VRCFT Module listening address
This commit is contained in:
Lorow 2024-07-04 20:51:19 +02:00 committed by GitHub
parent e171ad169b
commit 7e41a65aac
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
35 changed files with 2521 additions and 1318 deletions

File diff suppressed because it is too large Load Diff

View File

@ -28,15 +28,6 @@ LICENSE: GNU GPLv3
"""
import cv2
import numpy as np
from enum import IntEnum
class EyeId(IntEnum):
RIGHT = 0
LEFT = 1
BOTH = 2
SETTINGS = 3
def BLOB(self):

View File

@ -31,9 +31,8 @@ import serial
import serial.tools.list_ports
import threading
import time
import platform
from colorama import Fore
from config import EyeTrackConfig
from config import EyeTrackCameraConfig
from enum import Enum
import psutil, os
import sys
@ -71,16 +70,15 @@ def is_serial_capture_source(addr: str) -> bool:
"""
Returns True if the capture source address is a serial port.
"""
return (addr.startswith("COM") # Windows
or addr.startswith("/dev/cu") # macOS
or addr.startswith("/dev/tty") # Linux
return (
addr.startswith("COM") or addr.startswith("/dev/cu") or addr.startswith("/dev/tty") # Windows # macOS # Linux
)
class Camera:
def __init__(
self,
config: EyeTrackConfig,
config: EyeTrackCameraConfig,
camera_index: int,
cancellation_event: "threading.Event",
capture_event: "threading.Event",
@ -129,7 +127,9 @@ class Camera:
while True:
if self.cancellation_event.is_set():
print(f"{Fore.CYAN}[INFO] Exiting Capture thread{Fore.RESET}")
# openCV won't switch to a new source if provided with one
# so, we have to manually release the camera on exit
self.cv2_camera.release()
return
should_push = True
# If things aren't open, retry until they are. Don't let read requests come in any earlier
@ -257,20 +257,14 @@ class Camera:
jpeg = self.get_next_jpeg_frame()
if jpeg:
# Create jpeg frame from byte string
image = cv2.imdecode(
np.fromstring(jpeg, dtype=np.uint8), cv2.IMREAD_UNCHANGED
)
image = cv2.imdecode(np.fromstring(jpeg, dtype=np.uint8), cv2.IMREAD_UNCHANGED)
if image is None:
print(
f"{Fore.YELLOW}[WARN] Frame drop. Corrupted JPEG.{Fore.RESET}"
)
print(f"{Fore.YELLOW}[WARN] Frame drop. Corrupted JPEG.{Fore.RESET}")
return
# Discard the serial buffer. This is due to the fact that it
# may build up some outdated frames. A bit of a workaround here tbh.
if conn.in_waiting >= 32768:
print(
f"{Fore.CYAN}[INFO] Discarding the serial buffer ({conn.in_waiting} bytes){Fore.RESET}"
)
print(f"{Fore.CYAN}[INFO] Discarding the serial buffer ({conn.in_waiting} bytes){Fore.RESET}")
conn.reset_input_buffer()
self.buffer = b""
# Calculate the fps.
@ -314,9 +308,7 @@ class Camera:
return
try:
rate = 115200 if sys.platform == "darwin" else 3000000 # Higher baud rate not working on macOS
conn = serial.Serial(
baudrate=rate, port=port, xonxoff=False, dsrdtr=False, rtscts=False
)
conn = serial.Serial(baudrate=rate, port=port, xonxoff=False, dsrdtr=False, rtscts=False)
# Set explicit buffer size for serial.
if sys.platform == "win32":
buffer_size = 32768

View File

@ -26,18 +26,18 @@ LICENSE: GNU GPLv3
import PySimpleGUI as sg
from config import EyeTrackConfig
from config import EyeTrackSettingsConfig
from collections import deque
from threading import Event, Thread
from eye import EyeId
from eye_processor import EyeProcessor, EyeInfoOrigin
from enum import Enum
from queue import Queue, Empty
from camera import Camera, CameraState
from osc import EyeId
import cv2
from osc.OSCMessage import OSCMessageType, OSCMessage
from utils.misc_utils import PlaySound, SND_FILENAME, SND_ASYNC, resource_path
import numpy as np
import time
class CameraWidget:
@ -69,8 +69,6 @@ class CameraWidget:
self.main_config = main_config
self.eye_id = widget_id
self.settings_config = main_config.settings
self.configl = main_config.left_eye
self.configr = main_config.right_eye
self.settings = main_config.settings
if self.eye_id == EyeId.RIGHT:
self.config = main_config.right_eye
@ -156,12 +154,12 @@ class CameraWidget:
button_color="#6f4ca1",
tooltip="Start eye calibration. Look all arround to all extreams without blinking until sound is heard.",
),
sg.Button(
"3D Calibration",
sg.Button(
"3D Calibration",
key=self.gui_restart_3d_calibration,
button_color="#6f4ca1",
tooltip="Start 3d eye calibration, must have steamvr open and eyes in hmd",
),
button_color="#6f4ca1",
tooltip="Start 3d eye calibration, must have steamvr open and eyes in hmd",
),
sg.Button(
"Stop Calibration",
key=self.gui_stop_calibration,
@ -276,9 +274,9 @@ class CameraWidget:
def start(self):
# If we're already running, bail
if not self.cancellation_event.is_set():
return
self.cancellation_event.clear()
self.ransac_thread = Thread(target=self.ransac.run)
self.ransac_thread.start()
@ -286,7 +284,6 @@ class CameraWidget:
self.camera_thread.start()
def stop(self):
# If we're not running yet, bail
if self.cancellation_event.is_set():
return
@ -296,6 +293,15 @@ class CameraWidget:
self.ransac_thread.join()
self.camera_thread.join()
def on_config_update(self, data):
keys = set(data.keys())
model_keys = set(self.config.model_fields.keys())
# we only want to restart our stuff, if our stuff got updated
# at the model level
if model_keys.intersection(keys):
self.stop()
self.start()
def render(self, window, event, values):
if self.image_queue.qsize() > 2:
with self.image_queue.mutex:
@ -305,25 +311,14 @@ class CameraWidget:
changed = False
# If anything has changed in our configuration settings, change/update those.
if event == self.gui_save_tracking_button and values[self.gui_camera_addr] != self.config.capture_source:
print("\033[94m[INFO] New value: {}\033[0m".format(values[self.gui_camera_addr]))
try:
# Try storing ints as ints, for those using wired cameras.
self.config.capture_source = int(values[self.gui_camera_addr])
except ValueError:
if values[self.gui_camera_addr] == "":
self.config.capture_source = None
else:
if (
len(values[self.gui_camera_addr]) > 5
and "http" not in values[self.gui_camera_addr]
and ".mp4" not in values[self.gui_camera_addr]
and not values[self.gui_camera_addr].startswith('/dev') # For MacOS and Linux users
): # If http is not in camera address, add it.
self.config.capture_source = f"http://{values[self.gui_camera_addr]}/"
else:
self.config.capture_source = values[self.gui_camera_addr]
changed = True
# it's a save *and* restart button, we should just forward the event and let the manager handle it
if event == self.gui_save_tracking_button:
new_camera_address = values[self.gui_camera_addr]
print("\033[94m[INFO] New value: {}\033[0m".format(new_camera_address))
# we don't want to save yet, we can notify the listeners though
changed = self.main_config.update_eye_model_config(
self.eye_id, {"capture_source": new_camera_address}, should_save=False
)
if self.config.rotation_angle != values[self.gui_rotation_slider]:
self.config.rotation_angle = int(values[self.gui_rotation_slider])
@ -372,6 +367,7 @@ class CameraWidget:
self.x1, self.y1 = values[self.gui_roi_selection]
# todo, this is now kinda duplicated with the OSC implementation
if event == self.gui_restart_3d_calibration:
self.ransac.calibration_3d_frame_counter = -621
self.settings.gui_3d_calibration = True
@ -444,7 +440,6 @@ class CameraWidget:
graph.erase()
graph.draw_image(data=imgbytes, location=(0, 0))
if None not in (self.x0, self.y0, self.x1, self.y1):
self.figure = graph.draw_rectangle((self.x0, self.y0), (self.x1, self.y1), line_color="#6f4ca1")
except Empty:
@ -469,7 +464,6 @@ class CameraWidget:
if eye_info.info_type != EyeInfoOrigin.FAILURE: # and not eye_info.blink:
graph.update(background_color="white")
if not np.isnan(eye_info.x) and not np.isnan(eye_info.y):
graph.draw_circle(
(eye_info.x * -100, eye_info.y * -100),
eye_info.pupil_dilation * 25,
@ -485,7 +479,6 @@ class CameraWidget:
)
if not np.isnan(eye_info.blink):
graph.draw_line(
(-100, abs(eye_info.blink) * 2 * 200),
(-100, 100),
@ -502,6 +495,26 @@ class CameraWidget:
graph.update(background_color="red")
# Relay information to OSC
if eye_info.info_type != EyeInfoOrigin.FAILURE:
self.osc_queue.put((self.eye_id, eye_info))
osc_message = OSCMessage(
type=OSCMessageType.EYE_INFO,
data=(self.eye_id, eye_info),
)
self.osc_queue.put(osc_message)
except Empty:
pass
def recenter_eyes(self, osc_message: OSCMessage):
if osc_message.data is not bool:
return # just incase we get anything other than bool
if osc_message.data:
self.settings.gui_recenter_eyes = True
def recalibrate_eyes(self, osc_message: OSCMessage):
if osc_message.data is not bool:
return # just incase we get anything other than bool
if osc_message.data:
self.ransac.ibo.clear_filter()
self.ransac.calibration_frame_counter = self.config.calibration_samples
PlaySound("Audio/start.wav", SND_FILENAME | SND_ASYNC)

View File

@ -27,11 +27,13 @@ LICENSE: GNU GPLv3
import json
import os.path
import shutil
from eye import EyeId
from colorama import Fore
from pydantic import BaseModel
from typing import Union, List
from typing import Any, Union, List
import os
from eye import EyeId
CONFIG_FILE_NAME: str = "eyetrack_settings.json"
BACKUP_CONFIG_FILE_NAME: str = "eyetrack_settings.backup"
@ -55,6 +57,68 @@ class EyeTrackCameraConfig(BaseModel):
calibration_points_3d: List[List[Union[float, None]]] = []
def update_capture_source(self, new_camera_address: str):
if not new_camera_address:
self.capture_source = None
return
if new_camera_address.isnumeric():
self.capture_source = int(new_camera_address)
return
# we were passed an IP, probably, lets add HTTP:// to it
if len(new_camera_address) > 5 and not (
not new_camera_address.startswith(("http", "/dev")) or not new_camera_address.endswith(".mp4")
):
self.capture_source = f"http://{new_camera_address}"
return
self.capture_source = new_camera_address
def update(self, data: dict[str, Any]) -> bool:
"""
Updates the model one field at a time based on the provided data dict.
The dict has to be defined like
```
data = {
"model_field": value
}
```
If stale data is provided,
ex. User clicked on save and restart but didn't provide a new field
we skip it, assuming that it was just a call to restart the tracking, or a miss-click.
Some fields may require more validation, we take care of that with special methods.
defining a method like
```
def update_custom_field(value: type):
pass
```
will cause it to be picked up by this method and called with the current value.
Return values are ignored.
"""
for key, value in data.items():
old_value = getattr(self, key, None)
# no reason to update if it's the same value
if old_value == value:
return False
if hasattr(self, f"update_{key}"):
update_attr = getattr(self, f"update_{key}")
if callable(update_attr):
update_attr(value)
else:
setattr(self, "key", value)
return True
else:
print(f"\033[93m[WARN] Field {key} does not exist on {self}.\033[0m")
return False
class EyeTrackSettingsConfig(BaseModel):
gui_flip_x_axis_left: bool = False
@ -127,6 +191,24 @@ class EyeTrackSettingsConfig(BaseModel):
gui_vrc_native: bool = True
gui_pupil_dilation: bool = True
gui_VRCFTModulePort: int = 8889
gui_VRCFTModuleIPAddress: str = "127.0.0.1"
gui_ShouldEmulateEyeWiden: bool = False
gui_ShouldEmulateEyeSquint: bool = False
gui_ShouldEmulateEyebrows: bool = False
gui_WidenThresholdV1_min: float = 0.60
gui_WidenThresholdV1_max: float = 1
gui_WidenThresholdV2_min: float = 0.60
gui_WidenThresholdV2_max: float = 1.05
gui_SqueezeThresholdV1_min: float = 0.07
gui_SqueezeThresholdV1_max: float = 0.5
gui_SqueezeThresholdV2_min: float = 0.07
gui_SqueezeThresholdV2_max: float = -1
gui_EyebrowThresholdRising: float = 0.8
gui_EyebrowThresholdLowering: float = 0.15
gui_OutputMultiplier: float = 1
gui_use_module: bool = False
class EyeTrackConfig(BaseModel):
version: int = 1
@ -159,10 +241,62 @@ class EyeTrackConfig(BaseModel):
load_config = EyeTrackConfig()
return load_config
def validate_camera_address_conflict(self, eye_id, capture_source):
match eye_id:
case EyeId.RIGHT:
if self.left_eye.capture_source == capture_source:
print(
f"{Fore.YELLOW}[WARN] Capture source {capture_source} already in use by the left camera.{Fore.RESET}"
)
return False
case EyeId.LEFT:
if self.right_eye.capture_source == capture_source:
print(
f"{Fore.YELLOW}[WARN] Capture source {capture_source} already in use by the right camera.{Fore.RESET}"
)
return False
case _:
return False
return True
def update_eye_model_config(self, eye_id: EyeId, data: dict, should_save=True, should_notify=True) -> bool:
"""
A more granular method for updating a particular model so that everything that relies on it
will get notified about any changes. Note, it acts a bit like pub-sub,
we don't care what changes got passed, we will notify the listeners with them.
It's the listeners job to check if they want that update.
"""
# The app really doesn't like address clashes, so we have to validate it as soon as possible
# otherwise we crash
if "capture_source" in data and not self.validate_camera_address_conflict(eye_id, data["capture_source"]):
return False
match eye_id:
case EyeId.RIGHT:
changed = self.right_eye.update(data)
case EyeId.LEFT:
changed = self.left_eye.update(data)
case _:
return False
if should_save:
self.save()
if should_notify:
self.__notify_listeners(data)
return changed
def update(self, data, save=False):
"""
More of an internal method for modules to be able to update the config
and have other parts of the system react to changes
"""
for field, value in data.items():
setattr(self.settings, field, value)
self.__notify_listeners()
self.__notify_listeners(data)
if save:
self.save()
@ -186,6 +320,6 @@ class EyeTrackConfig(BaseModel):
print(f"[DEBUG] Registering listener {callback}")
self.__listeners.append(callback)
def __notify_listeners(self):
def __notify_listeners(self, data: dict):
for listener in self.__listeners:
listener()
listener(data)

View File

@ -31,21 +31,11 @@ import numpy as np
import time
import os
import cv2
from enums import EyeLR
from eye import EyeId
from one_euro_filter import OneEuroFilter
from utils.img_utils import safe_crop
from enum import IntEnum
import os
os.environ["OMP_NUM_THREADS"] = "1"
class EyeId(IntEnum):
RIGHT = 0
LEFT = 1
BOTH = 2
SETTINGS = 3
# Note.
# OpenCV on Windows will generate an error if the file path contains non-ASCII characters when using cv2.imread(), cv2.imwrite(), etc.
# https://stackoverflow.com/questions/43185605/how-do-i-read-an-image-from-a-path-with-unicode-characters

View File

@ -28,12 +28,22 @@ from __future__ import annotations
import types
from collections import namedtuple
from typing import Any, ClassVar, Dict, List, Optional, TYPE_CHECKING, Tuple, Type, TypeVar, Iterator, Mapping
from typing import (
Any,
ClassVar,
Dict,
List,
TYPE_CHECKING,
Tuple,
Type,
TypeVar,
Iterator,
Mapping,
)
__all__ = (
'Enum',
# 'EyeId',
'EyeLR',
"Enum",
"EyeLR",
)
if TYPE_CHECKING:
@ -43,9 +53,9 @@ if TYPE_CHECKING:
def _create_value_cls(name: str, comparable: bool):
# All the type ignores here are due to the type checker being unable to recognise
# Runtime type creation without exploding.
cls = namedtuple('_EnumValue_' + name, 'name value')
cls.__repr__ = lambda self: f'<{name}.{self.name}: {self.value!r}>' # type: ignore
cls.__str__ = lambda self: f'{name}.{self.name}' # type: ignore
cls = namedtuple("_EnumValue_" + name, "name value")
cls.__repr__ = lambda self: f"<{name}.{self.name}: {self.value!r}>" # type: ignore
cls.__str__ = lambda self: f"{name}.{self.name}" # type: ignore
if comparable:
cls.__le__ = lambda self, other: isinstance(other, self.__class__) and self.value <= other.value # type: ignore
cls.__ge__ = lambda self, other: isinstance(other, self.__class__) and self.value >= other.value # type: ignore
@ -55,7 +65,9 @@ def _create_value_cls(name: str, comparable: bool):
def _is_descriptor(obj):
return hasattr(obj, '__get__') or hasattr(obj, '__set__') or hasattr(obj, '__delete__')
return (
hasattr(obj, "__get__") or hasattr(obj, "__set__") or hasattr(obj, "__delete__")
)
class EnumMeta(type):
@ -65,7 +77,14 @@ class EnumMeta(type):
_enum_member_map_: ClassVar[Dict[str, Any]]
_enum_value_map_: ClassVar[Dict[Any, Any]]
def __new__(cls, name: str, bases: Tuple[type, ...], attrs: Dict[str, Any], *, comparable: bool = False) -> Self:
def __new__(
cls,
name: str,
bases: Tuple[type, ...],
attrs: Dict[str, Any],
*,
comparable: bool = False,
):
value_mapping = {}
member_mapping = {}
member_names = []
@ -73,7 +92,7 @@ class EnumMeta(type):
value_cls = _create_value_cls(name, comparable)
for key, value in list(attrs.items()):
is_descriptor = _is_descriptor(value)
if key[0] == '_' and not is_descriptor:
if key[0] == "_" and not is_descriptor:
continue
# Special case classmethod to just pass through
@ -95,10 +114,10 @@ class EnumMeta(type):
member_mapping[key] = new_value
attrs[key] = new_value
attrs['_enum_value_map_'] = value_mapping
attrs['_enum_member_map_'] = member_mapping
attrs['_enum_member_names_'] = member_names
attrs['_enum_value_cls_'] = value_cls
attrs["_enum_value_map_"] = value_mapping
attrs["_enum_member_map_"] = member_mapping
attrs["_enum_member_names_"] = member_names
attrs["_enum_value_cls_"] = value_cls
actual_cls = super().__new__(cls, name, bases, attrs)
value_cls._actual_enum_cls_ = actual_cls # type: ignore # Runtime attribute isn't understood
return actual_cls
@ -107,13 +126,15 @@ class EnumMeta(type):
return (cls._enum_member_map_[name] for name in cls._enum_member_names_)
def __reversed__(cls) -> Iterator[Any]:
return (cls._enum_member_map_[name] for name in reversed(cls._enum_member_names_))
return (
cls._enum_member_map_[name] for name in reversed(cls._enum_member_names_)
)
def __len__(cls) -> int:
return len(cls._enum_member_names_)
def __repr__(cls) -> str:
return f'<enum {cls.__name__}>'
return f"<enum {cls.__name__}>"
@property
def __members__(cls) -> Mapping[str, Any]:
@ -129,10 +150,10 @@ class EnumMeta(type):
return cls._enum_member_map_[key]
def __setattr__(cls, name: str, value: Any) -> None:
raise TypeError('Enums are immutable.')
raise TypeError("Enums are immutable.")
def __delattr__(cls, attr: str) -> None:
raise TypeError('Enums are immutable')
raise TypeError("Enums are immutable")
def __instancecheck__(self, instance: Any) -> bool:
# isinstance(x, Y)
@ -144,8 +165,9 @@ class EnumMeta(type):
if TYPE_CHECKING:
from enum import Enum
from enum import Enum, IntEnum
else:
class Enum(metaclass=EnumMeta):
@classmethod
def try_value(cls, value):
@ -154,11 +176,13 @@ else:
except (KeyError, TypeError):
return value
E = TypeVar('E', bound='Enum')
E = TypeVar("E", bound="Enum")
def create_unknown_value(cls: Type[E], val: Any) -> E:
value_cls = cls._enum_value_cls_ # type: ignore # This is narrowed below
name = f'unknown_{val}'
name = f"unknown_{val}"
return value_cls(name=name, value=val)
@ -171,26 +195,12 @@ def try_enum(cls: Type[E], val: Any) -> E:
return cls._enum_value_map_[val] # type: ignore # All errors are caught below
except (KeyError, TypeError, AttributeError):
return create_unknown_value(cls, val)
# The line above is based on the code in the following url
# https://github.com/Rapptz/discord.py/blob/f7e97954950ffb0e34238d70813454caa6f1a3ae/discord/enums.py
# class EyeId(Enum):
# # https://docs.python.org/3.9/library/enum.html#functional-api
# # > The reason for defaulting to 1 as the starting number and not 0 is that 0 is False in a boolean sense, but enum members all evaluate to True.
# RIGHT = 1
# LEFT = 2
# BOTH = 3
# SETTINGS = 4
#
# def __str__(self) -> str:
# return self.name
#
# def __int__(self) -> int:
# return self.value
class EyeLR(Enum):
LEFT = 1
RIGHT = 2
@ -199,4 +209,4 @@ class EyeLR(Enum):
return self.name
def __int__(self) -> int:
return self.value
return self.value

View File

@ -34,6 +34,7 @@ class EyeId(IntEnum):
BOTH = 2
SETTINGS = 3
ALGOSETTINGS = 4
VRCFTMODULESETTINGS = 5
class EyeInfoOrigin(Enum):

View File

@ -28,8 +28,6 @@ LICENSE: GNU GPLv3
------------------------------------------------------------------------------------------------------
"""
from operator import truth
from dataclasses import dataclass
import sys
import asyncio
import os
@ -41,14 +39,7 @@ from config import EyeTrackSettingsConfig
from pye3d.camera import CameraModel
from pye3d.detector_3d import Detector3D, DetectorMode
import queue
import threading
import numpy as np
import cv2
from enum import Enum
from one_euro_filter import OneEuroFilter
from utils.misc_utils import PlaySound, SND_FILENAME, SND_ASYNC, resource_path
import importlib
from osc import EyeId
from eye import EyeId
from osc_calibrate_filter import *
from daddy import External_Run_DADDY
from leap import External_Run_LEAP
@ -63,7 +54,6 @@ from ellipse_based_pupil_dilation import *
from AHSF import *
def run_once(f):
def wrapper(*args, **kwargs):
if not wrapper.has_run:
@ -184,9 +174,7 @@ class EyeProcessor:
min_cutoff = 0.0004
beta = 0.9
noisy_point = np.array([1, 1])
self.one_euro_filter = OneEuroFilter(
noisy_point, min_cutoff=min_cutoff, beta=beta
)
self.one_euro_filter = OneEuroFilter(noisy_point, min_cutoff=min_cutoff, beta=beta)
def output_images_and_update(self, threshold_image, output_information: EyeInfo):
try:
@ -201,11 +189,7 @@ class EyeProcessor:
self.previous_image = self.current_image
self.previous_rotation = self.config.rotation_angle
except: # If this fails it likely means that the images are not the same size for some reason.
print(
"\033[91m[ERROR] Size of frames to display are of unequal sizes.\033[0m"
)
# pass
print("\033[91m[ERROR] Size of frames to display are of unequal sizes.\033[0m")
def capture_crop_rotate_image(self):
# Get our current frame
@ -213,12 +197,8 @@ class EyeProcessor:
try:
# Get frame from capture source, crop to ROI
self.current_image = self.current_image[
int(self.config.roi_window_y) : int(
self.config.roi_window_y + self.config.roi_window_h
),
int(self.config.roi_window_x) : int(
self.config.roi_window_x + self.config.roi_window_w
),
int(self.config.roi_window_y) : int(self.config.roi_window_y + self.config.roi_window_h),
int(self.config.roi_window_x) : int(self.config.roi_window_x + self.config.roi_window_w),
]
self.ibo.change_roi(self.config.dict(include=self.roi_include_set))
@ -287,9 +267,8 @@ class EyeProcessor:
self.settings.ibo_filter_samples,
self.settings.ibo_average_output_samples,
)
if self.eyeopen < float(
self.settings.ibo_fully_close_eye_threshold
): # threshold so the eye fully closes
# threshold so the eye fully closes
if self.eyeopen < float(self.settings.ibo_fully_close_eye_threshold):
self.eyeopen = 0.0
if self.bd_blink == True:
@ -314,9 +293,7 @@ class EyeProcessor:
) = self.er_leap.run(self.current_image_gray, self.current_image_gray_clean)
# print(self.eyeopen)
if (
len(self.prev_y_list) >= 100
): # "lock" eye when close/blink IN TESTING, kinda broke
if len(self.prev_y_list) >= 100: # "lock" eye when close/blink IN TESTING, kinda broke
self.prev_y_list.pop(0)
self.prev_y_list.append(self.out_y)
else:
@ -369,18 +346,12 @@ class EyeProcessor:
def LEAPM(self):
self.thresh = self.current_image_gray.copy()
(
self.current_image_gray,
self.rawx,
self.rawy,
self.eyeopen,
) = self.er_leap.run(
(self.current_image_gray, self.rawx, self.rawy, self.eyeopen,) = self.er_leap.run(
self.current_image_gray, self.current_image_gray_clean
) # TODO: make own self var and LEAP toggle
self.thresh = self.current_image_gray.copy()
self.out_x, self.out_y, self.avg_velocity = cal.cal_osc(
self, self.rawx, self.rawy, self.angle
)
# todo: lorow, fix this as well
self.out_x, self.out_y, self.avg_velocity = cal.cal_osc(self, self.rawx, self.rawy, self.angle)
self.current_algorithm = EyeInfoOrigin.LEAP
# print(self.eyeopen)
@ -391,9 +362,7 @@ class EyeProcessor:
self.thresh = self.current_image_gray.copy()
self.rawx, self.rawy, self.radius = self.er_daddy.run(self.current_image_gray)
# Daddy also uses a one euro filter, so I'll have to use it twice, but I'm not going to think too much about it.
self.out_x, self.out_y, self.avg_velocity = cal.cal_osc(
self, self.rawx, self.rawy, self.angle
)
self.out_x, self.out_y, self.avg_velocity = cal.cal_osc(self, self.rawx, self.rawy, self.angle)
self.current_algorithm = EyeInfoOrigin.DADDY
def AHSFRACM(self):
@ -438,9 +407,7 @@ class EyeProcessor:
# if self.prev_x is None:
# self.prev_x = self.rawx
# self.prev_y = self.rawy
self.out_x, self.out_y, self.avg_velocity = cal.cal_osc(
self, self.rawx, self.rawy, self.angle
)
self.out_x, self.out_y, self.avg_velocity = cal.cal_osc(self, self.rawx, self.rawy, self.angle)
self.current_algorithm = EyeInfoOrigin.HSRAC
def HSRACM(self):
@ -458,9 +425,7 @@ class EyeProcessor:
pass
self.hasrac_en = True
self.rawx, self.rawy, self.thresh, self.radius = self.er_hsf.run(
self.current_image_gray
)
self.rawx, self.rawy, self.thresh, self.radius = self.er_hsf.run(self.current_image_gray)
(
self.rawx,
self.rawy,
@ -478,9 +443,7 @@ class EyeProcessor:
# if self.prev_x is None:
# self.prev_x = self.rawx
# self.prev_y = self.rawy
self.out_x, self.out_y, self.avg_velocity = cal.cal_osc(
self, self.rawx, self.rawy, self.angle
)
self.out_x, self.out_y, self.avg_velocity = cal.cal_osc(self, self.rawx, self.rawy, self.angle)
self.current_algorithm = EyeInfoOrigin.HSRAC
def HSFM(self):
@ -497,12 +460,8 @@ class EyeProcessor:
else:
pass
# todo: add process to initialise er_hsf when resolution changes
self.rawx, self.rawy, self.thresh, self.radius = self.er_hsf.run(
self.current_image_gray
)
self.out_x, self.out_y, self.avg_velocity = cal.cal_osc(
self, self.rawx, self.rawy, self.angle
)
self.rawx, self.rawy, self.thresh, self.radius = self.er_hsf.run(self.current_image_gray)
self.out_x, self.out_y, self.avg_velocity = cal.cal_osc(self, self.rawx, self.rawy, self.angle)
self.current_algorithm = EyeInfoOrigin.HSF
def RANSAC3DM(self):
@ -519,9 +478,7 @@ class EyeProcessor:
else:
pass
self.hasrac_en = False
current_image_gray_copy = (
self.current_image_gray.copy()
) # Duplicate before overwriting in RANSAC3D.
current_image_gray_copy = self.current_image_gray.copy() # Duplicate before overwriting in RANSAC3D.
(
self.rawx,
self.rawy,
@ -533,9 +490,7 @@ class EyeProcessor:
) = RANSAC3D(self, True)
if self.settings.gui_RANSACBLINK:
self.eyeopen = ranblink
self.out_x, self.out_y, self.avg_velocity = cal.cal_osc(
self, self.rawx, self.rawy, self.angle
)
self.out_x, self.out_y, self.avg_velocity = cal.cal_osc(self, self.rawx, self.rawy, self.angle)
self.current_algorithm = EyeInfoOrigin.RANSAC
def AHSFM(self):
@ -559,9 +514,7 @@ class EyeProcessor:
self.radius,
) = External_Run_AHSF(self.current_image_gray)
self.thresh = self.current_image_gray
self.out_x, self.out_y, self.avg_velocity = cal.cal_osc(
self, self.rawx, self.rawy, self.angle
)
self.out_x, self.out_y, self.avg_velocity = cal.cal_osc(self, self.rawx, self.rawy, self.angle)
self.current_algorithm = EyeInfoOrigin.HSF
def BLOBM(self):
@ -668,9 +621,7 @@ class EyeProcessor:
if self.settings.gui_HSRAC:
if self.er_hsf is None:
if self.eye_id in [EyeId.LEFT]:
self.er_hsf = External_Run_HSF(
self.settings.gui_skip_autoradius,
self.settings.gui_HSF_radius_left,
@ -727,7 +678,7 @@ class EyeProcessor:
f = True
while True:
# f = True
#print(self.capture_queue_incoming.qsize())
# print(self.capture_queue_incoming.qsize())
# Check to make sure we haven't been requested to close
if self.cancellation_event.is_set():
print("\033[94m[INFO] Exiting Tracking thread\033[0m")
@ -753,9 +704,7 @@ class EyeProcessor:
focal_length=self.config.focal_length,
resolution=(self.config.roi_window_w, self.config.roi_window_h),
)
self.detector_3d = Detector3D(
camera=self.camera_model, long_term_mode=DetectorMode.blocking
)
self.detector_3d = Detector3D(camera=self.camera_model, long_term_mode=DetectorMode.blocking)
try:
if self.capture_queue_incoming.empty():
@ -767,20 +716,17 @@ class EyeProcessor:
self.current_fps,
) = self.capture_queue_incoming.get(block=True, timeout=0.1)
except queue.Empty:
#print("No image available")
# print("No image available")
continue
if not self.capture_crop_rotate_image():
continue
self.current_image_gray = cv2.cvtColor(
self.current_image, cv2.COLOR_BGR2GRAY
)
self.current_image_gray = cv2.cvtColor(self.current_image, cv2.COLOR_BGR2GRAY)
self.current_image_gray_clean = (
self.current_image_gray.copy()
) # copy this frame to have a clean image for blink algo
if self.cancellation_event.is_set():
print("\033[94m[INFO] Exiting Tracking thread\033[0m")
return

View File

@ -32,9 +32,11 @@ import threading
from camera_widget import CameraWidget
from config import EyeTrackConfig
from eye import EyeId
from osc import VRChatOSCReceiver, VRChatOSC
from general_settings_widget import SettingsWidget
from algo_settings_widget import AlgoSettingsWidget
from settings.VRCFTModuleSettings import VRCFTSettingsWidget
from settings.general_settings_widget import SettingsWidget
from settings.algo_settings_widget import AlgoSettingsWidget
from osc.osc import OSCManager
from osc.OSCMessage import OSCMessage
from utils.misc_utils import is_nt, resource_path
if is_nt:
@ -49,11 +51,13 @@ RIGHT_EYE_NAME = "-RIGHTEYEWIDGET-"
LEFT_EYE_NAME = "-LEFTEYEWIDGET-"
SETTINGS_NAME = "-SETTINGSWIDGET-"
ALGO_SETTINGS_NAME = "-ALGOSETTINGSWIDGET-"
VRCFT_MODULE_SETTINGS_NAME = "-VRCFTSETTINGSWIDGET-"
LEFT_EYE_RADIO_NAME = "-LEFTEYERADIO-"
RIGHT_EYE_RADIO_NAME = "-RIGHTEYERADIO-"
BOTH_EYE_RADIO_NAME = "-BOTHEYERADIO-"
SETTINGS_RADIO_NAME = "-SETTINGSRADIO-"
ALGO_SETTINGS_RADIO_NAME = "-ALGOSETTINGSRADIO-"
VRCFT_MODULE_SETTINGS_RADIO_NAME = "-VRCFTSETTINGSRADIO-"
page_url = "https://github.com/RedHawk989/EyeTrackVR/releases/latest"
appversion = "EyeTrackApp 0.2.0 BETA 11"
@ -65,15 +69,12 @@ def main():
config.save()
cancellation_event = threading.Event()
ROSC = False
# Check to see if we can connect to our video source first. If not, bring up camera finding
# dialog.
try:
if config.settings.gui_update_check:
response = requests.get(
"https://api.github.com/repos/EyeTrackVR/EyeTrackVR/releases/latest"
)
response = requests.get("https://api.github.com/repos/EyeTrackVR/EyeTrackVR/releases/latest")
latestversion = response.json()["name"]
if (
appversion == latestversion
@ -104,14 +105,7 @@ def main():
except:
print("\033[91m[INFO] Could not check for updates. Please try again later.\033[0m")
# Check to see if we have an ROI. If not, bring up ROI finder GUI.
# Spawn worker threads
osc_queue: queue.Queue[tuple[bool, int, int]] = queue.Queue()
osc = VRChatOSC(cancellation_event, osc_queue, config)
osc_thread = threading.Thread(target=osc.run)
# start worker threads
osc_thread.start()
osc_queue: queue.Queue[OSCMessage] = queue.Queue()
eyes = [
CameraWidget(EyeId.RIGHT, config, osc_queue),
@ -121,8 +115,34 @@ def main():
settings = [
SettingsWidget(EyeId.SETTINGS, config),
AlgoSettingsWidget(EyeId.ALGOSETTINGS, config),
VRCFTSettingsWidget(EyeId.VRCFTMODULESETTINGS, config, osc_queue),
]
osc_manager = OSCManager(
osc_message_in_queue=osc_queue,
config=config,
)
config.register_listener_callback(osc_manager.update)
config.register_listener_callback(eyes[0].on_config_update)
config.register_listener_callback(eyes[1].on_config_update)
osc_manager.register_listeners(
config.settings.gui_osc_recenter_address,
[
eyes[0].recenter_eyes,
eyes[1].recenter_eyes,
],
)
osc_manager.register_listeners(
config.settings.gui_osc_recalibrate_address,
[
eyes[0].recalibrate_eyes,
eyes[1].recalibrate_eyes,
],
)
osc_manager.start()
layout = [
[
sg.Radio(
@ -160,6 +180,13 @@ def main():
default=(config.eye_display_id == EyeId.ALGOSETTINGS),
key=ALGO_SETTINGS_RADIO_NAME,
),
sg.Radio(
"VRCFT Module Settings",
"EYESELECTRADIO",
background_color="#292929",
default=(config.eye_display_id == EyeId.VRCFTMODULESETTINGS),
key=VRCFT_MODULE_SETTINGS_RADIO_NAME,
),
],
[
sg.Column(
@ -190,6 +217,13 @@ def main():
visible=(config.eye_display_id in [EyeId.ALGOSETTINGS]),
background_color="#424042",
),
sg.Column(
settings[2].get_layout(),
vertical_alignment="top",
key=VRCFT_MODULE_SETTINGS_NAME,
visible=(config.eye_display_id in [EyeId.VRCFTMODULESETTINGS]),
background_color="#424042",
),
],
]
@ -201,15 +235,10 @@ def main():
settings[0].start()
if config.eye_display_id in [EyeId.ALGOSETTINGS]:
settings[1].start()
# self.main_config.eye_display_id
if config.eye_display_id in [EyeId.VRCFTMODULESETTINGS]:
settings[2].start()
# the eye's needs to be running before it is passed to the OSC
if config.settings.gui_ROSC:
osc_receiver = VRChatOSCReceiver(cancellation_event, config, eyes)
osc_receiver_thread = threading.Thread(target=osc_receiver.run)
osc_receiver_thread.start()
ROSC = True
# Create the window
window = sg.Window(
f"{appversion}",
@ -228,15 +257,7 @@ def main():
for eye in eyes:
eye.stop()
cancellation_event.set()
# shut down worker threads
osc_thread.join()
# TODO: find a way to have this function run on join maybe??
# threading.Event() wont work because pythonosc spawns its own thread.
# only way i can see to get around this is an ugly while loop that only checks if a threading event is triggered
# and then call the pythonosc shutdown function
if ROSC:
osc_receiver.shutdown()
osc_receiver_thread.join()
osc_manager.shutdown()
print("\033[94m[INFO] Exiting EyeTrackApp\033[0m")
return
@ -245,9 +266,11 @@ def main():
eyes[1].stop()
settings[0].stop()
settings[1].stop()
settings[2].stop()
window[RIGHT_EYE_NAME].update(visible=True)
window[LEFT_EYE_NAME].update(visible=False)
window[SETTINGS_NAME].update(visible=False)
window[VRCFT_MODULE_SETTINGS_NAME].update(visible=False)
window[ALGO_SETTINGS_NAME].update(visible=False)
config.eye_display_id = EyeId.RIGHT
config.settings.tracker_single_eye = 2
@ -256,11 +279,13 @@ def main():
elif values[LEFT_EYE_RADIO_NAME] and config.eye_display_id != EyeId.LEFT:
settings[0].stop()
settings[1].stop()
settings[2].stop()
eyes[0].stop()
eyes[1].start()
window[RIGHT_EYE_NAME].update(visible=False)
window[LEFT_EYE_NAME].update(visible=True)
window[SETTINGS_NAME].update(visible=False)
window[VRCFT_MODULE_SETTINGS_NAME].update(visible=False)
window[ALGO_SETTINGS_NAME].update(visible=False)
config.eye_display_id = EyeId.LEFT
config.settings.tracker_single_eye = 1
@ -269,11 +294,13 @@ def main():
elif values[BOTH_EYE_RADIO_NAME] and config.eye_display_id != EyeId.BOTH:
settings[0].stop()
settings[1].stop()
settings[2].stop()
eyes[1].start()
eyes[0].start()
window[LEFT_EYE_NAME].update(visible=True)
window[RIGHT_EYE_NAME].update(visible=True)
window[SETTINGS_NAME].update(visible=False)
window[VRCFT_MODULE_SETTINGS_NAME].update(visible=False)
window[ALGO_SETTINGS_NAME].update(visible=False)
config.eye_display_id = EyeId.BOTH
config.settings.tracker_single_eye = 0
@ -284,28 +311,43 @@ def main():
eyes[1].stop()
settings[1].stop()
settings[0].start()
settings[2].stop()
window[RIGHT_EYE_NAME].update(visible=False)
window[LEFT_EYE_NAME].update(visible=False)
window[SETTINGS_NAME].update(visible=True)
window[VRCFT_MODULE_SETTINGS_NAME].update(visible=False)
window[ALGO_SETTINGS_NAME].update(visible=False)
config.eye_display_id = EyeId.SETTINGS
config.save()
elif (
values[ALGO_SETTINGS_RADIO_NAME]
and config.eye_display_id != EyeId.ALGOSETTINGS
):
elif values[ALGO_SETTINGS_RADIO_NAME] and config.eye_display_id != EyeId.ALGOSETTINGS:
eyes[0].stop()
eyes[1].stop()
settings[0].stop()
settings[1].start()
settings[2].stop()
window[RIGHT_EYE_NAME].update(visible=False)
window[LEFT_EYE_NAME].update(visible=False)
window[SETTINGS_NAME].update(visible=False)
window[VRCFT_MODULE_SETTINGS_NAME].update(visible=False)
window[ALGO_SETTINGS_NAME].update(visible=True)
config.eye_display_id = EyeId.ALGOSETTINGS
config.save()
elif values[VRCFT_MODULE_SETTINGS_RADIO_NAME] and config.eye_display_id != EyeId.VRCFTMODULESETTINGS:
eyes[0].stop()
eyes[1].stop()
settings[0].stop()
settings[1].stop()
settings[2].start()
window[RIGHT_EYE_NAME].update(visible=False)
window[LEFT_EYE_NAME].update(visible=False)
window[SETTINGS_NAME].update(visible=False)
window[VRCFT_MODULE_SETTINGS_NAME].update(visible=True)
window[ALGO_SETTINGS_NAME].update(visible=False)
config.eye_display_id = EyeId.VRCFTMODULESETTINGS
config.save()
else:
# Otherwise, render all
for eye in eyes:
@ -315,5 +357,6 @@ def main():
if setting.started():
setting.render(window, event, values)
if __name__ == "__main__":
main()
main()

View File

@ -32,9 +32,7 @@ from functools import lru_cache
import cv2
import numpy as np
from utils.misc_utils import clamp
from utils.img_utils import safe_crop
from enum import IntEnum
import psutil
import sys
import os
@ -49,16 +47,9 @@ else:
process.nice(psutil.BELOW_NORMAL_PRIORITY_CLASS) # Windows
process.nice()
class EyeId(IntEnum):
RIGHT = 0
LEFT = 1
BOTH = 2
SETTINGS = 3
# from line_profiler_pycharm import profile
video_path = "ezgif.com-gif-maker.avi"
imshow_enable = False
calc_print_enable = False
@ -151,10 +142,7 @@ class HaarSurroundFeature:
def get_kernel(self):
# Defined here, but not yet used?
# Create a kernel filled with the value of self.val_out
kernel = (
np.ones(shape=(2 * self.r_out - 1, 2 * self.r_out - 1), dtype=np.float64)
* self.val_out
)
kernel = np.ones(shape=(2 * self.r_out - 1, 2 * self.r_out - 1), dtype=np.float64) * self.val_out
# Set the values of the inner area of the kernel using array slicing
start = self.r_out - self.r_in
@ -173,9 +161,7 @@ def to_gray(frame):
@lru_cache(maxsize=lru_maxsize_vvs)
def get_frameint_empty_array(frame_shape, pad, x_step, y_step, r_in, r_out):
frame_int_dtype = np.intc
frame_pad = np.empty(
(frame_shape[0] + (pad * 2), frame_shape[1] + (pad * 2)), dtype=np.uint8
)
frame_pad = np.empty((frame_shape[0] + (pad * 2), frame_shape[1] + (pad * 2)), dtype=np.uint8)
row, col = frame_pad.shape
@ -212,9 +198,7 @@ def get_frameint_empty_array(frame_shape, pad, x_step, y_step, r_in, r_out):
out_p01 = np.empty(len_syx, dtype=frame_int_dtype)
out_p10 = np.empty(len_syx, dtype=frame_int_dtype)
response_list = np.empty(len_syx, dtype=np.float64) # or np.int32
frame_conv = np.zeros(
shape=(row - 2 * pad, col - 2 * pad), dtype=np.uint8
) # or np.float64
frame_conv = np.zeros(shape=(row - 2 * pad, col - 2 * pad), dtype=np.uint8) # or np.float64
frame_conv_stride = frame_conv[::y_step, ::x_step]
return (
@ -361,21 +345,14 @@ class AutoRadiusCalc(object):
self.adj_comp_flag = False
return self.radius_cand_list[self.radius_middle_index]
else:
if (
self.left_index <= self.right_index
and self.left_index != self.radius_middle_index
):
if (self.left_item[1] + self.response_list[-1][1]) < (
self.right_item[1] + self.response_list[-1][1]
):
if self.left_index <= self.right_index and self.left_index != self.radius_middle_index:
if (self.left_item[1] + self.response_list[-1][1]) < (self.right_item[1] + self.response_list[-1][1]):
self.right_item = self.response_list[-1]
self.right_index = self.radius_middle_index - 1
self.radius_middle_index = (self.left_index + self.right_index) // 2
self.adj_comp_flag = False
return self.radius_cand_list[self.radius_middle_index]
if (self.left_item[1] + self.response_list[-1][1]) > (
self.right_item[1] + self.response_list[-1][1]
):
if (self.left_item[1] + self.response_list[-1][1]) > (self.right_item[1] + self.response_list[-1][1]):
self.left_item = self.response_list[-1]
self.left_index = self.radius_middle_index + 1
self.radius_middle_index = (self.left_index + self.right_index) // 2
@ -409,21 +386,11 @@ class AutoRadiusCalc(object):
self.adj_comp_flag = True
return default_radius
elif sort_res[0] == auto_radius_range[0]:
self.radius_cand_list = [
i
for i in range(
auto_radius_range[0], default_radius, auto_radius_step
)
][1:]
self.radius_cand_list = [i for i in range(auto_radius_range[0], default_radius, auto_radius_step)][1:]
self.adj_comp_flag = False
return self.radius_cand_list.pop()
else:
self.radius_cand_list = [
i
for i in range(
default_radius, auto_radius_range[1], auto_radius_step
)
][1:]
self.radius_cand_list = [i for i in range(default_radius, auto_radius_range[1], auto_radius_step)][1:]
self.adj_comp_flag = False
return self.radius_cand_list.pop()
else:
@ -497,9 +464,7 @@ class CenterCorrection(object):
self.frame_mask = None
self.frame_bin = None
self.frame_final = None
self.morph_kernel = cv2.getStructuringElement(
cv2.MORPH_RECT, (kernel_size, kernel_size)
)
self.morph_kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (kernel_size, kernel_size))
self.morph_kernel2 = np.ones((3, 3))
self.hist_index = np.arange(256)
self.hist = np.empty((256, 1))
@ -536,20 +501,14 @@ class CenterCorrection(object):
)
# bottleneck
self.frame_bin = cv2.threshold(gray_frame, frame_thr, 1, cv2.THRESH_BINARY_INV)[
1
]
self.frame_bin = cv2.threshold(gray_frame, frame_thr, 1, cv2.THRESH_BINARY_INV)[1]
cropped_x, cropped_y, cropped_w, cropped_h = cv2.boundingRect(self.frame_bin)
self.frame_final = cv2.bitwise_and(self.frame_bin, self.frame_mask)
# bottleneck
self.frame_final = cv2.morphologyEx(
self.frame_final, cv2.MORPH_CLOSE, self.morph_kernel
)
self.frame_final = cv2.morphologyEx(
self.frame_final, cv2.MORPH_OPEN, self.morph_kernel
)
self.frame_final = cv2.morphologyEx(self.frame_final, cv2.MORPH_CLOSE, self.morph_kernel)
self.frame_final = cv2.morphologyEx(self.frame_final, cv2.MORPH_OPEN, self.morph_kernel)
if (cropped_h, cropped_w) == self.frame_shape:
# Not detected.
@ -568,9 +527,7 @@ class CenterCorrection(object):
else:
base_x, base_y = center_x, center_y
contours, _ = cv2.findContours(
self.frame_final, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE
)
contours, _ = cv2.findContours(self.frame_final, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE)
contours_box = [cv2.boundingRect(cnt) for cnt in contours]
contours_dist = np.array(
[
@ -580,9 +537,7 @@ class CenterCorrection(object):
)
if len(contours_box):
cropped_x2, cropped_y2, cropped_w2, cropped_h2 = contours_box[
contours_dist.argmin()
]
cropped_x2, cropped_y2, cropped_w2, cropped_h2 = contours_box[contours_dist.argmin()]
x = cropped_x2 + cropped_w2 // 2
y = cropped_y2 + cropped_h2 // 2
else:
@ -688,9 +643,7 @@ class HSF_cls(object):
self.cvparam.radius = self.auto_radius_calc.get_radius()
if self.auto_radius_calc.adj_comp_flag:
self.now_modeo = (
self.cv_modeo[2] if not skip_blink_detect else self.cv_modeo[3]
)
self.now_modeo = self.cv_modeo[2] if not skip_blink_detect else self.cv_modeo[3]
radius, pad, step, hsf = self.cvparam.get_rpsh()
@ -723,13 +676,9 @@ class HSF_cls(object):
response_list,
frame_conv,
frame_conv_stride,
) = get_frameint_empty_array(
gray_frame.shape, pad, step[0], step[1], hsf.r_in, hsf.r_out
)
) = get_frameint_empty_array(gray_frame.shape, pad, step[0], step[1], hsf.r_in, hsf.r_out)
# BORDER_CONSTANT is faster than BORDER_REPLICATE There seems to be almost no negative impact when BORDER_CONSTANT is used.
cv2.copyMakeBorder(
gray_frame, pad, pad, pad, pad, cv2.BORDER_CONSTANT, dst=frame_pad
)
cv2.copyMakeBorder(gray_frame, pad, pad, pad, pad, cv2.BORDER_CONSTANT, dst=frame_pad)
cv2.integral(frame_pad, sum=frame_int, sdepth=cv2.CV_32S)
self.timedict["int_img"].append(timeit.default_timer() - int_start_time)
@ -785,18 +734,10 @@ class HSF_cls(object):
if self.blink_detector.response_len() < blink_init_frames:
self.blink_detector.add_response(cv2.mean(cropped_image)[0])
upper_x = center_x + max(
20, radius
) # self.center_correct.center_q1_radius
lower_x = center_x - max(
20, radius
) # self.center_correct.center_q1_radius
upper_y = center_y + max(
20, radius
) # self.center_correct.center_q1_radius
lower_y = center_y - max(
20, radius
) # self.center_correct.center_q1_radius
upper_x = center_x + max(20, radius) # self.center_correct.center_q1_radius
lower_x = center_x - max(20, radius) # self.center_correct.center_q1_radius
upper_y = center_y + max(20, radius) # self.center_correct.center_q1_radius
lower_y = center_y - max(20, radius) # self.center_correct.center_q1_radius
self.center_q1.add_response(
cv2.mean(
@ -831,19 +772,13 @@ class HSF_cls(object):
else:
# pass
if not self.center_correct.setup_comp:
self.center_correct.init_array(
gray_frame.shape, self.center_q1.quartile_1, radius
)
self.center_correct.init_array(gray_frame.shape, self.center_q1.quartile_1, radius)
elif self.center_correct.frame_shape != gray_frame.shape:
"""The resolution should have changed and the statistics should have changed, so essentially the statistics
need to be reworked, but implementation will be postponed as viability is the highest priority."""
self.center_correct.init_array(
gray_frame.shape, self.center_q1.quartile_1, radius
)
self.center_correct.init_array(gray_frame.shape, self.center_q1.quartile_1, radius)
center_x, center_y = self.center_correct.correction(
gray_frame, center_x, center_y
)
center_x, center_y = self.center_correct.correction(gray_frame, center_x, center_y)
# Define the center point and radius
center_xy = (center_x, center_y)
upper_x = center_x + radius
@ -851,9 +786,7 @@ class HSF_cls(object):
upper_y = center_y + radius
lower_y = center_y - radius
# Crop the image using the calculated bounds
cropped_image = safe_crop(
gray_frame, lower_x, lower_y, upper_x, upper_y
)
cropped_image = safe_crop(gray_frame, lower_x, lower_y, upper_x, upper_y)
# cropbox = [clamp(val, 0, gray_frame.shape[i]) for i, val in
# zip([1, 0, 1, 0], [lower_x, lower_y, upper_x, upper_y])] # debug code
@ -874,10 +807,7 @@ class HSF_cls(object):
# print('Pixel position:', center_xy)
if imshow_enable:
if (
self.now_modeo != self.cv_modeo[0]
and self.now_modeo != self.cv_modeo[1]
):
if self.now_modeo != self.cv_modeo[0] and self.now_modeo != self.cv_modeo[1]:
if 0 in cropped_image.shape:
# If shape contains 0, it is not detected well.
pass

View File

@ -30,10 +30,8 @@ import numpy as np
import time
import os
import cv2
from enums import EyeLR
from eye import EyeId
from one_euro_filter import OneEuroFilter
from utils.img_utils import safe_crop
from enum import IntEnum
import psutil
import sys
@ -48,13 +46,6 @@ else:
process.nice()
class EyeId(IntEnum):
RIGHT = 0
LEFT = 1
BOTH = 2
SETTINGS = 3
# higher intensity means more closed/ more white/less pupil
# Hm I need an acronym for this, any ideas?
@ -392,9 +383,6 @@ class IntensityBasedOpeness:
self.prev_val = eyeopen
try:
noisy_point = np.array(
[float(eyeopen), float(eyeopen)]
) # fliter our values with a One Euro Filter
point_hat = self.one_euro_filter(noisy_point)
eyeopenx = point_hat[0]
eyeopeny = point_hat[1]

View File

@ -1,428 +0,0 @@
"""
------------------------------------------------------------------------------------------------------
,@@@@@@
@@@@@@@@@@@ @@@
@@@@@@@@@@@@ @@@@@@@@@@@
@@@@@@@@@@@@@ @@@@@@@@@@@@@@
@@@@@@@/ ,@@@@@@@@@@@@@
/@@@@@@@@@@@@@@@ @@@@@@@@
@@@@@@@@@@@@@@@@@@@@@@@@ @@@@@
@@@@@@@@ @@@@@
,@@@ @@@@&
@@@@@@. @@@@
@@@ @@@@@@@@@/ @@@@@
,@@@. @@@@@@((@ @@@@(
//@@@ ,, @@@@ @@@@@
@@@( @@@@@@@
@@@ @ @@@@@@@@#
@@@@@@@@@@@@@@@@@
@@@@@@@@@@@@@(
Copyright (c) 2023 EyeTrackVR <3
LICENSE: GNU GPLv3
------------------------------------------------------------------------------------------------------
"""
from pythonosc import udp_client
from pythonosc import osc_server
from pythonosc import dispatcher
from config import EyeTrackConfig
from utils.misc_utils import PlaySound, SND_FILENAME, SND_ASYNC
from enum import IntEnum
import queue
import threading
import time
class EyeId(IntEnum):
RIGHT = 0
LEFT = 1
BOTH = 2
SETTINGS = 3
ALGOSETTINGS = 4
def eyelid_transformer(self, eye_blink):
if self.config.osc_invert_eye_close:
return float(1 - eye_blink)
else:
return float(eye_blink)
se = False
falloff = False
def output_osc(eye_x, eye_y, eye_blink, last_blink, pupil_dilation, avg_velocity, self):
global se, falloff
if self.config.gui_osc_vrcft_v1:
if self.main_config.eye_display_id in [
EyeId.RIGHT,
EyeId.LEFT,
]: # we are in single eye mode
se = True
self.client.send_message(self.config.osc_left_eye_x_address, eye_x)
self.client.send_message(self.config.osc_right_eye_x_address, eye_x)
self.client.send_message(self.config.osc_eyes_y_address, eye_y)
self.client.send_message(
self.config.osc_right_eye_close_address,
eyelid_transformer(self, eye_blink),
)
self.client.send_message(
self.config.osc_left_eye_close_address,
eyelid_transformer(self, eye_blink),
)
else:
se = False
if self.eye_id in [EyeId.LEFT] and not se: # left eye, send data to left
self.l_eye_x = eye_x
self.l_eye_blink = eye_blink
self.l_eye_velocity = avg_velocity
if self.l_eye_blink == 0.0:
if last_blink > 0.15: # when binary blink is on, blinks may be too fast for OSC so we repeat them.
for i in range(4):
self.client.send_message(
self.config.osc_left_eye_close_address,
eyelid_transformer(self, self.l_eye_blink),
)
last_blink = time.time() - last_blink
self.l_eye_x = self.r_eye_x
self.client.send_message(self.config.osc_left_eye_x_address, self.l_eye_x)
self.left_y = eye_y
self.client.send_message(
self.config.osc_left_eye_close_address,
eyelid_transformer(self, self.l_eye_blink),
)
elif self.eye_id in [EyeId.RIGHT] and not se: # Right eye, send data to right
self.r_eye_x = eye_x
self.r_eye_blink = eye_blink
self.l_eye_velocity = avg_velocity
if self.r_eye_blink == 0.0:
if last_blink > 0.15: # when binary blink is on, blinks may be too fast for OSC so we repeat them.
# print("REPEATING R BLINK")
for i in range(4):
self.client.send_message(
self.config.osc_right_eye_close_address,
eyelid_transformer(self, self.r_eye_blink),
)
last_blink = time.time() - last_blink
if self.config.gui_outer_side_falloff:
if self.l_eye_blink == 0.0: # if both eyes closed and DEF is enables, blink
self.client.send_message(
self.config.osc_left_eye_close_address,
eyelid_transformer(self, self.r_eye_blink),
)
self.client.send_message(
self.config.osc_right_eye_close_address,
eyelid_transformer(self, self.r_eye_blink),
)
self.r_eye_x = self.l_eye_x
self.client.send_message(self.config.osc_right_eye_x_address, eye_x)
self.right_y = eye_y
self.client.send_message(
self.config.osc_right_eye_close_address,
eyelid_transformer(self, self.r_eye_blink),
)
if self.main_config.eye_display_id in [EyeId.BOTH] and self.right_y != 621 and self.left_y != 621:
y = (self.right_y + self.left_y) / 2
self.client.send_message(self.config.osc_eyes_y_address, y)
if self.config.gui_osc_vrcft_v2:
if self.main_config.eye_display_id in [
EyeId.RIGHT,
EyeId.LEFT,
]: # we are in single eye mode
se = True
self.client.send_message("/avatar/parameters/v2/EyeX", eye_x)
self.client.send_message("/avatar/parameters/v2/EyeY", eye_y)
self.client.send_message(
"/avatar/parameters/v2/EyeLid",
eyelid_transformer(self, eye_blink),
)
else:
se = False
if self.eye_id in [EyeId.LEFT] and not se: # left eye, send data to left
self.l_eye_x = eye_x
self.l_eye_blink = eye_blink
self.r_eye_velocity = avg_velocity
if self.l_eye_blink == 0.0:
if last_blink > 0.15: # when binary blink is on, blinks may be too fast for OSC so we repeat them.
for i in range(4):
self.client.send_message(
"/avatar/parameters/v2/EyeLidLeft",
eyelid_transformer(self, self.l_eye_blink),
)
last_blink = time.time() - last_blink
if self.config.gui_outer_side_falloff:
if self.r_eye_blink == 0.0: # if both eyes closed and DEF is enables, blink
self.client.send_message(
"/avatar/parameters/v2/EyeLidLeft",
eyelid_transformer(self, self.l_eye_blink),
)
self.client.send_message(
"/avatar/parameters/v2/EyeLidRight",
eyelid_transformer(self, self.l_eye_blink),
)
self.l_eye_x = self.r_eye_x
self.client.send_message("/avatar/parameters/v2/EyeLeftX", self.l_eye_x)
self.left_y = eye_y
if self.left_y != 621:
self.client.send_message("/avatar/parameters/v2/EyeLeftY", self.left_y)
self.client.send_message(
"/avatar/parameters/v2/EyeLidLeft",
eyelid_transformer(self, self.l_eye_blink),
)
elif self.eye_id in [EyeId.RIGHT] and not se: # Right eye, send data to right
self.r_eye_x = eye_x
self.r_eye_blink = eye_blink
self.r_eye_velocity = avg_velocity
if self.r_eye_blink == 0.0:
if last_blink > 0.15: # when binary blink is on, blinks may be too fast for OSC so we repeat them.
for i in range(4):
self.client.send_message(
"/avatar/parameters/v2/EyeLidRight",
eyelid_transformer(self, self.r_eye_blink),
)
last_blink = time.time() - last_blink
if self.config.gui_outer_side_falloff:
if self.l_eye_blink == 0.0: # if both eyes closed and DEF is enables, blink
self.client.send_message(
"/avatar/parameters/v2/EyeLidLeft",
eyelid_transformer(self, self.r_eye_blink),
)
self.client.send_message(
"/avatar/parameters/v2/EyeLidRight",
eyelid_transformer(self, self.r_eye_blink),
)
self.r_eye_x = self.l_eye_x
self.client.send_message("/avatar/parameters/v2/EyeRightX", self.r_eye_x)
self.right_y = eye_y
if self.right_y != 621:
self.client.send_message("/avatar/parameters/v2/EyeRightY", self.right_y)
self.client.send_message(
"/avatar/parameters/v2/EyeLidRight",
eyelid_transformer(self, self.r_eye_blink),
)
if self.config.gui_vrc_native: # VRC NATIVE
if self.main_config.eye_display_id in [
EyeId.RIGHT,
EyeId.LEFT,
]: # we are in single eye mode
se = True
if eye_blink == 0.0:
if last_blink > 0.2: # when binary blink is on, blinks may be too fast for OSC so we repeat them.
for i in range(5):
self.client.send_message("/tracking/eye/EyesClosedAmount", float(1 - eye_blink))
eye_blink += 0.02 # TODO finish tuning value
last_blink = time.time() - last_blink
else:
self.client.send_message("/tracking/eye/EyesClosedAmount", float(1 - eye_blink))
self.client.send_message(
"/tracking/eye/LeftRightVec",
[float(eye_x), float(eye_y), 1.0, float(eye_x), float(eye_y), 1.0],
) # vrc native ET
else:
se = False
if self.eye_id in [EyeId.LEFT] and not se: # left eye, send data to left
self.l_eye_x = eye_x
self.l_eye_blink = eye_blink
self.left_y = eye_y
self.l_eye_velocity = avg_velocity
self.client.send_message(
self.config.osc_left_eye_close_address,
eyelid_transformer(self, eye_blink),
)
if self.l_eye_blink == 0.0:
if last_blink > 0.2: # when binary blink is on, blinks may be too fast for OSC so we repeat them.
for i in range(5):
self.client.send_message("/tracking/eye/EyesClosedAmount", float(1 - eye_blink))
last_blink = time.time() - last_blink
if self.config.gui_outer_side_falloff:
if self.r_eye_blink == 0.0: # if both eyes closed and DEF is enables, blink
self.client.send_message("/tracking/eye/EyesClosedAmount", float(1 - eye_blink))
self.l_eye_x = self.r_eye_x
elif self.eye_id in [EyeId.RIGHT] and not se: # Right eye, send data to right
self.r_eye_x = eye_x
self.r_eye_blink = eye_blink
self.right_y = eye_y
self.r_eye_velocity = avg_velocity
self.client.send_message(
self.config.osc_right_eye_close_address,
eyelid_transformer(self, eye_blink),
)
if self.r_eye_blink == 0.0:
if last_blink > 0.2: # when binary blink is on, blinks may be too fast for OSC so we repeat them.
for i in range(5):
self.client.send_message("/tracking/eye/EyesClosedAmount", float(1 - eye_blink))
last_blink = time.time() - last_blink
if self.config.gui_outer_side_falloff:
if self.l_eye_blink == 0.0: # if both eyes closed and DEF is enables, blink
self.client.send_message("/tracking/eye/EyesClosedAmount", float(0))
self.r_eye_x = self.l_eye_x
if self.main_config.eye_display_id in [EyeId.BOTH] and self.r_eye_blink != 621 and self.r_eye_blink != 621:
if self.r_eye_blink == 0.0 or self.l_eye_blink == 0.0:
if last_blink > 0.2: # when binary blink is on, blinks may be too fast for OSC so we repeat them.
for i in range(5):
self.client.send_message("/tracking/eye/EyesClosedAmount", float(1))
last_blink = time.time() - last_blink
eye_blink = (self.r_eye_blink + self.l_eye_blink) / 2
self.client.send_message("/tracking/eye/EyesClosedAmount", float(1 - eye_blink))
if self.main_config.eye_display_id in [EyeId.BOTH] and self.right_y != 621 and self.left_y != 621:
eye_y = (self.right_y + self.left_y) / 2
if not se:
# vrc native ET (z values may need tweaking, they act like a scalar)
self.client.send_message(
"/tracking/eye/LeftRightVec",
[
float(self.l_eye_x),
float(self.left_y),
1.0,
float(self.r_eye_x),
float(self.right_y),
1.0,
],
)
class VRChatOSC:
# Use a tuple of blink (true, blinking, false, not), x, y for now.
def __init__(
self,
cancellation_event: threading.Event,
msg_queue: queue.Queue[tuple[bool, int, int]],
main_config: EyeTrackConfig,
):
self.main_config = main_config
self.config = main_config.settings
self.client = udp_client.SimpleUDPClient(
self.config.gui_osc_address, int(self.config.gui_osc_port)
) # use OSC port and address that was set in the config
self.cancellation_event = cancellation_event
self.msg_queue = msg_queue
self.eye_id = EyeId.RIGHT
self.left_y = 621
self.right_y = 621
self.r_eye_x = 0
self.l_eye_x = 0
self.r_eye_blink = 0.7
self.l_eye_blink = 0.7
self.l_eye_velocity = 0
self.r_eye_velocity = 1
def run(self):
start = time.time()
last_blink = time.time()
while True:
if self.cancellation_event.is_set():
print("\033[94m[INFO] Exiting OSC Queue\033[0m")
return
try:
(self.eye_id, eye_info) = self.msg_queue.get(block=True, timeout=0.1)
except:
continue
output_osc(
eye_info.x,
eye_info.y,
eye_info.blink,
last_blink,
eye_info.pupil_dilation,
eye_info.avg_velocity,
self,
)
class VRChatOSCReceiver:
def __init__(self, cancellation_event: threading.Event, main_config: EyeTrackConfig, eyes: []):
self.config = main_config.settings
self.cancellation_event = cancellation_event
self.dispatcher = dispatcher.Dispatcher()
self.eyes = eyes # we cant import CameraWidget so any type it is
try:
self.server = osc_server.OSCUDPServer(
(self.config.gui_osc_address, int(self.config.gui_osc_receiver_port)),
self.dispatcher,
)
except:
print(f"\033[91m[ERROR] OSC Receive port: {self.config.gui_osc_receiver_port} occupied.\033[0m")
def shutdown(self):
print("\033[94m[INFO] Exiting OSC Receiver\033[0m")
try:
self.server.shutdown()
except:
pass
def recenter_eyes(self, address, osc_value):
if type(osc_value) != bool:
return # just incase we get anything other than bool
if osc_value:
for eye in self.eyes:
eye.settings.gui_recenter_eyes = True
def recalibrate_eyes(self, address, osc_value):
if type(osc_value) != bool:
return # just incase we get anything other than bool
if osc_value:
for eye in self.eyes:
eye.ransac.ibo.clear_filter()
eye.ransac.calibration_frame_counter = self.config.calibration_samples
PlaySound("Audio/start.wav", SND_FILENAME | SND_ASYNC)
def run(self):
# bind what function to run when specified OSC message is received
try:
self.dispatcher.map(self.config.gui_osc_recalibrate_address, self.recalibrate_eyes)
self.dispatcher.map(self.config.gui_osc_recenter_address, self.recenter_eyes)
# start the server
print("\033[92m[INFO] VRChatOSCReceiver serving on {}\033[0m".format(self.server.server_address))
self.server.serve_forever()
except:
print(f"\033[91m[ERROR] OSC Receive port: {self.config.gui_osc_receiver_port} occupied.\033[0m")

View File

@ -0,0 +1,13 @@
import dataclasses
from enum import IntEnum
class OSCMessageType(IntEnum):
EYE_INFO = 1
VRCFT_MODULE_INFO = 2
@dataclasses.dataclass
class OSCMessage:
type: OSCMessageType
data: any

View File

@ -0,0 +1,17 @@
from pythonosc.udp_client import SimpleUDPClient
from osc.OSCMessage import OSCMessage
class VRCFTModuleSender:
set_command_pattern = "/command/{}/{}/"
def send(self, osc_message: OSCMessage, client: SimpleUDPClient):
command = osc_message.data.get("command", None)
field_to_send = osc_message.data.get("field", None)
value_to_send = osc_message.data.get("value", None)
if not command or not all([field_to_send, value_to_send is not None]):
print("[ERROR] Misconfiguration in received OSC message for the VRCFT Module")
return
client.send_message(self.set_command_pattern.format(command, field_to_send), value_to_send)

View File

@ -0,0 +1,307 @@
from pythonosc.udp_client import SimpleUDPClient
from eye import EyeId
from osc.OSCMessage import OSCMessage
from config import EyeTrackConfig, EyeTrackSettingsConfig
from enum import IntEnum
import time
def _eyelid_transformer(config, eye_blink):
if config.osc_invert_eye_close:
return float(1 - eye_blink)
else:
return float(eye_blink)
class OutputType(IntEnum):
V1_PARAMS = 1
V2_PARAMS = 2
NATIVE_PARAMS = 3
class VRChatOSCSender:
def __init__(self):
self.is_single_eye = False
self.falloff_enabled = False
self.left_y = 621
self.right_y = 621
self.r_eye_x = 0
self.l_eye_x = 0
self.r_eye_blink = 0.7
self.l_eye_blink = 0.7
self.l_eye_velocity = 0
self.r_eye_velocity = 1
self.left_last_blink = time.time()
self.right_last_blink = time.time()
def output_osc_info(
self,
osc_message: OSCMessage,
client: SimpleUDPClient,
main_config: EyeTrackConfig,
config: EyeTrackSettingsConfig,
):
eye_id, eye_info = osc_message.data
self.is_single_eye = self.get_is_single_eye(main_config.eye_display_id)
output_method = None
if config.gui_vrc_native:
output_method = self.output_native
if config.gui_osc_vrcft_v1:
output_method = self.output_v1_params
if config.gui_osc_vrcft_v2:
output_method = self.output_v2_params
if output_method:
output_method(
main_config=main_config,
config=config,
client=client,
eye_x=eye_info.x,
eye_y=eye_info.y,
eye_blink=eye_info.blink,
avg_velocity=eye_info.avg_velocity,
eye_id=eye_id,
)
@staticmethod
def get_is_single_eye(eye_display_id):
return eye_display_id in [EyeId.RIGHT, EyeId.LEFT]
def update_eye_state(self, eye_id, eye_x, eye_y, eye_blink, avg_velocity):
if eye_id == EyeId.LEFT:
self.l_eye_x = eye_x
self.l_eye_blink = eye_blink
self.left_y = eye_y
self.l_eye_velocity = avg_velocity
if eye_id == EyeId.RIGHT:
self.r_eye_x = eye_x
self.r_eye_blink = eye_blink
self.right_y = eye_y
self.r_eye_velocity = avg_velocity
def output_native(self, main_config, config, client, eye_x, eye_y, eye_blink, avg_velocity, eye_id):
default_eye_blink_params = {
"eye_id": eye_id,
"client": client,
"config": config,
}
self.update_eye_state(
eye_id=eye_id,
eye_x=eye_x,
eye_y=eye_y,
eye_blink=eye_blink,
avg_velocity=avg_velocity,
)
if self.is_single_eye:
self.output_osc_native_blink(
**default_eye_blink_params,
)
client.send_message(
"/tracking/eye/LeftRightVec",
[float(eye_x), float(eye_y), 1.0, float(eye_x), float(eye_y), 1.0],
)
if eye_id in [EyeId.LEFT, EyeId.RIGHT] and not self.is_single_eye:
self.output_osc_native_blink(**default_eye_blink_params, single_eye_mode=False)
if main_config.eye_display_id == EyeId.BOTH and self.r_eye_blink != 621 and self.r_eye_blink != 621:
self.output_osc_native_blink(**default_eye_blink_params)
if not self.is_single_eye:
# vrc native ET (z values may need tweaking, they act like a scalar)
client.send_message(
"/tracking/eye/LeftRightVec",
[
float(self.l_eye_x),
float(self.left_y),
1.0,
float(self.r_eye_x),
float(self.right_y),
1.0,
],
)
def output_v1_params(self, main_config, config, client, eye_x, eye_y, eye_blink, avg_velocity, eye_id):
default_eye_blink_params = {
"eye_id": eye_id,
"client": client,
"config": config,
"left_eye_blink_address": config.osc_left_eye_close_address,
"right_eye_blink_address": config.osc_right_eye_close_address,
}
self.update_eye_state(
eye_id=eye_id,
eye_x=eye_x,
eye_y=eye_y,
eye_blink=eye_blink,
avg_velocity=avg_velocity,
)
if self.is_single_eye:
client.send_message(config.osc_left_eye_x_address, eye_x)
client.send_message(config.osc_right_eye_x_address, eye_x)
client.send_message(config.osc_eyes_y_address, eye_y)
self.output_vrcft_blink_data(**default_eye_blink_params)
if eye_id in [EyeId.LEFT, EyeId.RIGHT] and not self.is_single_eye:
self.output_vrcft_blink_data(**default_eye_blink_params, single_eye_mode=False)
if eye_id == EyeId.LEFT:
client.send_message(config.osc_left_eye_x_address, self.l_eye_x)
self.left_y = eye_y
client.send_message(
config.osc_left_eye_close_address,
_eyelid_transformer(config, self.l_eye_blink),
)
if eye_id == EyeId.RIGHT:
client.send_message(config.osc_right_eye_x_address, self.r_eye_x)
self.right_y = eye_y
client.send_message(
config.osc_right_eye_close_address,
_eyelid_transformer(config, self.r_eye_blink),
)
if main_config.eye_display_id == EyeId.BOTH and self.right_y != 621 and self.left_y != 621:
y = (self.right_y + self.left_y) / 2
client.send_message(config.osc_eyes_y_address, y)
def output_v2_params(self, main_config, config, client, eye_x, eye_y, eye_blink, avg_velocity, eye_id):
default_eye_blink_params = {
"eye_id": eye_id,
"client": client,
"config": config,
}
self.update_eye_state(
eye_id=eye_id,
eye_x=eye_x,
eye_y=eye_y,
eye_blink=eye_blink,
avg_velocity=avg_velocity,
)
if self.is_single_eye:
client.send_message("/avatar/parameters/v2/EyeX", eye_x)
client.send_message("/avatar/parameters/v2/EyeY", eye_y)
self.output_vrcft_blink_data(
**default_eye_blink_params,
left_eye_blink_address="/avatar/parameters/v2/EyeLid",
right_eye_blink_address="/avatar/parameters/v2/EyeLid",
)
if eye_id in [EyeId.LEFT, EyeId.RIGHT] and not self.is_single_eye:
self.output_vrcft_blink_data(
**default_eye_blink_params,
left_eye_blink_address="/avatar/parameters/v2/EyeLidLeft",
right_eye_blink_address="/avatar/parameters/v2/EyeLidRight",
single_eye_mode=False,
)
if eye_id == EyeId.LEFT:
client.send_message("/avatar/parameters/v2/EyeLeftX", self.l_eye_x)
if self.left_y != 621:
client.send_message("/avatar/parameters/v2/EyeLeftY", eye_y)
client.send_message(
"/avatar/parameters/v2/EyeLidLeft",
_eyelid_transformer(config, self.l_eye_blink),
)
if eye_id == EyeId.RIGHT:
client.send_message("/avatar/parameters/v2/EyeRightX", self.r_eye_x)
if eye_y != 621:
client.send_message("/avatar/parameters/v2/EyeRightY", eye_y)
client.send_message(
"/avatar/parameters/v2/EyeLidRight",
_eyelid_transformer(config, self.r_eye_blink),
)
def output_vrcft_blink_data(
self,
eye_id: EyeId,
client: SimpleUDPClient,
config,
left_eye_blink_address,
right_eye_blink_address,
single_eye_mode=True,
):
active_eye_blink = self.r_eye_blink if eye_id == EyeId.RIGHT else self.l_eye_blink
falloff_blink = self.r_eye_blink if eye_id == EyeId.LEFT else self.l_eye_blink
blink_address = right_eye_blink_address if eye_id == EyeId.RIGHT else left_eye_blink_address
side_name = "left" if eye_id == EyeId.RIGHT else "right"
last_side_blink = getattr(self, f"{side_name}_last_blink")
if single_eye_mode:
# in case of v1 params, we have to send the same data do each eye separately.
# so in case of v2 params, we will be generating one unnecessary call more
client.send_message(left_eye_blink_address, _eyelid_transformer(config, active_eye_blink))
client.send_message(right_eye_blink_address, _eyelid_transformer(config, active_eye_blink))
elif eye_id in [EyeId.RIGHT, EyeId.LEFT] and not single_eye_mode:
if active_eye_blink == 0.0:
if last_side_blink > 0.20:
for _ in range(5):
client.send_message(blink_address, _eyelid_transformer(config, active_eye_blink))
setattr(self, f"{side_name}_last_blink", time.time() - last_side_blink)
if config.gui_outer_side_falloff:
if falloff_blink == 0.0:
client.send_message(left_eye_blink_address, _eyelid_transformer(config, self.l_eye_blink))
client.send_message(right_eye_blink_address, _eyelid_transformer(config, self.r_eye_blink))
client.send_message(blink_address, _eyelid_transformer(config, active_eye_blink))
def output_osc_native_blink(
self,
eye_id: EyeId,
client,
config,
single_eye_mode=True,
):
blink_address = "/tracking/eye/EyesClosedAmount"
active_eye_blink = self.r_eye_blink if eye_id == EyeId.RIGHT else self.l_eye_blink
falloff_blink = self.r_eye_blink if eye_id == EyeId.LEFT else self.l_eye_blink
side_name = "left" if eye_id == EyeId.RIGHT else "right"
last_side_blink = getattr(self, f"{side_name}_last_blink")
def send_native_binary_blink(address: str, blink_value):
if last_side_blink > 0.2:
for _ in range(5):
client.send_message(address, float(1 - blink_value))
setattr(self, f"{side_name}_last_blink", time.time() - last_side_blink)
if single_eye_mode:
if active_eye_blink == 0.0:
send_native_binary_blink(blink_address, active_eye_blink)
else:
client.send_message(blink_address, float(1 - active_eye_blink))
if eye_id in [EyeId.RIGHT, EyeId.LEFT] and not single_eye_mode:
client.send_message(
blink_address,
_eyelid_transformer(config, 1 - active_eye_blink),
)
if active_eye_blink == 0.0:
send_native_binary_blink(blink_address, active_eye_blink)
if config.gui_outer_side_falloff:
if falloff_blink == 0.0:
client.send_message(blink_address, float(1 - active_eye_blink))
if eye_id == EyeId.BOTH and self.r_eye_blink != 621 and self.r_eye_blink != 621:
if self.r_eye_blink == 0.0 or self.l_eye_blink == 0.0:
send_native_binary_blink(blink_address, active_eye_blink)
# this has a nasty habit of permanent-squint FIXME
averaged_eye_blink = (self.r_eye_blink + self.l_eye_blink) / 2
client.send_message(blink_address, float(1 - averaged_eye_blink))

View File

214
EyeTrackApp/osc/osc.py Normal file
View File

@ -0,0 +1,214 @@
"""
------------------------------------------------------------------------------------------------------
,@@@@@@
@@@@@@@@@@@ @@@
@@@@@@@@@@@@ @@@@@@@@@@@
@@@@@@@@@@@@@ @@@@@@@@@@@@@@
@@@@@@@/ ,@@@@@@@@@@@@@
/@@@@@@@@@@@@@@@ @@@@@@@@
@@@@@@@@@@@@@@@@@@@@@@@@ @@@@@
@@@@@@@@ @@@@@
,@@@ @@@@&
@@@@@@. @@@@
@@@ @@@@@@@@@/ @@@@@
,@@@. @@@@@@((@ @@@@(
//@@@ ,, @@@@ @@@@@
@@@( @@@@@@@
@@@ @ @@@@@@@@#
@@@@@@@@@@@@@@@@@
@@@@@@@@@@@@@(
Copyright (c) 2023 EyeTrackVR <3
LICENSE: GNU GPLv3
------------------------------------------------------------------------------------------------------
"""
from time import sleep
from typing import Dict, Optional, Iterable, Callable
from pythonosc import udp_client
from pythonosc import osc_server
from pythonosc import dispatcher
from config import EyeTrackConfig
from osc.OSCMessage import OSCMessage, OSCMessageType
from osc.VRCFTModuleMessenger import VRCFTModuleSender
from osc.VRChatOSCSender import VRChatOSCSender
import queue
import threading
class OSCManager:
def __init__(
self,
osc_message_in_queue: queue.Queue[OSCMessage],
config: EyeTrackConfig,
):
self.sender_cancellation_event = threading.Event()
self.receiver_cancellation_event = threading.Event()
self.listeners = {}
self.osc_message_in_queue = osc_message_in_queue
self.config = config
self.settings = config.settings
self.osc_sender: Optional[OSCSender] = None
self.osc_receiver = None
self.osc_sender_thread: Optional[threading.Thread] = None
self.osc_receiver_thread: Optional[threading.Thread] = None
def start(self):
self.setup_sender()
self.setup_receiver()
def setup_sender(self):
print(f"\033[92m[INFO] Setting up OSC sender\033[0m")
self.sender_cancellation_event.clear()
self.osc_sender = OSCSender(self.sender_cancellation_event, self.osc_message_in_queue, self.config)
self.osc_sender_thread = threading.Thread(target=self.osc_sender.run)
self.osc_sender_thread.start()
def setup_receiver(self):
if self.settings.gui_ROSC:
self.receiver_cancellation_event.clear()
print(f"\033[92m[INFO] Setting up OSC receiver\033[0m")
self.osc_receiver = OSCReceiver(self.receiver_cancellation_event, self.config, self.listeners)
self.osc_receiver_thread = threading.Thread(target=self.osc_receiver.run)
self.osc_receiver_thread.start()
def register_listeners(self, osc_address: str, callbacks: Iterable[Callable]):
if not self.listeners.get(osc_address):
self.listeners[osc_address] = []
self.listeners[osc_address].extend(callbacks)
def update(self, data: dict):
keys = set(data.keys())
sender_trigger_keys = {
"gui_osc_port",
"gui_VRCFTModulePort",
"gui_VRCFTModuleIPAddress",
"gui_use_module",
}
if sender_trigger_keys.intersection(keys):
self.stop_sender()
self.setup_sender()
receiver_trigger_keys = {
"gui_ROSC",
"gui_osc_receiver_port",
}
if receiver_trigger_keys.intersection(keys):
self.stop_receiver()
self.setup_receiver()
def shutdown(self):
self.stop_sender()
self.stop_receiver()
def stop_sender(self):
self.sender_cancellation_event.set()
self.osc_sender_thread.join()
def stop_receiver(self):
if self.osc_receiver_thread:
self.receiver_cancellation_event.set()
self.osc_receiver_thread.join()
class OSCSender:
def __init__(
self,
cancellation_event: threading.Event,
msg_queue: queue.Queue[OSCMessage],
main_config: EyeTrackConfig,
):
self.cancellation_event = cancellation_event
self.msg_queue = msg_queue
self.main_config = main_config
self.config = main_config.settings
self.vrc_sender = VRChatOSCSender()
self.module_sender = VRCFTModuleSender()
self.vrc_client = None
self.vrcft_client = None
def run(self):
self.vrc_client = udp_client.SimpleUDPClient(self.config.gui_osc_address, int(self.config.gui_osc_port))
self.vrcft_client = udp_client.SimpleUDPClient(
self.config.gui_VRCFTModuleIPAddress,
int(self.config.gui_VRCFTModulePort),
)
vrc_osc_output_client = self.vrc_client
if self.config.gui_use_module:
vrc_osc_output_client = self.vrcft_client
while not self.cancellation_event.is_set():
try:
osc_message: OSCMessage = self.msg_queue.get(block=True, timeout=0.1)
match osc_message.type:
case OSCMessageType.EYE_INFO:
self.vrc_sender.output_osc_info(
osc_message=osc_message,
client=vrc_osc_output_client,
main_config=self.main_config,
config=self.config,
)
case OSCMessageType.VRCFT_MODULE_INFO:
self.module_sender.send(osc_message=osc_message, client=self.vrcft_client)
case _:
raise Exception("Encountered message without a handler %s", osc_message.type)
except queue.Empty:
continue
class OSCReceiver:
def __init__(
self,
cancellation_event: threading.Event,
main_config: EyeTrackConfig,
listeners: Dict[str, Callable[[OSCMessage], None]],
):
self.config = main_config.settings
self.cancellation_event = cancellation_event
self.dispatcher = dispatcher.Dispatcher()
self.listeners = listeners
self.server_thread = None
try:
# this thing sucks ass god fucking damn it.
# like, there is no way of shutting it down UNLESS you run it in a thread
# which is kinda dumb, but oh well.
# Also, it doesn't shutdown properly. It's STILL connected to the port
self.server = osc_server.OSCUDPServer(
(self.config.gui_osc_address, int(self.config.gui_osc_receiver_port)),
self.dispatcher,
)
except Exception: # noqa
print(f"\033[91m[ERROR] OSC Receive port: {self.config.gui_osc_receiver_port} occupied.\033[0m")
def shutdown(self):
print("\033[94m[INFO] Exiting OSC Receiver\033[0m")
try:
self.server.shutdown()
self.server_thread.join()
except Exception: # noqa
pass
def handle_osc_message(self, address, value):
for listener in self.listeners.get(address, []):
listener(OSCMessage(type=OSCMessageType.EYE_INFO, data=value))
def run(self):
try:
self.dispatcher.set_default_handler(self.handle_osc_message)
print("\033[92m[INFO] OSC Listening on {}\033[0m".format(self.server.server_address))
self.server_thread = threading.Thread(target=self.server.serve_forever)
self.server_thread.start()
while not self.cancellation_event.is_set():
sleep(10)
self.shutdown()
except Exception: # noqa:
print(f"\033[91m[ERROR] OSC Receive port: {self.config.gui_osc_receiver_port} occupied.\033[0m")

View File

@ -28,7 +28,7 @@ LICENSE: Summer Software Distribution License 1.0
"""
import cv2
import numpy as np
from enum import IntEnum
from eye import EyeId
from utils.img_utils import safe_crop
from utils.misc_utils import clamp
import os
@ -46,13 +46,6 @@ else:
process.nice()
class EyeId(IntEnum):
RIGHT = 0
LEFT = 1
BOTH = 2
SETTINGS = 3
def ellipse_model(data, y, f):
"""
There is no need to make this process a function, since making the process a function will slow it down a little by calling it.

View File

@ -6,7 +6,7 @@ from colorama import Fore
from config import EyeTrackConfig, EyeTrackSettingsConfig
from threading import Event
from osc import EyeId # TODO this is bad, fix this
from eye import EyeId
class BaseSettingsWidget:
@ -25,9 +25,7 @@ class BaseSettingsWidget:
self.main_config = main_config
self.config = main_config.settings
self.initialized_modules = self._initialize_modules(
settings_modules=settings_modules, widget_id=widget_id
)
self.initialized_modules = self._initialize_modules(settings_modules=settings_modules, widget_id=widget_id)
self.general_settings_layout = []
for module in self.initialized_modules:
@ -41,8 +39,16 @@ class BaseSettingsWidget:
background_color="#424042",
),
],
[sg.Text("", background_color="#424042"), ],
[sg.Button("Reset settings to default", key=self.reset_button_key, button_color="#c40e23")],
[
sg.Text("", background_color="#424042"),
],
[
sg.Button(
"Reset settings to default",
key=self.reset_button_key,
button_color="#c40e23",
)
],
]
self.cancellation_event = (

View File

@ -0,0 +1,34 @@
from queue import Queue
from config import EyeTrackConfig
from eye import EyeId
from osc.OSCMessage import OSCMessage, OSCMessageType
from settings.BaseSettings import BaseSettingsWidget
from settings.modules.VRCFTSettingsModule import VRCFTSettingsModule
class VRCFTSettingsWidget(BaseSettingsWidget):
def __init__(self, widget_id: EyeId, main_config: EyeTrackConfig, osc_queue_in: Queue[OSCMessage]):
self.osc_queue = osc_queue_in
settings_modules = [
VRCFTSettingsModule,
]
super().__init__(widget_id, main_config, settings_modules)
def _update_and_save_config(self, validated_data: dict):
self.main_config.update(validated_data, save=True)
for field, value in validated_data.items():
self.osc_queue.put(
OSCMessage(
type=OSCMessageType.VRCFT_MODULE_INFO,
data={
"command": "set",
"field": field,
"value": value,
},
)
)
self.is_saving = False

View File

@ -20,14 +20,12 @@
@@@@@@@@@@@@@(
Copyright (c) 2023 EyeTrackVR <3
LICENSE: GNU GPLv3
LICENSE: GNU GPLv3
------------------------------------------------------------------------------------------------------
"""
import PySimpleGUI as sg
from config import EyeTrackConfig
from osc import EyeId
from eye import EyeId
from settings.BaseSettings import BaseSettingsWidget
from settings.modules.AdvancedTrackingAlgoSettingsModule import (

View File

@ -25,7 +25,7 @@ LICENSE: GNU GPLv3
"""
from config import EyeTrackConfig
from osc import EyeId
from eye import EyeId
from settings.BaseSettings import BaseSettingsWidget
from settings.modules.GeneralSettingsModule import GeneralSettingsModule
@ -41,4 +41,3 @@ class SettingsWidget(BaseSettingsWidget):
OSCSettingsModule,
]
super().__init__(widget_id, main_config, settings_modules)

View File

@ -1,11 +1,34 @@
def check_is_float_convertible(v: str):
"""Check if value provided can be converted to a float or double.
import ipaddress
PySimpleGUI does not support floats or doubles in UI, so we have to make sure
that what the user typed in is correct
def check_is_float_convertible(v: str):
"""
Check if value provided can be converted to a float or double.
PySimpleGUI does not support floats or doubles in UI, so we have to make sure
that what the user typed in is correct
"""
try:
float(v)
return v
except ValueError:
raise ValueError("Please provide a proper number")
def try_convert_to_float(v: str):
"""
Check if value provided can be converted to a float and return converted result
Basically what `check_is_float_convertible` does but returns a float
"""
try:
return float(check_is_float_convertible(v))
except ValueError:
raise
def check_is_ip_address(v: str):
try:
ipaddress.IPv4Address(v)
return v
except ValueError:
raise ValueError("Please provide a valid IP Address")

View File

@ -15,6 +15,7 @@ class OSCValidationModel(BaseValidationModel):
gui_vrc_native: bool
gui_osc_vrcft_v1: bool
gui_osc_vrcft_v2: bool
gui_use_module: bool
@model_validator(mode="after")
def check_osc_vrcft_versions(self):
@ -42,12 +43,22 @@ class OSCSettingsModule(BaseSettingsModule):
self.gui_vrc_native = f"-VRCNATIVE{widget_id}-"
self.gui_osc_vrcft_v1 = f"-OSCVRCFTV1{widget_id}-"
self.gui_osc_vrcft_v2 = f"-OSCVRCFTV2{widget_id}-"
self.gui_use_module = f"-OSCUSEMODULE{widget_id}-"
def get_layout(self):
return [
[
sg.Text("OSC Settings:", background_color="#242224"),
],
[
sg.Checkbox(
"Use ETVR VRCFT Module",
default=self.config.gui_use_module,
key=self.gui_use_module,
background_color="#424042",
tooltip="Toggle output to VRCFT Module or just regular OSC port",
),
],
[
sg.Checkbox(
"VRC Native Eyetracking",

View File

@ -0,0 +1,225 @@
from typing import Iterable
import PySimpleGUI as sg
from pydantic import AfterValidator
from typing_extensions import Annotated
from settings.modules.BaseModule import BaseSettingsModule, BaseValidationModel
from settings.modules.CommonFieldValidators import check_is_ip_address, try_convert_to_float
class VRCFTSettingsModuleValidationModel(BaseValidationModel):
gui_VRCFTModulePort: int
gui_VRCFTModuleIPAddress: Annotated[str, AfterValidator(check_is_ip_address)]
gui_ShouldEmulateEyeWiden: bool
gui_ShouldEmulateEyeSquint: bool
gui_ShouldEmulateEyebrows: bool
gui_WidenThresholdV1_min: float
gui_WidenThresholdV1_max: float
gui_WidenThresholdV2_min: float
gui_WidenThresholdV2_max: float
gui_SqueezeThresholdV1_min: float
gui_SqueezeThresholdV1_max: float
gui_SqueezeThresholdV2_min: float
gui_SqueezeThresholdV2_max: float
gui_EyebrowThresholdRising: float
gui_EyebrowThresholdLowering: float
# this is a hack. I don't like it, but that's what I gotta do to make both, Pydantic and PySimpleGUI happy
gui_OutputMultiplier: Annotated[float, AfterValidator(try_convert_to_float)]
class VRCFTSettingsModule(BaseSettingsModule):
def __init__(self, config, widget_id, **kwargs):
super().__init__(config=config, widget_id=widget_id, **kwargs)
self.validation_model = VRCFTSettingsModuleValidationModel
self.gui_VRCFTModulePort = f"-VRCFTSETTINGSPORTNUMBER{widget_id}"
self.gui_VRCFTModuleIPAddress = f"-VRCFTSETTINGSIPNUMBER{widget_id}"
self.gui_ShouldEmulateEyeWiden = f"-VRCFTSETTINGSEMULATEWIDEN{widget_id}"
self.gui_ShouldEmulateEyeSquint = f"-VRCFTSETTINGSEMULATEEYEWIDEN{widget_id}"
self.gui_ShouldEmulateEyebrows = f"-VRCFTSETTINGSEMULATEEYEBROWS{widget_id}"
self.gui_WidenThresholdV1_min = f"-VRCFTSETTINGSWIDENTHRESHOLDV1MIN{widget_id}"
self.gui_WidenThresholdV1_max = f"-VRCFTSETTINGSWIDENTHRESHOLDV1MAX{widget_id}"
self.gui_WidenThresholdV2_min = f"-VRCFTSETTINGSWIDENTHRESHOLDV2MIN{widget_id}"
self.gui_WidenThresholdV2_max = f"-VRCFTSETTINGSWIDENTHRESHOLDV2MAX{widget_id}"
self.gui_SqueezeThresholdV1_min = f"-VRCFTSETTINGSSQUEEZETHRESHOLDV1MIN{widget_id}"
self.gui_SqueezeThresholdV1_max = f"-VRCFTSETTINGSSQUEEZETHRESHOLDV1MAX{widget_id}"
self.gui_SqueezeThresholdV2_min = f"-VRCFTSETTINGSSQUEEZETHRESHOLDV2MIN{widget_id}"
self.gui_SqueezeThresholdV2_max = f"-VRCFTSETTINGSSQUEEZETHRESHOLDV2MAX{widget_id}"
self.gui_EyebrowThresholdRising = f"-VRCFTSETTINGSEYEBROWTHRESHOLDRISING{widget_id}"
self.gui_EyebrowThresholdLowering = f"-VRCFTSETTINGSEYEBROWTHRESHOLDLOWERING{widget_id}"
self.gui_OutputMultiplier = f"-VRCFTSETTINGSOUTPUTMULTIPLIER{widget_id}"
def get_layout(self) -> Iterable:
return [
[
sg.Text("Emulation selection:", background_color="#242224"),
],
[
sg.Checkbox(
"Emulate Eye Widen",
default=self.config.gui_ShouldEmulateEyeWiden,
key=self.gui_ShouldEmulateEyeWiden,
background_color="#424042",
),
sg.Checkbox(
"Emulate Eye Squint",
default=self.config.gui_ShouldEmulateEyeSquint,
key=self.gui_ShouldEmulateEyeSquint,
background_color="#424042",
),
sg.Checkbox(
"Emulate Eyebrows",
default=self.config.gui_ShouldEmulateEyebrows,
key=self.gui_ShouldEmulateEyebrows,
background_color="#424042",
),
],
[
sg.Text("General Module Settings:", background_color="#242224"),
],
[
sg.Text("VRCFT Module listening IP", background_color="#242224"),
sg.InputText(
self.config.gui_VRCFTModuleIPAddress,
key=self.gui_VRCFTModuleIPAddress,
size=(0, 10),
tooltip="Ip on which the module should listen.",
),
sg.Text("port", background_color="#242224"),
sg.InputText(
self.config.gui_VRCFTModulePort,
key=self.gui_VRCFTModulePort,
size=(0, 10),
tooltip="UDP port on which the module should listen.",
),
],
[
sg.Text("VRCFT Module output multiplier", background_color="#242224"),
sg.InputText(
self.config.gui_OutputMultiplier,
key=self.gui_OutputMultiplier,
size=(0, 10),
tooltip="Output multiplier adjusts the output by the given amount",
),
],
[
sg.Text("Eye Widen thresholds:", background_color="#424042"),
],
[
sg.Text("V1 Min:", background_color="#424042"),
sg.Slider(
range=(0, 1),
resolution=0.01,
default_value=self.config.gui_WidenThresholdV1_min,
orientation="h",
key=self.gui_WidenThresholdV1_min,
background_color="#424042",
tooltip="Controls the point at which the emulation should start for v1 params, reacts to openness",
),
sg.Text("V1 Max:", background_color="#424042"),
sg.Slider(
range=(0, 2),
resolution=0.01,
default_value=self.config.gui_WidenThresholdV1_max,
orientation="h",
key=self.gui_WidenThresholdV1_max,
background_color="#424042",
tooltip="Controls the maximum range of widen emulation",
),
],
[
sg.Text("V2 Min:", background_color="#424042"),
sg.Slider(
range=(0, 2),
resolution=0.01,
default_value=self.config.gui_WidenThresholdV2_min,
orientation="h",
key=self.gui_WidenThresholdV2_min,
background_color="#424042",
tooltip="Controls the point at which the emulation should start for v2 params, reacts to openness",
),
sg.Text("V2 Max:", background_color="#424042"),
sg.Slider(
range=(0, 2),
resolution=0.01,
default_value=self.config.gui_WidenThresholdV2_max,
orientation="h",
key=self.gui_WidenThresholdV2_max,
background_color="#424042",
tooltip="Controls the maximum range of widen emulation",
),
],
[
sg.Text("Eye Squeeze thresholds:", background_color="#424042"),
],
[
sg.Text("V1 Min:", background_color="#424042"),
sg.Slider(
range=(0, 1),
resolution=0.01,
default_value=self.config.gui_SqueezeThresholdV1_min,
orientation="h",
key=self.gui_SqueezeThresholdV1_min,
background_color="#424042",
tooltip="Controls the point at which the emulation should start for v1 params, reacts to openness",
),
sg.Text("V1 Max:", background_color="#424042"),
sg.Slider(
range=(0, 2),
resolution=0.01,
default_value=self.config.gui_SqueezeThresholdV1_max,
orientation="h",
key=self.gui_SqueezeThresholdV1_max,
background_color="#424042",
tooltip="Controls the maximum range of squeeze emulation",
),
],
[
sg.Text("V2 Min:", background_color="#424042"),
sg.Slider(
range=(0, 1),
resolution=0.01,
default_value=self.config.gui_SqueezeThresholdV2_min,
orientation="h",
key=self.gui_SqueezeThresholdV2_min,
background_color="#424042",
tooltip="Controls the point at which the emulation should start for v2 params, reacts to openness",
),
sg.Text("V2 Max:", background_color="#424042"),
sg.Slider(
range=(-2, 0),
resolution=0.01,
default_value=self.config.gui_SqueezeThresholdV2_max,
orientation="h",
key=self.gui_SqueezeThresholdV2_max,
background_color="#424042",
tooltip="Controls the maximum range of squeeze emulation",
),
],
[
sg.Text("Eyebrow emulation Thresholds:", background_color="#424042"),
],
[
sg.Text("Rising:", background_color="#424042"),
sg.Slider(
range=(0, 1),
resolution=0.01,
default_value=self.config.gui_EyebrowThresholdRising,
orientation="h",
key=self.gui_EyebrowThresholdRising,
background_color="#424042",
tooltip="Controls the point at which the emulation should start, reacts to openness",
),
sg.Text("Lowering:", background_color="#424042"),
sg.Slider(
range=(0, 2),
resolution=0.01,
default_value=self.config.gui_EyebrowThresholdLowering,
orientation="h",
key=self.gui_EyebrowThresholdLowering,
background_color="#424042",
tooltip="Controls the maximum range of eyebrows emulation",
),
],
]

View File

@ -1,12 +1,6 @@
import numpy as np
from enum import IntEnum
class EyeId(IntEnum):
RIGHT = 0
LEFT = 1
BOTH = 2
SETTINGS = 3
from eye import EyeId
def velocity_falloff(self, var, out_x, out_y):

View File

@ -7,15 +7,21 @@ from typing import Union
is_nt = True if os.name == "nt" else False
def PlaySound(*args, **kwargs): pass
def PlaySound(*args, **kwargs):
pass
SND_FILENAME = SND_ASYNC = 1
if is_nt:
import winsound
PlaySound = winsound.PlaySound
SND_FILENAME = winsound.SND_FILENAME
SND_ASYNC = winsound.SND_ASYNC
def clamp(x, low, high):
return max(low, min(x, high))
@ -42,7 +48,7 @@ class FastMedian:
self.more, self.__median = None, None
if inits is not None:
[self + x for x in inits]
# When full, push the median of current values to next list, then reset.
def __add__(self, x):
self.__median = None
@ -52,11 +58,11 @@ class FastMedian:
self.more + self.__medianPrim(self.all)
# It's going to be slower because of the re-allocation.
self.all = [] # reset
# If there is a next list, ask its median. Else, work it out locally.
def median(self):
return self.more.median() if self.more else self.__medianPrim(self.all)
# Only recompute median if we do not know it already.
def __medianPrim(self, all):
if self.__median is None:

View File

@ -1,5 +1,5 @@
import pytest
from EyeTrackApp.config import (
from config import (
EyeTrackConfig,
EyeTrackCameraConfig,
EyeTrackSettingsConfig,

View File

@ -42,7 +42,7 @@ requires = ["poetry-core>=1.0.0"]
build-backend = "poetry.core.masonry.api"
[tool.pytest.ini_options]
addopts = "-ra -q --cov=. --cov-report html --cov-report term"
addopts = "-ra -q"
pythonpath = "."
python_files = [
"test_*.py"

View File

@ -0,0 +1,20 @@
import dataclasses
@dataclasses.dataclass
class EyeInfoMock:
x: int
y: int
blink: float
pupil_dilation: float
avg_velocity: float
class SimpleUDPClientMock:
def __init__(self, osc_address, port):
self.osc_address = osc_address
self.port = port
self.messages = []
def send_message(self, address, value):
self.messages.append((address, value))

View File

@ -1,314 +0,0 @@
import dataclasses
import threading
from queue import Queue
from time import sleep
from unittest import mock
import pytest
from EyeTrackApp.osc import VRChatOSC
@dataclasses.dataclass
class EyeInfoMock:
x: int
y: int
blink: float
pupil_dilation: float
avg_velocity: float
class SimpleUDPClientMock:
def __init__(self, osc_address, port):
self.osc_address = osc_address
self.port = port
self.messages = []
def send_message(self, address, value):
self.messages.append((address, value))
@pytest.mark.parametrize(
"eye_id,messages,expected_outcome",
[
(
0,
[
EyeInfoMock(
x=0,
y=0,
blink=1,
pupil_dilation=0,
avg_velocity=0,
),
],
[
("/avatar/parameters/v2/EyeX", 0),
("/avatar/parameters/v2/EyeY", 0),
("/avatar/parameters/v2/EyeLid", 1.0),
],
),
(
1,
[
EyeInfoMock(
x=0,
y=0,
blink=1,
pupil_dilation=0,
avg_velocity=0,
),
],
[
("/avatar/parameters/v2/EyeX", 0),
("/avatar/parameters/v2/EyeY", 0),
("/avatar/parameters/v2/EyeLid", 1.0),
],
),
],
)
def test_send_command_v2_params_single_eye(main_config_v2_params, eye_id, messages, expected_outcome):
with mock.patch("EyeTrackApp.osc.udp_client.SimpleUDPClient", SimpleUDPClientMock):
cancellation_event = threading.Event()
msg_queue = Queue()
client = VRChatOSC(
main_config=main_config_v2_params,
msg_queue=msg_queue,
cancellation_event=cancellation_event,
)
osc_thread = threading.Thread(target=client.run)
osc_thread.start()
for message in messages:
sleep(0.01)
msg_queue.put((eye_id, message))
cancellation_event.set()
osc_thread.join()
assert msg_queue.empty()
assert client.client.messages == expected_outcome
@pytest.mark.parametrize(
"eye_data,expected_outcome",
[
(
[
(
0,
EyeInfoMock(
x=0,
y=0,
blink=1,
pupil_dilation=1,
avg_velocity=0,
),
),
(
1,
EyeInfoMock(
x=10,
y=5,
blink=0.5,
pupil_dilation=1,
avg_velocity=0,
),
),
],
[
("/avatar/parameters/v2/EyeRightX", 0),
("/avatar/parameters/v2/EyeRightY", 0),
("/avatar/parameters/v2/EyeLidRight", 1.0),
("/avatar/parameters/v2/EyeLeftX", 10),
("/avatar/parameters/v2/EyeLeftY", 5),
("/avatar/parameters/v2/EyeLidLeft", 0.5),
],
),
# binary blink
(
[
(
0,
EyeInfoMock(
x=0,
y=0,
blink=0,
pupil_dilation=1,
avg_velocity=0,
),
),
(
1,
EyeInfoMock(
x=10,
y=5,
blink=0,
pupil_dilation=1,
avg_velocity=0,
),
),
],
[
("/avatar/parameters/v2/EyeLidRight", 0.0),
("/avatar/parameters/v2/EyeLidRight", 0.0),
("/avatar/parameters/v2/EyeLidRight", 0.0),
("/avatar/parameters/v2/EyeLidRight", 0.0),
("/avatar/parameters/v2/EyeRightX", 0),
("/avatar/parameters/v2/EyeRightY", 0),
("/avatar/parameters/v2/EyeLidRight", 0.0),
("/avatar/parameters/v2/EyeLidLeft", 0.0),
("/avatar/parameters/v2/EyeLidLeft", 0.0),
("/avatar/parameters/v2/EyeLidLeft", 0.0),
("/avatar/parameters/v2/EyeLidLeft", 0.0),
("/avatar/parameters/v2/EyeLeftX", 0),
("/avatar/parameters/v2/EyeLeftY", 5),
("/avatar/parameters/v2/EyeLidLeft", 0.0),
],
),
],
)
def test_send_command_v2_params_dual_eye(main_config_v2_params, eye_data, expected_outcome):
main_config_v2_params.eye_display_id = 2
with mock.patch("EyeTrackApp.osc.udp_client.SimpleUDPClient", SimpleUDPClientMock):
cancellation_event = threading.Event()
msg_queue = Queue()
client = VRChatOSC(
main_config=main_config_v2_params,
msg_queue=msg_queue,
cancellation_event=cancellation_event,
)
osc_thread = threading.Thread(target=client.run)
osc_thread.start()
for eye_id, message in eye_data:
sleep(0.01)
msg_queue.put((eye_id, message))
cancellation_event.set()
osc_thread.join()
assert msg_queue.empty()
assert client.client.messages == expected_outcome
@pytest.mark.parametrize(
"eye_data,expected_outcome",
[
(
[
(
0,
EyeInfoMock(
x=0,
y=0,
blink=0,
pupil_dilation=1,
avg_velocity=0,
),
),
(
1,
EyeInfoMock(
x=10,
y=5,
blink=0,
pupil_dilation=1,
avg_velocity=0,
),
),
],
[
("/avatar/parameters/v2/EyeLidRight", 0.0),
("/avatar/parameters/v2/EyeLidRight", 0.0),
("/avatar/parameters/v2/EyeLidRight", 0.0),
("/avatar/parameters/v2/EyeLidRight", 0.0),
("/avatar/parameters/v2/EyeRightX", 0),
("/avatar/parameters/v2/EyeRightY", 0),
("/avatar/parameters/v2/EyeLidRight", 0.0),
("/avatar/parameters/v2/EyeLidLeft", 0.0),
("/avatar/parameters/v2/EyeLidLeft", 0.0),
("/avatar/parameters/v2/EyeLidLeft", 0.0),
("/avatar/parameters/v2/EyeLidLeft", 0.0),
("/avatar/parameters/v2/EyeLidLeft", 0.0),
("/avatar/parameters/v2/EyeLidRight", 0.0),
("/avatar/parameters/v2/EyeLeftX", 0),
("/avatar/parameters/v2/EyeLeftY", 5),
("/avatar/parameters/v2/EyeLidLeft", 0.0),
],
),
(
[
(
0,
EyeInfoMock(
x=0,
y=0,
blink=0,
pupil_dilation=1,
avg_velocity=0,
),
),
(
1,
EyeInfoMock(
x=10,
y=5,
blink=0,
pupil_dilation=1,
avg_velocity=0,
),
),
],
[
# binary blink
("/avatar/parameters/v2/EyeLidRight", 0.0),
("/avatar/parameters/v2/EyeLidRight", 0.0),
("/avatar/parameters/v2/EyeLidRight", 0.0),
("/avatar/parameters/v2/EyeLidRight", 0.0),
("/avatar/parameters/v2/EyeRightX", 0),
("/avatar/parameters/v2/EyeRightY", 0),
# side falloff
("/avatar/parameters/v2/EyeLidRight", 0.0),
("/avatar/parameters/v2/EyeLidLeft", 0.0),
# binary blink again
("/avatar/parameters/v2/EyeLidLeft", 0.0),
("/avatar/parameters/v2/EyeLidLeft", 0.0),
("/avatar/parameters/v2/EyeLidLeft", 0.0),
("/avatar/parameters/v2/EyeLidLeft", 0.0),
("/avatar/parameters/v2/EyeLidRight", 0.0),
("/avatar/parameters/v2/EyeLeftX", 0),
("/avatar/parameters/v2/EyeLeftY", 5),
("/avatar/parameters/v2/EyeLidLeft", 0.0),
],
),
],
)
def test_send_command_v2_params_eye_outer_side_falloff(main_config_v2_params, eye_data, expected_outcome):
main_config_v2_params.eye_display_id = 2
main_config_v2_params.settings.gui_outer_side_falloff = True
with mock.patch("EyeTrackApp.osc.udp_client.SimpleUDPClient", SimpleUDPClientMock):
cancellation_event = threading.Event()
msg_queue = Queue()
client = VRChatOSC(
main_config=main_config_v2_params,
msg_queue=msg_queue,
cancellation_event=cancellation_event,
)
osc_thread = threading.Thread(target=client.run)
osc_thread.start()
for eye_id, message in eye_data:
sleep(0.01)
msg_queue.put((eye_id, message))
cancellation_event.set()
osc_thread.join()
assert msg_queue.empty()
assert client.client.messages == expected_outcome

View File

@ -0,0 +1,269 @@
from queue import Queue
from time import sleep
from unittest import mock
import pytest
from osc.osc import OSCManager, OSCMessage
from osc.OSCMessage import OSCMessageType
from tests import EyeInfoMock, SimpleUDPClientMock
@pytest.mark.parametrize(
"messages,expected_outcome",
[
(
[
OSCMessage(
type=OSCMessageType.EYE_INFO,
data=(
0,
EyeInfoMock(
x=0,
y=0,
blink=1,
pupil_dilation=0,
avg_velocity=0,
),
),
),
],
[
("/tracking/eye/EyesClosedAmount", 0.0),
("/tracking/eye/LeftRightVec", [0.0, 0.0, 1.0, 0.0, 0.0, 1.0]),
],
),
(
[
OSCMessage(
type=OSCMessageType.EYE_INFO,
data=(
1,
EyeInfoMock(
x=0,
y=0,
blink=1,
pupil_dilation=0,
avg_velocity=0,
),
),
),
],
[
("/tracking/eye/EyesClosedAmount", 0.0),
("/tracking/eye/LeftRightVec", [0.0, 0.0, 1.0, 0.0, 0.0, 1.0]),
],
),
],
)
def test_send_command_native_params_single_eye(main_config_native_params, messages, expected_outcome):
with mock.patch("EyeTrackApp.osc.osc.udp_client.SimpleUDPClient", SimpleUDPClientMock):
msg_queue = Queue()
client = OSCManager(
config=main_config_native_params,
osc_message_in_queue=msg_queue,
)
client.start()
for message in messages:
sleep(0.01)
msg_queue.put(message)
client.shutdown()
assert msg_queue.empty()
assert client.osc_sender.client.messages == expected_outcome
@pytest.mark.parametrize(
"eye_data,expected_outcome",
[
(
[
OSCMessage(
type=OSCMessageType.EYE_INFO,
data=(
0,
EyeInfoMock(
x=0,
y=0,
blink=1,
pupil_dilation=1,
avg_velocity=0,
),
),
),
OSCMessage(
type=OSCMessageType.EYE_INFO,
data=(
1,
EyeInfoMock(
x=10,
y=5,
blink=0.5,
pupil_dilation=1,
avg_velocity=0,
),
),
),
],
[
("/tracking/eye/EyesClosedAmount", 0.0),
("/tracking/eye/EyesClosedAmount", 0.0),
# we're expecting 621 as left_y here because that's the default value
# before the first state update with real data, but that's ok
# we're gonna be like 10 messages deep before anyone starts playing
# and if they already are, they won't be able to notice
("/tracking/eye/LeftRightVec", [0.0, 621.0, 1.0, 0.0, 0.0, 1.0]),
("/tracking/eye/EyesClosedAmount", 0.5),
("/tracking/eye/EyesClosedAmount", 0.5),
("/tracking/eye/LeftRightVec", [0.0, 5.0, 1.0, 0.0, 0.0, 1.0]),
],
),
# binary blink
(
[
OSCMessage(
type=OSCMessageType.EYE_INFO,
data=(
0,
EyeInfoMock(
x=0,
y=0,
blink=0,
pupil_dilation=1,
avg_velocity=0,
),
),
),
OSCMessage(
type=OSCMessageType.EYE_INFO,
data=(
1,
EyeInfoMock(
x=10,
y=5,
blink=0,
pupil_dilation=1,
avg_velocity=0,
),
),
),
],
[
("/tracking/eye/EyesClosedAmount", 1.0),
("/tracking/eye/EyesClosedAmount", 1.0),
("/tracking/eye/EyesClosedAmount", 1.0),
("/tracking/eye/EyesClosedAmount", 1.0),
("/tracking/eye/EyesClosedAmount", 1.0),
("/tracking/eye/EyesClosedAmount", 1.0),
("/tracking/eye/LeftRightVec", [0.0, 621.0, 1.0, 0.0, 0.0, 1.0]),
("/tracking/eye/EyesClosedAmount", 1.0),
("/tracking/eye/EyesClosedAmount", 1.0),
("/tracking/eye/EyesClosedAmount", 1.0),
("/tracking/eye/EyesClosedAmount", 1.0),
("/tracking/eye/EyesClosedAmount", 1.0),
("/tracking/eye/EyesClosedAmount", 1.0),
("/tracking/eye/LeftRightVec", [0.0, 5.0, 1.0, 0.0, 0.0, 1.0]),
],
),
],
)
def test_send_command_native_params_dual_eye(main_config_native_params, eye_data, expected_outcome):
main_config_native_params.eye_display_id = 2
with mock.patch("EyeTrackApp.osc.osc.udp_client.SimpleUDPClient", SimpleUDPClientMock):
msg_queue = Queue()
client = OSCManager(
config=main_config_native_params,
osc_message_in_queue=msg_queue,
)
client.start()
for message in eye_data:
sleep(0.01)
msg_queue.put(message)
client.shutdown()
assert msg_queue.empty()
assert client.osc_sender.client.messages == expected_outcome
@pytest.mark.parametrize(
"eye_data,expected_outcome",
[
(
[
OSCMessage(
type=OSCMessageType.EYE_INFO,
data=(
0,
EyeInfoMock(
x=0,
y=0,
blink=0,
pupil_dilation=1,
avg_velocity=0,
),
),
),
OSCMessage(
type=OSCMessageType.EYE_INFO,
data=(
1,
EyeInfoMock(
x=10,
y=5,
blink=0,
pupil_dilation=1,
avg_velocity=0,
),
),
),
],
[
("/tracking/eye/EyesClosedAmount", 1.0),
("/tracking/eye/EyesClosedAmount", 1.0),
("/tracking/eye/EyesClosedAmount", 1.0),
("/tracking/eye/EyesClosedAmount", 1.0),
("/tracking/eye/EyesClosedAmount", 1.0),
("/tracking/eye/EyesClosedAmount", 1.0),
("/tracking/eye/LeftRightVec", [0.0, 621.0, 1.0, 0.0, 0.0, 1.0]),
("/tracking/eye/EyesClosedAmount", 1.0),
("/tracking/eye/EyesClosedAmount", 1.0),
("/tracking/eye/EyesClosedAmount", 1.0),
("/tracking/eye/EyesClosedAmount", 1.0),
("/tracking/eye/EyesClosedAmount", 1.0),
("/tracking/eye/EyesClosedAmount", 1.0),
("/tracking/eye/EyesClosedAmount", 1.0),
("/tracking/eye/EyesClosedAmount", 1.0),
("/tracking/eye/EyesClosedAmount", 1.0),
("/tracking/eye/EyesClosedAmount", 1.0),
("/tracking/eye/EyesClosedAmount", 1.0),
("/tracking/eye/EyesClosedAmount", 1.0),
("/tracking/eye/LeftRightVec", [0.0, 5.0, 1.0, 0.0, 0.0, 1.0]),
],
),
],
)
def test_send_command_native_params_eye_outer_side_falloff(main_config_native_params, eye_data, expected_outcome):
main_config_native_params.eye_display_id = 2
main_config_native_params.settings.gui_outer_side_falloff = True
with mock.patch("EyeTrackApp.osc.osc.udp_client.SimpleUDPClient", SimpleUDPClientMock):
msg_queue = Queue()
client = OSCManager(
config=main_config_native_params,
osc_message_in_queue=msg_queue,
)
client.start()
for message in eye_data:
msg_queue.put(message)
sleep(1)
client.shutdown()
assert msg_queue.empty()
assert client.osc_sender.client.messages == expected_outcome

274
tests/test_osc_v1_params.py Normal file
View File

@ -0,0 +1,274 @@
from queue import Queue
from time import sleep
from unittest import mock
import pytest
from osc.osc import OSCManager, OSCMessage
from osc.OSCMessage import OSCMessageType
from tests import EyeInfoMock, SimpleUDPClientMock
@pytest.mark.parametrize(
"messages,expected_outcome",
[
(
[
OSCMessage(
type=OSCMessageType.EYE_INFO,
data=(
0,
EyeInfoMock(
x=0,
y=0,
blink=1,
pupil_dilation=0,
avg_velocity=0,
),
),
),
],
[
("/avatar/parameters/LeftEyeX", 0),
("/avatar/parameters/RightEyeX", 0),
("/avatar/parameters/EyesY", 0),
("/avatar/parameters/LeftEyeLidExpandedSqueeze", 1.0),
("/avatar/parameters/RightEyeLidExpandedSqueeze", 1.0),
],
),
(
[
OSCMessage(
type=OSCMessageType.EYE_INFO,
data=(
1,
EyeInfoMock(
x=0,
y=0,
blink=1,
pupil_dilation=0,
avg_velocity=0,
),
),
),
],
[
("/avatar/parameters/LeftEyeX", 0),
("/avatar/parameters/RightEyeX", 0),
("/avatar/parameters/EyesY", 0),
("/avatar/parameters/LeftEyeLidExpandedSqueeze", 1.0),
("/avatar/parameters/RightEyeLidExpandedSqueeze", 1.0),
],
),
],
)
def test_send_command_v1_params_single_eye(main_config_v1_params, messages, expected_outcome):
with mock.patch("EyeTrackApp.osc.osc.udp_client.SimpleUDPClient", SimpleUDPClientMock):
msg_queue = Queue()
client = OSCManager(
config=main_config_v1_params,
osc_message_in_queue=msg_queue,
)
client.start()
for message in messages:
sleep(0.01)
msg_queue.put(message)
client.shutdown()
assert msg_queue.empty()
assert client.osc_sender.client.messages == expected_outcome
@pytest.mark.parametrize(
"eye_data,expected_outcome",
[
(
[
OSCMessage(
type=OSCMessageType.EYE_INFO,
data=(
0,
EyeInfoMock(
x=0,
y=0,
blink=1,
pupil_dilation=1,
avg_velocity=0,
),
),
),
OSCMessage(
type=OSCMessageType.EYE_INFO,
data=(
1,
EyeInfoMock(
x=10,
y=5,
blink=0.5,
pupil_dilation=1,
avg_velocity=0,
),
),
),
],
[
("/avatar/parameters/RightEyeLidExpandedSqueeze", 1.0),
("/avatar/parameters/RightEyeX", 0),
("/avatar/parameters/RightEyeLidExpandedSqueeze", 1.0),
("/avatar/parameters/LeftEyeLidExpandedSqueeze", 0.5),
("/avatar/parameters/LeftEyeX", 0),
("/avatar/parameters/LeftEyeLidExpandedSqueeze", 0.5),
("/avatar/parameters/EyesY", 2.5),
],
),
# binary blink
(
[
OSCMessage(
type=OSCMessageType.EYE_INFO,
data=(
0,
EyeInfoMock(
x=0,
y=0,
blink=0,
pupil_dilation=1,
avg_velocity=0,
),
),
),
OSCMessage(
type=OSCMessageType.EYE_INFO,
data=(
1,
EyeInfoMock(
x=10,
y=5,
blink=0,
pupil_dilation=1,
avg_velocity=0,
),
),
),
],
[
("/avatar/parameters/RightEyeLidExpandedSqueeze", 0.0),
("/avatar/parameters/RightEyeLidExpandedSqueeze", 0.0),
("/avatar/parameters/RightEyeLidExpandedSqueeze", 0.0),
("/avatar/parameters/RightEyeLidExpandedSqueeze", 0.0),
("/avatar/parameters/RightEyeLidExpandedSqueeze", 0.0),
("/avatar/parameters/RightEyeLidExpandedSqueeze", 0.0),
("/avatar/parameters/RightEyeX", 0),
("/avatar/parameters/RightEyeLidExpandedSqueeze", 0.0),
("/avatar/parameters/LeftEyeLidExpandedSqueeze", 0.0),
("/avatar/parameters/LeftEyeLidExpandedSqueeze", 0.0),
("/avatar/parameters/LeftEyeLidExpandedSqueeze", 0.0),
("/avatar/parameters/LeftEyeLidExpandedSqueeze", 0.0),
("/avatar/parameters/LeftEyeLidExpandedSqueeze", 0.0),
("/avatar/parameters/LeftEyeLidExpandedSqueeze", 0.0),
("/avatar/parameters/LeftEyeX", 0),
("/avatar/parameters/LeftEyeLidExpandedSqueeze", 0.0),
("/avatar/parameters/EyesY", 2.5),
],
),
],
)
def test_send_command_v1_params_dual_eye(main_config_v1_params, eye_data, expected_outcome):
main_config_v1_params.eye_display_id = 2
with mock.patch("EyeTrackApp.osc.osc.udp_client.SimpleUDPClient", SimpleUDPClientMock):
msg_queue = Queue()
client = OSCManager(
config=main_config_v1_params,
osc_message_in_queue=msg_queue,
)
client.start()
for message in eye_data:
sleep(0.01)
msg_queue.put(message)
client.shutdown()
assert msg_queue.empty()
assert client.osc_sender.client.messages == expected_outcome
@pytest.mark.parametrize(
"eye_data,expected_outcome",
[
(
[
OSCMessage(
type=OSCMessageType.EYE_INFO,
data=(
0,
EyeInfoMock(
x=0,
y=0,
blink=0,
pupil_dilation=1,
avg_velocity=0,
),
),
),
OSCMessage(
type=OSCMessageType.EYE_INFO,
data=(
1,
EyeInfoMock(
x=10,
y=5,
blink=0,
pupil_dilation=1,
avg_velocity=0,
),
),
),
],
[
("/avatar/parameters/RightEyeLidExpandedSqueeze", 0.0),
("/avatar/parameters/RightEyeLidExpandedSqueeze", 0.0),
("/avatar/parameters/RightEyeLidExpandedSqueeze", 0.0),
("/avatar/parameters/RightEyeLidExpandedSqueeze", 0.0),
("/avatar/parameters/RightEyeLidExpandedSqueeze", 0.0),
("/avatar/parameters/RightEyeLidExpandedSqueeze", 0.0),
("/avatar/parameters/RightEyeX", 0),
("/avatar/parameters/RightEyeLidExpandedSqueeze", 0.0),
("/avatar/parameters/LeftEyeLidExpandedSqueeze", 0.0),
("/avatar/parameters/LeftEyeLidExpandedSqueeze", 0.0),
("/avatar/parameters/LeftEyeLidExpandedSqueeze", 0.0),
("/avatar/parameters/LeftEyeLidExpandedSqueeze", 0.0),
("/avatar/parameters/LeftEyeLidExpandedSqueeze", 0.0),
("/avatar/parameters/LeftEyeLidExpandedSqueeze", 0.0),
("/avatar/parameters/RightEyeLidExpandedSqueeze", 0.0),
("/avatar/parameters/LeftEyeLidExpandedSqueeze", 0.0),
("/avatar/parameters/LeftEyeX", 0),
("/avatar/parameters/LeftEyeLidExpandedSqueeze", 0.0),
("/avatar/parameters/EyesY", 2.5),
],
),
],
)
def test_send_command_v1_params_eye_outer_side_falloff(main_config_v1_params, eye_data, expected_outcome):
main_config_v1_params.eye_display_id = 2
main_config_v1_params.settings.gui_outer_side_falloff = True
with mock.patch("EyeTrackApp.osc.osc.udp_client.SimpleUDPClient", SimpleUDPClientMock):
msg_queue = Queue()
client = OSCManager(
config=main_config_v1_params,
osc_message_in_queue=msg_queue,
)
client.start()
for message in eye_data:
msg_queue.put(message)
sleep(1)
client.shutdown()
assert msg_queue.empty()
assert client.osc_sender.client.messages == expected_outcome

275
tests/test_osc_v2_params.py Normal file
View File

@ -0,0 +1,275 @@
from queue import Queue
from time import sleep
from unittest import mock
import pytest
from osc.osc import OSCManager, OSCMessage
from osc.OSCMessage import OSCMessageType
from tests import EyeInfoMock, SimpleUDPClientMock
@pytest.mark.parametrize(
"messages,expected_outcome",
[
(
[
OSCMessage(
type=OSCMessageType.EYE_INFO,
data=(
0,
EyeInfoMock(
x=0,
y=0,
blink=1,
pupil_dilation=0,
avg_velocity=0,
),
),
),
],
[
("/avatar/parameters/v2/EyeX", 0),
("/avatar/parameters/v2/EyeY", 0),
("/avatar/parameters/v2/EyeLid", 1.0),
("/avatar/parameters/v2/EyeLid", 1.0),
],
),
(
[
OSCMessage(
type=OSCMessageType.EYE_INFO,
data=(
1,
EyeInfoMock(
x=0,
y=0,
blink=1,
pupil_dilation=0,
avg_velocity=0,
),
),
),
],
[
("/avatar/parameters/v2/EyeX", 0),
("/avatar/parameters/v2/EyeY", 0),
("/avatar/parameters/v2/EyeLid", 1.0),
("/avatar/parameters/v2/EyeLid", 1.0),
],
),
],
)
def test_send_command_v2_params_single_eye(main_config_v2_params, messages, expected_outcome):
with mock.patch("EyeTrackApp.osc.osc.udp_client.SimpleUDPClient", SimpleUDPClientMock):
msg_queue = Queue()
client = OSCManager(
config=main_config_v2_params,
osc_message_in_queue=msg_queue,
)
client.start()
for message in messages:
sleep(0.01)
msg_queue.put(message)
client.shutdown()
assert msg_queue.empty()
assert client.osc_sender.client.messages == expected_outcome
@pytest.mark.parametrize(
"eye_data,expected_outcome",
[
(
[
OSCMessage(
type=OSCMessageType.EYE_INFO,
data=(
0,
EyeInfoMock(
x=0,
y=0,
blink=1,
pupil_dilation=1,
avg_velocity=0,
),
),
),
OSCMessage(
type=OSCMessageType.EYE_INFO,
data=(
1,
EyeInfoMock(
x=10,
y=5,
blink=0.5,
pupil_dilation=1,
avg_velocity=0,
),
),
),
],
[
("/avatar/parameters/v2/EyeLidRight", 1.0),
("/avatar/parameters/v2/EyeRightX", 0),
("/avatar/parameters/v2/EyeRightY", 0),
("/avatar/parameters/v2/EyeLidRight", 1.0),
("/avatar/parameters/v2/EyeLidLeft", 0.5),
("/avatar/parameters/v2/EyeLeftX", 10),
("/avatar/parameters/v2/EyeLeftY", 5),
("/avatar/parameters/v2/EyeLidLeft", 0.5),
],
),
# binary blink
(
[
OSCMessage(
type=OSCMessageType.EYE_INFO,
data=(
0,
EyeInfoMock(
x=0,
y=0,
blink=0,
pupil_dilation=1,
avg_velocity=0,
),
),
),
OSCMessage(
type=OSCMessageType.EYE_INFO,
data=(
1,
EyeInfoMock(
x=10,
y=5,
blink=0,
pupil_dilation=1,
avg_velocity=0,
),
),
),
],
[
("/avatar/parameters/v2/EyeLidRight", 0.0),
("/avatar/parameters/v2/EyeLidRight", 0.0),
("/avatar/parameters/v2/EyeLidRight", 0.0),
("/avatar/parameters/v2/EyeLidRight", 0.0),
("/avatar/parameters/v2/EyeLidRight", 0.0),
("/avatar/parameters/v2/EyeLidRight", 0.0),
("/avatar/parameters/v2/EyeRightX", 0),
("/avatar/parameters/v2/EyeRightY", 0),
("/avatar/parameters/v2/EyeLidRight", 0.0),
("/avatar/parameters/v2/EyeLidLeft", 0.0),
("/avatar/parameters/v2/EyeLidLeft", 0.0),
("/avatar/parameters/v2/EyeLidLeft", 0.0),
("/avatar/parameters/v2/EyeLidLeft", 0.0),
("/avatar/parameters/v2/EyeLidLeft", 0.0),
("/avatar/parameters/v2/EyeLidLeft", 0.0),
("/avatar/parameters/v2/EyeLeftX", 10),
("/avatar/parameters/v2/EyeLeftY", 5),
("/avatar/parameters/v2/EyeLidLeft", 0.0),
],
),
],
)
def test_send_command_v2_params_dual_eye(main_config_v2_params, eye_data, expected_outcome):
main_config_v2_params.eye_display_id = 2
with mock.patch("EyeTrackApp.osc.osc.udp_client.SimpleUDPClient", SimpleUDPClientMock):
msg_queue = Queue()
client = OSCManager(
config=main_config_v2_params,
osc_message_in_queue=msg_queue,
)
client.start()
for message in eye_data:
sleep(0.01)
msg_queue.put(message)
client.shutdown()
assert msg_queue.empty()
assert client.osc_sender.client.messages == expected_outcome
@pytest.mark.parametrize(
"eye_data,expected_outcome",
[
(
[
OSCMessage(
type=OSCMessageType.EYE_INFO,
data=(
0,
EyeInfoMock(
x=0,
y=0,
blink=0,
pupil_dilation=1,
avg_velocity=0,
),
),
),
OSCMessage(
type=OSCMessageType.EYE_INFO,
data=(
1,
EyeInfoMock(
x=10,
y=5,
blink=0,
pupil_dilation=1,
avg_velocity=0,
),
),
),
],
[
("/avatar/parameters/v2/EyeLidRight", 0.0),
("/avatar/parameters/v2/EyeLidRight", 0.0),
("/avatar/parameters/v2/EyeLidRight", 0.0),
("/avatar/parameters/v2/EyeLidRight", 0.0),
("/avatar/parameters/v2/EyeLidRight", 0.0),
("/avatar/parameters/v2/EyeLidRight", 0.0),
("/avatar/parameters/v2/EyeRightX", 0),
("/avatar/parameters/v2/EyeRightY", 0),
("/avatar/parameters/v2/EyeLidRight", 0.0),
("/avatar/parameters/v2/EyeLidLeft", 0.0),
("/avatar/parameters/v2/EyeLidLeft", 0.0),
("/avatar/parameters/v2/EyeLidLeft", 0.0),
("/avatar/parameters/v2/EyeLidLeft", 0.0),
("/avatar/parameters/v2/EyeLidLeft", 0.0),
("/avatar/parameters/v2/EyeLidLeft", 0.0),
("/avatar/parameters/v2/EyeLidRight", 0.0),
("/avatar/parameters/v2/EyeLidLeft", 0.0),
("/avatar/parameters/v2/EyeLeftX", 10),
("/avatar/parameters/v2/EyeLeftY", 5),
("/avatar/parameters/v2/EyeLidLeft", 0.0),
],
),
],
)
def test_send_command_v2_params_eye_outer_side_falloff(main_config_v2_params, eye_data, expected_outcome):
main_config_v2_params.eye_display_id = 2
main_config_v2_params.settings.gui_outer_side_falloff = True
with mock.patch("EyeTrackApp.osc.osc.udp_client.SimpleUDPClient", SimpleUDPClientMock):
msg_queue = Queue()
client = OSCManager(
config=main_config_v2_params,
osc_message_in_queue=msg_queue,
)
client.start()
for message in eye_data:
msg_queue.put(message)
sleep(1)
client.shutdown()
assert msg_queue.empty()
assert client.osc_sender.client.messages == expected_outcome