Merge pull request #80 from Blu3u/HSF-and-new-algos-feature-branch

Fix issues in existing USB-based eye tracker implementation to enable proper functionality.
This commit is contained in:
Prohurtz 2023-04-09 18:18:20 -05:00 committed by GitHub
commit e3a932fb5d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 86 additions and 57 deletions

View File

@ -1,15 +1,25 @@
from config import EyeTrackConfig from config import EyeTrackConfig
from enum import Enum from enum import Enum
import threading
import queue
import cv2 import cv2
import queue
import serial import serial
import serial.tools.list_ports
import threading
import time import time
import numpy as np import numpy as np
WAIT_TIME = 0.1 WAIT_TIME = 0.1
# Serial communication protocol:
# header-begin (2 bytes)
# header-type (2 bytes)
# packet-size (2 bytes)
# packet (packet-size bytes)
ETVR_HEADER = b'\xff\xa0'
ETVR_HEADER_FRAME = b'\xff\xa1'
ETVR_HEADER_LEN = 6
class CameraState(Enum): class CameraState(Enum):
CONNECTING = 0 CONNECTING = 0
@ -39,9 +49,11 @@ class Camera:
self.wired_camera: "cv2.VideoCapture" = None self.wired_camera: "cv2.VideoCapture" = None
self.serial_connection = None self.serial_connection = None
self.last_frame_time = time.time()
self.frame_number = 0 self.frame_number = 0
self.fps = 0
self.start = True self.start = True
self.serialByteBuffer = b'' self.buffer = b''
self.error_message = "\033[93m[WARN] Capture source {} not found, retrying...\033[0m" self.error_message = "\033[93m[WARN] Capture source {} not found, retrying...\033[0m"
@ -66,7 +78,8 @@ class Camera:
or self.camera_status == CameraState.DISCONNECTED or self.camera_status == CameraState.DISCONNECTED
or self.config.capture_source != self.current_capture_source or self.config.capture_source != self.current_capture_source
): ):
port = self.current_capture_source port = self.config.capture_source
self.current_capture_source = port
self.start_serial_connection(port) self.start_serial_connection(port)
else: else:
if ( if (
@ -109,75 +122,92 @@ class Camera:
self.wired_camera.set(cv2.CAP_PROP_POS_FRAMES, 0) self.wired_camera.set(cv2.CAP_PROP_POS_FRAMES, 0)
raise RuntimeError("Problem while getting frame") raise RuntimeError("Problem while getting frame")
frame_number = self.wired_camera.get(cv2.CAP_PROP_POS_FRAMES) frame_number = self.wired_camera.get(cv2.CAP_PROP_POS_FRAMES)
fps = self.wired_camera.get(cv2.CAP_PROP_FPS) self.fps = self.wired_camera.get(cv2.CAP_PROP_FPS)
if should_push: if should_push:
self.push_image_to_queue(image, frame_number, fps) self.push_image_to_queue(image, frame_number, self.fps)
except: except:
print("\033[93m[INFO] Capture source problem, assuming camera disconnected, waiting for reconnect.\033[0m") print("\033[93m[WARN] Capture source problem, assuming camera disconnected, waiting for reconnect.\033[0m")
self.camera_status = CameraState.DISCONNECTED self.camera_status = CameraState.DISCONNECTED
pass pass
def get_next_packet_bounds(self):
beg = -1
while beg == -1:
self.buffer += self.serial_connection.read(2048)
beg = self.buffer.find(ETVR_HEADER + ETVR_HEADER_FRAME)
# Discard any data before the frame header.
if beg > 0:
self.buffer = self.buffer[beg:]
beg = 0
# We know exactly how long the jpeg packet is
end = int.from_bytes(self.buffer[4:6], signed=False, byteorder="little")
self.buffer += self.serial_connection.read(end - len(self.buffer))
return beg, end
def get_next_jpeg_frame(self):
beg, end = self.get_next_packet_bounds()
jpeg = self.buffer[beg+ETVR_HEADER_LEN:end+ETVR_HEADER_LEN]
self.buffer = self.buffer[end+ETVR_HEADER_LEN:]
return jpeg
def get_serial_camera_picture(self, should_push): def get_serial_camera_picture(self, should_push):
start = time.time()
try: try:
bytes = self.serialByteBuffer
if self.serial_connection.in_waiting: if self.serial_connection.in_waiting:
bytes += self.serial_connection.read(4096) # Read in initial bytes jpeg = self.get_next_jpeg_frame()
if jpeg:
a = bytes.find(b'\xff\xd8') # Find start byte for jpeg image # Create jpeg frame from byte string
b = bytes.find(b'\xff\xd9') # Fine end byte for jpeg image image = cv2.imdecode(np.fromstring(jpeg, dtype=np.uint8), cv2.IMREAD_UNCHANGED)
if image is None:
# If the first found end byte is before the start byte, keep reading in serial print("image not found")
# data and discarding the old data until the start byte is before the end byte return
while a > b: # Discard the serial buffer. This is due to the fact that it
bytes = bytes[a:] # may build up some outdated frames. A bit of a workaround here tbh.
a = bytes.find(b'\xff\xd8') self.serial_connection.reset_input_buffer()
b = bytes.find(b'\xff\xd9') self.buffer = b''
if a == -1 or b == -1: # Calculate the fps.
bytes += self.serial_connection.read(2048) current_frame_time = time.time()
delta_time = current_frame_time - self.last_frame_time
if a != -1 and b != -1: # If there is jpeg data self.last_frame_time = current_frame_time
jpg = bytes[a:b + 2] # Create the string of bytes for the current jpeg if delta_time > 0:
bytes = bytes[b + 2:] # Clear the buffer until the end of our current jpeg self.fps = 1 / delta_time
self.serialByteBuffer = bytes self.frame_number = self.frame_number + 1
if should_push:
if jpg: self.push_image_to_queue(image, self.frame_number, self.fps)
# Create jpeg frame from byte string
image = cv2.imdecode(np.fromstring(jpg, dtype=np.uint8), cv2.IMREAD_UNCHANGED)
if image is None:
print("image not found")
else:
self.frame_number = self.frame_number + 1
fps = 1 / (time.time() - start) # Calculate FPS - This could use a better implementation
if should_push:
self.push_image_to_queue(image, self.frame_number, fps)
except UnboundLocalError as ex: except UnboundLocalError as ex:
print(ex) print(ex)
except Exception as ex: except Exception:
print(ex.__class__) print("\033[93m[WARN]Serial capture source problem, assuming camera disconnected, waiting for reconnect.\033[0m")
print( self.serial_connection.close()
"\033[93m[INFO]Serial capture source problem, assuming camera disconnected, waiting for reconnect.\033[0m")
self.camera_status = CameraState.DISCONNECTED self.camera_status = CameraState.DISCONNECTED
pass pass
def start_serial_connection(self, port): def start_serial_connection(self, port):
if self.serial_connection is not None and self.serial_connection.is_open:
# Do nothing. The connection is already open on this port.
if self.serial_connection.port == port:
return
# Otherwise, close the connection before trying to reopen.
self.serial_connection.close()
com_ports = [tuple(p) for p in list(serial.tools.list_ports.comports())]
# Do not try connecting if no such port i.e. device was unplugged.
if not any(p for p in com_ports if port in p):
return
try: try:
serialInst = serial.Serial() conn = serial.Serial(
print("setting baud rate") baudrate = 3000000,
serialInst.baudrate = 2000000 port = port,
print("baud rate set") xonxoff=False,
dsrdtr=False,
rtscts=False)
serialInst.port = port conn.reset_input_buffer()
serialInst.setDTR(False)
serialInst.setRTS(False)
serialInst.open() print(f"\033[94m[INFO] Serial Tracker successfully connected on {port}\033[0m")
print("port open") self.serial_connection = conn
self.serial_connection = serialInst
self.camera_status = CameraState.CONNECTED self.camera_status = CameraState.CONNECTED
except: except Exception:
print("Error Opening Serial Port") self.camera_status = CameraState.DISCONNECTED
def push_image_to_queue(self, image, frame_number, fps): def push_image_to_queue(self, image, frame_number, fps):
# If there's backpressure, just yell. We really shouldn't have this unless we start getting # If there's backpressure, just yell. We really shouldn't have this unless we start getting

View File

@ -9,18 +9,17 @@ repository = "https://github.com/RedHawk989/EyeTrackVR"
[tool.poetry.dependencies] [tool.poetry.dependencies]
python = "~3.10.0" python = "~3.10.0"
python-osc = "^1.8.0" python-osc = "^1.8.0"
requests = "^2.28.0" requests = "^2.28.1"
opencv-python = "^4.6.0.66" opencv-python = "^4.6.0.66"
numpy = "~1.23.5" numpy = "~1.23.5"
pye3d = "^0.3.1.post1" pye3d = "^0.3.1.post1"
pysimplegui = "^4.60.4" pysimplegui = "^4.60.4"
pydantic = "^1.10.2" pydantic = "^1.10.2"
requests = "~2.2.8.2" pyserial = "^3.5"
winotify = [ winotify = [
{ version = "^1.1.0", platform = 'win32' } { version = "^1.1.0", platform = 'win32' }
] ]
onnxruntime = "^1.13.1" onnxruntime = "^1.13.1"
serial = "~0.0.97"
[tool.poetry.group.dev.dependencies] [tool.poetry.group.dev.dependencies]
black = "^22.10.0" black = "^22.10.0"
pyinstaller = "^5.6.2" pyinstaller = "^5.6.2"