diff --git a/EyeTrackApp/Camera/CameraFactory.py b/EyeTrackApp/Camera/CameraFactory.py new file mode 100644 index 0000000..027ec88 --- /dev/null +++ b/EyeTrackApp/Camera/CameraFactory.py @@ -0,0 +1,33 @@ +import cv2 +import numpy as np +import queue +import serial +import serial.tools.list_ports +import threading +import time +from colorama import Fore +from config import EyeTrackCameraConfig +from enum import Enum +import psutil, os +import sys + +from Camera.CameraState import CameraState +from Camera.SerialCamera import SerialCamera +from Camera.SystemCamera import SystemCamera +from Camera.ICameraSource import ICameraSource +from Camera.UDP_Camera.UDP_Camera import UDP_Camera + +# Sorry for the (non-OOP) Python devs. Factory time! +class CameraFactory: + @staticmethod + def get_camera_from_string_type(sourceName: str) -> ICameraSource: + sourceName = str(sourceName) # prevents int to be entered + if sourceName.lower().startswith("com") or sourceName.lower().startswith("/dev/cu") or sourceName.lower().startswith("/dev/tty"): # Windows # macOS # Linux + print(f"{Fore.CYAN}[INFO] Serial camera selected {Fore.RESET}") + return SerialCamera + elif sourceName.lower() == "udp": + print(f"{Fore.YELLOW}[WARN] UDP selected. Prepare for bugs from BOTAlex. Unfinished and extreme alpha. {Fore.RESET}") + return UDP_Camera + else: + print(f"{Fore.CYAN}[INFO] System camera selected {Fore.RESET}") + return SystemCamera \ No newline at end of file diff --git a/EyeTrackApp/Camera/CameraState.py b/EyeTrackApp/Camera/CameraState.py new file mode 100644 index 0000000..493018b --- /dev/null +++ b/EyeTrackApp/Camera/CameraState.py @@ -0,0 +1,6 @@ +from enum import Enum + +class CameraState(Enum): + CONNECTING = 0 + CONNECTED = 1 + DISCONNECTED = 2 \ No newline at end of file diff --git a/EyeTrackApp/Camera/ICameraSource.py b/EyeTrackApp/Camera/ICameraSource.py new file mode 100644 index 0000000..4eec3e5 --- /dev/null +++ b/EyeTrackApp/Camera/ICameraSource.py @@ -0,0 +1,114 @@ +import cv2 +import numpy as np +import queue +import serial +import serial.tools.list_ports +import threading +import time +from colorama import Fore +from config import EyeTrackCameraConfig +from enum import Enum +import psutil, os +import sys +from Camera.CameraState import CameraState +from abc import ABC, abstractmethod + + +# This is when C# dev does Python dev +class ICameraSource: + def __init__( + self, + config: EyeTrackCameraConfig, + camera_index: int, + cancellation_event: "threading.Event", + capture_event: "threading.Event", + camera_status_outgoing: "queue.Queue[CameraState]", + camera_output_outgoing: "queue.Queue(maxsize=20)", + ): + self.camera_status = CameraState.CONNECTING + self.config = config + self.camera_index = camera_index + self.camera_address = config.capture_source + self.camera_status_outgoing = camera_status_outgoing + self.camera_output_outgoing = camera_output_outgoing + self.capture_event = capture_event + self.cancellation_event = cancellation_event + self.current_capture_source = config.capture_source + self.cv2_camera: "cv2.VideoCapture" = None + + self.serial_connection = None + self.last_frame_time = time.time() + self.frame_number = 0 + self.fps = 0 + self.bps = 0 + self.start = True + self.buffer = b"" + self.pf_fps = 0 + self.prevft = 0 + self.newft = 0 + self.fl = [0] + + self.extraInit() + + process = psutil.Process(os.getpid()) # set process priority to low + try: + sys.getwindowsversion() + except AttributeError: + process.nice(10) # UNIX: 0 low 10 high + process.nice() + else: + process.nice(psutil.HIGH_PRIORITY_CLASS) # Windows + process.nice() + # See https://learn.microsoft.com/en-us/windows/win32/api/processthreadsapi/nf-processthreadsapi-getpriorityclass#return-value for values + + + self.error_message = f"{Fore.YELLOW}[WARN] Capture source {{}} not found, retrying...{Fore.RESET}" + + def __del__(self): + pass + + def push_image_to_queue(self, image, frame_number, fps): + # If there's backpressure, just yell. We really shouldn't have this unless we start getting + # some sort of capture event conflict though. + qsize = self.camera_output_outgoing.qsize() + if qsize > 1: + print( + f"{Fore.YELLOW}[WARN] CAPTURE QUEUE BACKPRESSURE OF {qsize}. CHECK FOR CRASH OR TIMING ISSUES IN ALGORITHM.{Fore.RESET}" + ) + pass + self.camera_output_outgoing.put((image, frame_number, fps)) + self.capture_event.clear() + + @abstractmethod + def run(self): + pass + + def extraInit(self): + pass + + def set_output_queue(self, camera_output_outgoing: "queue.Queue"): + self.camera_output_outgoing = camera_output_outgoing + + def get_stream_fps(self): + """Based on how many times this method gets called""" + + # Calculate the fps. + current_frame_time = time.time() + delta_time = current_frame_time - self.last_frame_time + self.last_frame_time = current_frame_time + + # Avoid division by zero + if delta_time > 0: + fps = 1.0 / delta_time + else: + fps = 0 + + # Smooth the FPS using a moving average + self.fl.append(fps) + if len(self.fl) > 60: + self.fl.pop(0) # Keep the list length constant + + # Compute average FPS + fps = sum(self.fl) / len(self.fl) + + return fps diff --git a/EyeTrackApp/Camera/SerialCamera.py b/EyeTrackApp/Camera/SerialCamera.py new file mode 100644 index 0000000..64bca3e --- /dev/null +++ b/EyeTrackApp/Camera/SerialCamera.py @@ -0,0 +1,146 @@ +import struct +import cv2 +import numpy as np +import queue +import serial +import serial.tools.list_ports +import threading +import time +from colorama import Fore +from config import EyeTrackCameraConfig +from enum import Enum +import psutil, os +import sys +from Camera.CameraState import CameraState +from Camera.ICameraSource import ICameraSource +import socket + +WAIT_TIME = 0.1 +# Serial communication protocol: +# header-begin (2 bytes) +# header-type (2 bytes) +# packet-size (2 bytes) +# packet (packet-size bytes) +ETVR_HEADER = b"\xff\xa0" +ETVR_HEADER_FRAME = b"\xff\xa1" +ETVR_HEADER_LEN = 6 + +class SerialCamera(ICameraSource): + def run(self): + while True: + if self.cancellation_event.is_set(): + print(f"{Fore.CYAN}[INFO] Exiting Capture thread{Fore.RESET}") + # openCV won't switch to a new source if provided with one + # so, we have to manually release the camera on exit + + return + should_push = True + # If things aren't open, retry until they are. Don't let read requests come in any earlier + # than this, otherwise we can deadlock ourselves. + if self.config.capture_source != None and self.config.capture_source != "": + self.current_capture_source = self.config.capture_source + addr = str(self.current_capture_source) + + if ( + self.serial_connection is None + or self.camera_status == CameraState.DISCONNECTED + or self.config.capture_source != self.current_capture_source + ): + port = self.config.capture_source + self.current_capture_source = port + self.start_serial_connection(port) + + else: + # We don't have a capture source to try yet, wait for one to show up in the GUI. + if self.cancellation_event.wait(WAIT_TIME): + self.camera_status = CameraState.DISCONNECTED + return + # Assuming we can access our capture source, wait for another thread to request a capture. + # Cycle every so often to see if our cancellation token has fired. This basically uses a + # python event as a context-less, resettable one-shot channel. + if should_push and not self.capture_event.wait(timeout=0.001): + continue + if self.config.capture_source != None: + addr = str(self.current_capture_source) + self.get_serial_camera_picture(should_push) + if not should_push: + # if we get all the way down here, consider ourselves connected + self.camera_status = CameraState.CONNECTED + + def get_next_packet_bounds(self): + beg = -1 + while beg == -1: + self.buffer += self.serial_connection.read(2048) + beg = self.buffer.find(ETVR_HEADER + ETVR_HEADER_FRAME) + # Discard any data before the frame header. + if beg > 0: + self.buffer = self.buffer[beg:] + beg = 0 + # We know exactly how long the jpeg packet is + end = int.from_bytes(self.buffer[4:6], signed=False, byteorder="little") + self.buffer += self.serial_connection.read(end - len(self.buffer)) + return beg, end + + def get_next_jpeg_frame(self): + beg, end = self.get_next_packet_bounds() + jpeg = self.buffer[beg + ETVR_HEADER_LEN : end + ETVR_HEADER_LEN] + self.buffer = self.buffer[end + ETVR_HEADER_LEN :] + return jpeg + + def get_serial_camera_picture(self, should_push): + conn = self.serial_connection + if conn is None: + return + try: + if conn.in_waiting: + jpeg = self.get_next_jpeg_frame() + if jpeg: + # Create jpeg frame from byte string + image = cv2.imdecode(np.fromstring(jpeg, dtype=np.uint8), cv2.IMREAD_UNCHANGED) + if image is None: + print(f"{Fore.YELLOW}[WARN] Frame drop. Corrupted JPEG.{Fore.RESET}") + return + # Discard the serial buffer. This is due to the fact that it + # may build up some outdated frames. A bit of a workaround here tbh. + if conn.in_waiting >= 32768: + print(f"{Fore.CYAN}[INFO] Discarding the serial buffer ({conn.in_waiting} bytes){Fore.RESET}") + conn.reset_input_buffer() + self.buffer = b"" + + fps = self.get_stream_fps() + + if should_push: + self.push_image_to_queue(image, self.frame_number, fps) + except Exception: + print( + f"{Fore.YELLOW}[WARN] Serial capture source problem, assuming camera disconnected, waiting for reconnect.{Fore.RESET}" + ) + conn.close() + self.camera_status = CameraState.DISCONNECTED + pass + + def start_serial_connection(self, port): + if self.serial_connection is not None and self.serial_connection.is_open: + # Do nothing. The connection is already open on this port. + if self.serial_connection.port == port: + return + # Otherwise, close the connection before trying to reopen. + self.serial_connection.close() + com_ports = [tuple(p) for p in list(serial.tools.list_ports.comports())] + # Do not try connecting if no such port i.e. device was unplugged. + if not any(p for p in com_ports if port in p): + return + try: + rate = 115200 if sys.platform == "darwin" else 3000000 # Higher baud rate not working on macOS + conn = serial.Serial(baudrate=rate, port=port, xonxoff=False, dsrdtr=False, rtscts=False) + # Set explicit buffer size for serial. + if sys.platform == "win32": + buffer_size = 32768 + conn.set_buffer_size(rx_size=buffer_size, tx_size=buffer_size) + + print(f"{Fore.CYAN}[INFO] ETVR Serial Tracker device connected on {port}{Fore.RESET}") + self.serial_connection = conn + self.camera_status = CameraState.CONNECTED + except Exception: + print(f"{Fore.CYAN}[INFO] Failed to connect on {port}{Fore.RESET}") + self.camera_status = CameraState.DISCONNECTED \ No newline at end of file diff --git a/EyeTrackApp/Camera/SystemCamera.py b/EyeTrackApp/Camera/SystemCamera.py new file mode 100644 index 0000000..3c4d254 --- /dev/null +++ b/EyeTrackApp/Camera/SystemCamera.py @@ -0,0 +1,120 @@ +import struct +import cv2 +import numpy as np +import queue +import serial +import serial.tools.list_ports +import threading +import time +from colorama import Fore +from config import EyeTrackCameraConfig +from enum import Enum +import psutil, os +import sys +from Camera.CameraState import CameraState +from Camera.ICameraSource import ICameraSource +import socket + +WAIT_TIME = 0.1 + +class SystemCamera(ICameraSource): + def run(self): + OPENCV_PARAMS = [ + cv2.CAP_PROP_OPEN_TIMEOUT_MSEC, + 5000, + cv2.CAP_PROP_READ_TIMEOUT_MSEC, + 5000, + ] + while True: + if self.cancellation_event.is_set(): + print(f"{Fore.CYAN}[INFO] Exiting Capture thread{Fore.RESET}") + # openCV won't switch to a new source if provided with one + # so, we have to manually release the camera on exit + + addr = str(self.current_capture_source) + self.cv2_camera.release() + + return + should_push = True + # If things aren't open, retry until they are. Don't let read requests come in any earlier + # than this, otherwise we can deadlock ourselves. + if self.config.capture_source != None and self.config.capture_source != "": + self.current_capture_source = self.config.capture_source + addr = str(self.current_capture_source) + + if ( + self.cv2_camera is None + or not self.cv2_camera.isOpened() + or self.camera_status == CameraState.DISCONNECTED + or self.config.capture_source != self.current_capture_source + ): + print(self.error_message.format(self.config.capture_source)) + # This requires a wait, otherwise we can error and possible screw up the camera + # firmware. Fickle things. + if self.cancellation_event.wait(WAIT_TIME): + return + self.current_capture_source = self.config.capture_source + # self.cv2_camera = cv2.VideoCapture(self.current_capture_source) + + self.cv2_camera = cv2.VideoCapture() + self.cv2_camera.setExceptionMode(True) + # https://github.com/opencv/opencv/blob/4.8.0/modules/videoio/include/opencv2/videoio.hpp#L803 + self.cv2_camera.open(self.current_capture_source) + should_push = False + else: + # We don't have a capture source to try yet, wait for one to show up in the GUI. + if self.cancellation_event.wait(WAIT_TIME): + self.camera_status = CameraState.DISCONNECTED + return + # Assuming we can access our capture source, wait for another thread to request a capture. + # Cycle every so often to see if our cancellation token has fired. This basically uses a + # python event as a context-less, resettable one-shot channel. + if should_push and not self.capture_event.wait(timeout=0.001): + continue + if self.config.capture_source != None: + addr = str(self.current_capture_source) + self.get_cv2_camera_picture(should_push) + if not should_push: + # if we get all the way down here, consider ourselves connected + self.camera_status = CameraState.CONNECTED + + def get_cv2_camera_picture(self, should_push): + try: + ret, image = self.cv2_camera.read() + height, width = image.shape[:2] # Calculate the aspect ratio + if int(width) > 680: + aspect_ratio = float(width) / float( + height + ) # Determine the new height based on the desired maximum width + new_height = int(680 / aspect_ratio) + image = cv2.resize(image, (680, new_height)) + if not ret: + self.cv2_camera.set(cv2.CAP_PROP_POS_FRAMES, 0) + raise RuntimeError("Problem while getting frame") + frame_number = self.cv2_camera.get(cv2.CAP_PROP_POS_FRAMES) + current_frame_time = time.time() + delta_time = current_frame_time - self.last_frame_time + if delta_time > 0: + current_fps = 1 / delta_time + else: + current_fps = 0 + self.last_frame_time = current_frame_time + + if len(self.fl) < 60: + self.fl.append(current_fps) + else: + self.fl.pop(0) + self.fl.append(current_fps) + + self.fps = sum(self.fl) / len(self.fl) + self.bps = image.nbytes * self.fps + + + if should_push: + self.push_image_to_queue(image, frame_number, self.fps) + except: + print( + f"{Fore.YELLOW}[WARN] Capture source problem, assuming camera disconnected, waiting for reconnect.{Fore.RESET}" + ) + self.camera_status = CameraState.DISCONNECTED + pass \ No newline at end of file diff --git a/EyeTrackApp/Camera/UDP_Camera/PacketHeader.py b/EyeTrackApp/Camera/UDP_Camera/PacketHeader.py new file mode 100644 index 0000000..9d22938 --- /dev/null +++ b/EyeTrackApp/Camera/UDP_Camera/PacketHeader.py @@ -0,0 +1,12 @@ +import struct +import numpy as np +import ctypes + +class PacketHeader: + def __init__(self, headerFormat, dataView, rawDataSize): + self.frame_num: int|None = None + self.id: int|None = None + self.image_buf_size: int|None = None + self.totalPackets: int|None = None + + self.frame_num, self.id, self.image_buf_size, self.totalPackets = struct.unpack_from(headerFormat, dataView, 0) diff --git a/EyeTrackApp/Camera/UDP_Camera/UDP_Camera.py b/EyeTrackApp/Camera/UDP_Camera/UDP_Camera.py new file mode 100644 index 0000000..2a634b7 --- /dev/null +++ b/EyeTrackApp/Camera/UDP_Camera/UDP_Camera.py @@ -0,0 +1,155 @@ +import struct +import cv2 +import numpy as np +import queue +import serial +import serial.tools.list_ports +import threading +import time +from colorama import Fore +from config import EyeTrackCameraConfig +from enum import Enum +import psutil, os +import sys +from Camera.CameraState import CameraState +from Camera.ICameraSource import ICameraSource +import socket +from .PacketHeader import PacketHeader # Python imports stupid. missed a single "." + +WAIT_TIME = 0.1 + +IMAGE_BUFFER_SIZE = 1024 +NUM_MAX_FRAGMENTS = 12 +HEADER_FORMAT = "iiii" + +RED = "\033[91m" +GREEN = "\033[92m" +RESET = "\033[0m" + +# I do not like slow slow slow python - BOTAlex + +class UDP_Camera(ICameraSource): + def extraInit(self): + self.host = "0.0.0.0" + self.port = 3333 + self.num_loaded = 0 + self.packets = [None] * NUM_MAX_FRAGMENTS + self.headerSize = struct.calcsize(HEADER_FORMAT) + self.rawDataBuffer = np.zeros(IMAGE_BUFFER_SIZE + self.headerSize, dtype=np.uint8) + self.rawFullDataBuffer = np.zeros(IMAGE_BUFFER_SIZE*NUM_MAX_FRAGMENTS, dtype=np.uint8) + self.imageBuffView = memoryview(self.rawFullDataBuffer) + self.currentFrameNum = 0 + self.totalDataSize = 0 + + self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + self.sock.bind((self.host, self.port)) + + def run(self): + while True: + if self.cancellation_event.is_set(): + print(f"{Fore.CYAN}[INFO] Exiting Capture thread{Fore.RESET}") + # openCV won't switch to a new source if provided with one + # so, we have to manually release the camera on exit + + return + + bufferView = memoryview(self.rawDataBuffer) + n, senderAddr = self.sock.recvfrom_into(bufferView) + self.handle_packet(n, senderAddr) + + def saveDataToBuff(self, dataView: memoryview, packet: PacketHeader): + payload_start = self.headerSize + payload_end = self.headerSize + packet.image_buf_size + payload = dataView[payload_start:payload_end] + + #time.sleep(0.05) + + offset = IMAGE_BUFFER_SIZE * packet.id # Adjust offset based on payload size + self.imageBuffView[offset: offset + packet.image_buf_size] = payload + # print(f"Packet {packet.id}: {offset}->{offset + packet.image_buf_size}") + + def resetImageBuffer(self): + self.rawFullDataBuffer = np.zeros(IMAGE_BUFFER_SIZE*NUM_MAX_FRAGMENTS, dtype=np.uint8) + + def handle_packet(self, dataSize, senderAddr): + bufferView = memoryview(self.rawDataBuffer) + packet = PacketHeader(HEADER_FORMAT, bufferView, dataSize) + if packet.id < 0 or packet.id >= NUM_MAX_FRAGMENTS: + return + + # Send acknowledgment + # self.sock.sendto(f"{packet.id}:{packet.frame_num}:ACK".encode(), senderAddr) + # print(packet.id) + + # if self.num_loaded > 0 and packet.id != 0: + # self.sock.sendto(f"ERR".encode(), senderAddr) + + if (packet.id == 0 or packet.frame_num != self.currentFrameNum): + self.packets: list[PacketHeader|None] = [None] * NUM_MAX_FRAGMENTS # Reset packets + self.num_loaded = 0 + self.currentFrameNum = packet.frame_num + self.rawFullDataBuffer[:] = 0 + # print(f"Reset frame capture. total packets: {packet.totalPackets}") + + # if self.packets[0] is not None: + # print(f"Got packet id: {packet.id} (total: {self.packets[0].totalPackets} loaded: {self.num_loaded})") + + if (self.packets is not None + and self.currentFrameNum == packet.frame_num + and self.packets[0] is not None + and not packet in self.packets + or packet.id == 0): + self.num_loaded += 1 + self.packets[packet.id] = packet + self.saveDataToBuff(bufferView, packet) + + if (packet.id < len(self.packets) and self.packets[0] is not None and self.num_loaded >= self.packets[0].totalPackets + or self.num_loaded >= NUM_MAX_FRAGMENTS): + # if self.packets[0] is not None: + # formatted_list = "[" + ", ".join(f"{RED}x{RESET}" if item is None else f"{GREEN}x{RESET}" for item in self.packets[:self.packets[0].totalPackets]) + "]" + # print(formatted_list) + + self.process_and_push_image() + + self.num_loaded = 0 + self.packets: list[PacketHeader|None] = [None] * NUM_MAX_FRAGMENTS # Reset packets + self.rawFullDataBuffer[:] = 0 + + + def process_and_push_image(self): + image = cv2.imdecode(self.rawFullDataBuffer, cv2.IMREAD_UNCHANGED) + if image is None: + print(f"{Fore.YELLOW}[WARN] Frame drop. Corrupted JPEG.{Fore.RESET}") + return + + # print("Frame") + + self.camera_status = CameraState.CONNECTED + + current_frame_time = time.time() + delta_time = current_frame_time - self.last_frame_time + self.last_frame_time = current_frame_time + + # Avoid division by zero + if delta_time > 0: + fps = 1.0 / delta_time + else: + fps = 0 + + # Smooth the FPS using a moving average + self.fl.append(fps) + if len(self.fl) > 60: + self.fl.pop(0) # Keep the list length constant + + # Compute average FPS + self.fps = sum(self.fl) / len(self.fl) + + # Compute bandwidth per second (bps) + self.bps = image.nbytes * self.fps + + # Increment frame count + self.frame_number += 1 + + # Push the frame to queue + self.push_image_to_queue(image, self.frame_number, self.fps) + diff --git a/EyeTrackApp/EBPD_LEFT.png b/EyeTrackApp/EBPD_LEFT.png new file mode 100644 index 0000000..f6376af Binary files /dev/null and b/EyeTrackApp/EBPD_LEFT.png differ diff --git a/EyeTrackApp/camera.py b/EyeTrackApp/camera.py deleted file mode 100644 index 60947a1..0000000 --- a/EyeTrackApp/camera.py +++ /dev/null @@ -1,341 +0,0 @@ -""" ------------------------------------------------------------------------------------------------------- - - ,@@@@@@ - @@@@@@@@@@@ @@@ - @@@@@@@@@@@@ @@@@@@@@@@@ - @@@@@@@@@@@@@ @@@@@@@@@@@@@@ - @@@@@@@/ ,@@@@@@@@@@@@@ - /@@@@@@@@@@@@@@@ @@@@@@@@ - @@@@@@@@@@@@@@@@@@@@@@@@ @@@@@ - @@@@@@@@ @@@@@ - ,@@@ @@@@& - @@@@@@. @@@@ - @@@ @@@@@@@@@/ @@@@@ - ,@@@. @@@@@@((@ @@@@( - //@@@ ,, @@@@ @@@@@ - @@@( @@@@@@@ - @@@ @ @@@@@@@@# - @@@@@@@@@@@@@@@@@ - @@@@@@@@@@@@@( - -Copyright (c) 2025 EyeTrackVR <3 -LICENSE: Babble Software Distribution License 1.0 ------------------------------------------------------------------------------------------------------- -""" - -import cv2 -import numpy as np -import queue -import serial -import serial.tools.list_ports -import threading -import time -from colorama import Fore -from config import EyeTrackCameraConfig -from enum import Enum -import psutil, os -import sys - - -process = psutil.Process(os.getpid()) # set process priority to low -try: - sys.getwindowsversion() -except AttributeError: - process.nice(10) # UNIX: 0 low 10 high - process.nice() -else: - process.nice(psutil.HIGH_PRIORITY_CLASS) # Windows - process.nice() - # See https://learn.microsoft.com/en-us/windows/win32/api/processthreadsapi/nf-processthreadsapi-getpriorityclass#return-value for values - -WAIT_TIME = 0.1 -# Serial communication protocol: -# header-begin (2 bytes) -# header-type (2 bytes) -# packet-size (2 bytes) -# packet (packet-size bytes) -ETVR_HEADER = b"\xff\xa0" -ETVR_HEADER_FRAME = b"\xff\xa1" -ETVR_HEADER_LEN = 6 - - -class CameraState(Enum): - CONNECTING = 0 - CONNECTED = 1 - DISCONNECTED = 2 - - -def is_serial_capture_source(addr: str) -> bool: - """ - Returns True if the capture source address is a serial port. - """ - return ( - addr.startswith("COM") or addr.startswith("/dev/cu") or addr.startswith("/dev/tty") # Windows # macOS # Linux - ) - - -class Camera: - def __init__( - self, - config: EyeTrackCameraConfig, - camera_index: int, - cancellation_event: "threading.Event", - capture_event: "threading.Event", - camera_status_outgoing: "queue.Queue[CameraState]", - camera_output_outgoing: "queue.Queue(maxsize=20)", - ): - - self.camera_status = CameraState.CONNECTING - self.config = config - self.camera_index = camera_index - self.camera_address = config.capture_source - self.camera_status_outgoing = camera_status_outgoing - self.camera_output_outgoing = camera_output_outgoing - self.capture_event = capture_event - self.cancellation_event = cancellation_event - self.current_capture_source = config.capture_source - self.cv2_camera: "cv2.VideoCapture" = None - - self.serial_connection = None - self.last_frame_time = time.time() - self.frame_number = 0 - self.fps = 0 - self.bps = 0 - self.start = True - self.buffer = b"" - self.pf_fps = 0 - self.prevft = 0 - self.newft = 0 - self.fl = [0] - - - - self.error_message = f"{Fore.YELLOW}[WARN] Capture source {{}} not found, retrying...{Fore.RESET}" - - def __del__(self): - if self.serial_connection is not None: - self.serial_connection.close() - - def set_output_queue(self, camera_output_outgoing: "queue.Queue"): - self.camera_output_outgoing = camera_output_outgoing - - def run(self): - OPENCV_PARAMS = [ - cv2.CAP_PROP_OPEN_TIMEOUT_MSEC, - 5000, - cv2.CAP_PROP_READ_TIMEOUT_MSEC, - 5000, - ] - while True: - if self.cancellation_event.is_set(): - print(f"{Fore.CYAN}[INFO] Exiting Capture thread{Fore.RESET}") - # openCV won't switch to a new source if provided with one - # so, we have to manually release the camera on exit - - addr = str(self.current_capture_source) - if is_serial_capture_source(addr): - pass # TODO: find a nicer way to stop the com port - # self.serial_connection.close() - else: - self.cv2_camera.release() - - return - should_push = True - # If things aren't open, retry until they are. Don't let read requests come in any earlier - # than this, otherwise we can deadlock ourselves. - if self.config.capture_source != None and self.config.capture_source != "": - self.current_capture_source = self.config.capture_source - addr = str(self.current_capture_source) - if is_serial_capture_source(addr): - if ( - self.serial_connection is None - or self.camera_status == CameraState.DISCONNECTED - or self.config.capture_source != self.current_capture_source - ): - port = self.config.capture_source - self.current_capture_source = port - self.start_serial_connection(port) - else: - if ( - self.cv2_camera is None - or not self.cv2_camera.isOpened() - or self.camera_status == CameraState.DISCONNECTED - or self.config.capture_source != self.current_capture_source - ): - print(self.error_message.format(self.config.capture_source)) - # This requires a wait, otherwise we can error and possible screw up the camera - # firmware. Fickle things. - if self.cancellation_event.wait(WAIT_TIME): - return - self.current_capture_source = self.config.capture_source - # self.cv2_camera = cv2.VideoCapture(self.current_capture_source) - - self.cv2_camera = cv2.VideoCapture() - self.cv2_camera.setExceptionMode(True) - # https://github.com/opencv/opencv/blob/4.8.0/modules/videoio/include/opencv2/videoio.hpp#L803 - self.cv2_camera.open(self.current_capture_source) - should_push = False - else: - # We don't have a capture source to try yet, wait for one to show up in the GUI. - if self.cancellation_event.wait(WAIT_TIME): - self.camera_status = CameraState.DISCONNECTED - return - # Assuming we can access our capture source, wait for another thread to request a capture. - # Cycle every so often to see if our cancellation token has fired. This basically uses a - # python event as a context-less, resettable one-shot channel. - if should_push and not self.capture_event.wait(timeout=0.001): - continue - if self.config.capture_source != None: - addr = str(self.current_capture_source) - if is_serial_capture_source(addr): - self.get_serial_camera_picture(should_push) - else: - self.get_cv2_camera_picture(should_push) - if not should_push: - # if we get all the way down here, consider ourselves connected - self.camera_status = CameraState.CONNECTED - - def get_cv2_camera_picture(self, should_push): - try: - ret, image = self.cv2_camera.read() - height, width = image.shape[:2] # Calculate the aspect ratio - if int(width) > 680: - aspect_ratio = float(width) / float( - height - ) # Determine the new height based on the desired maximum width - new_height = int(680 / aspect_ratio) - image = cv2.resize(image, (680, new_height)) - if not ret: - self.cv2_camera.set(cv2.CAP_PROP_POS_FRAMES, 0) - raise RuntimeError("Problem while getting frame") - frame_number = self.cv2_camera.get(cv2.CAP_PROP_POS_FRAMES) - current_frame_time = time.time() - delta_time = current_frame_time - self.last_frame_time - if delta_time > 0: - current_fps = 1 / delta_time - else: - current_fps = 0 - self.last_frame_time = current_frame_time - - if len(self.fl) < 60: - self.fl.append(current_fps) - else: - self.fl.pop(0) - self.fl.append(current_fps) - - self.fps = sum(self.fl) / len(self.fl) - self.bps = image.nbytes * self.fps - - - if should_push: - self.push_image_to_queue(image, frame_number, self.fps) - except: - print( - f"{Fore.YELLOW}[WARN] Capture source problem, assuming camera disconnected, waiting for reconnect.{Fore.RESET}" - ) - self.camera_status = CameraState.DISCONNECTED - pass - - def get_next_packet_bounds(self): - beg = -1 - while beg == -1: - self.buffer += self.serial_connection.read(2048) - beg = self.buffer.find(ETVR_HEADER + ETVR_HEADER_FRAME) - # Discard any data before the frame header. - if beg > 0: - self.buffer = self.buffer[beg:] - beg = 0 - # We know exactly how long the jpeg packet is - end = int.from_bytes(self.buffer[4:6], signed=False, byteorder="little") - self.buffer += self.serial_connection.read(end - len(self.buffer)) - return beg, end - - def get_next_jpeg_frame(self): - beg, end = self.get_next_packet_bounds() - jpeg = self.buffer[beg + ETVR_HEADER_LEN : end + ETVR_HEADER_LEN] - self.buffer = self.buffer[end + ETVR_HEADER_LEN :] - return jpeg - - def get_serial_camera_picture(self, should_push): - conn = self.serial_connection - if conn is None: - return - try: - if conn.in_waiting: - jpeg = self.get_next_jpeg_frame() - if jpeg: - # Create jpeg frame from byte string - image = cv2.imdecode(np.fromstring(jpeg, dtype=np.uint8), cv2.IMREAD_UNCHANGED) - if image is None: - print(f"{Fore.YELLOW}[WARN] Frame drop. Corrupted JPEG.{Fore.RESET}") - return - # Discard the serial buffer. This is due to the fact that it - # may build up some outdated frames. A bit of a workaround here tbh. - if conn.in_waiting >= 32768: - print(f"{Fore.CYAN}[INFO] Discarding the serial buffer ({conn.in_waiting} bytes){Fore.RESET}") - conn.reset_input_buffer() - self.buffer = b"" - # Calculate the fps. - current_frame_time = time.time() - delta_time = current_frame_time - self.last_frame_time - self.last_frame_time = current_frame_time - self.fps = (self.fps + self.pf_fps) / 2 - self.newft = time.time() - self.fps = 1 / (self.newft - self.prevft) - self.prevft = self.newft - self.fps = int(self.fps) - if len(self.fl) < 60: - self.fl.append(self.fps) - else: - self.fl.pop(0) - self.fl.append(self.fps) - self.fps = sum(self.fl) / len(self.fl) - self.bps = image.nbytes * self.fps - self.frame_number = self.frame_number + 1 - if should_push: - self.push_image_to_queue(image, self.frame_number, self.fps) - except Exception: - print( - f"{Fore.YELLOW}[WARN] Serial capture source problem, assuming camera disconnected, waiting for reconnect.{Fore.RESET}" - ) - conn.close() - self.camera_status = CameraState.DISCONNECTED - pass - - def start_serial_connection(self, port): - if self.serial_connection is not None and self.serial_connection.is_open: - # Do nothing. The connection is already open on this port. - if self.serial_connection.port == port: - return - # Otherwise, close the connection before trying to reopen. - self.serial_connection.close() - com_ports = [tuple(p) for p in list(serial.tools.list_ports.comports())] - # Do not try connecting if no such port i.e. device was unplugged. - if not any(p for p in com_ports if port in p): - return - try: - rate = 115200 if sys.platform == "darwin" else 3000000 # Higher baud rate not working on macOS - conn = serial.Serial(baudrate=rate, port=port, xonxoff=False, dsrdtr=False, rtscts=False) - # Set explicit buffer size for serial. - if sys.platform == "win32": - buffer_size = 32768 - conn.set_buffer_size(rx_size=buffer_size, tx_size=buffer_size) - - print(f"{Fore.CYAN}[INFO] ETVR Serial Tracker device connected on {port}{Fore.RESET}") - self.serial_connection = conn - self.camera_status = CameraState.CONNECTED - except Exception: - print(f"{Fore.CYAN}[INFO] Failed to connect on {port}{Fore.RESET}") - self.camera_status = CameraState.DISCONNECTED - - def push_image_to_queue(self, image, frame_number, fps): - # If there's backpressure, just yell. We really shouldn't have this unless we start getting - # some sort of capture event conflict though. - qsize = self.camera_output_outgoing.qsize() - if qsize > 1: - print( - f"{Fore.YELLOW}[WARN] CAPTURE QUEUE BACKPRESSURE OF {qsize}. CHECK FOR CRASH OR TIMING ISSUES IN ALGORITHM.{Fore.RESET}" - ) - self.camera_output_outgoing.put((image, frame_number, fps)) - self.capture_event.clear() diff --git a/EyeTrackApp/camera_widget.py b/EyeTrackApp/camera_widget.py index a8bf982..a59e963 100644 --- a/EyeTrackApp/camera_widget.py +++ b/EyeTrackApp/camera_widget.py @@ -32,7 +32,8 @@ import math from eye import EyeId from eye_processor import EyeProcessor, EyeInfoOrigin from queue import Queue, Empty -from camera import Camera, CameraState +from Camera.CameraState import CameraState +from Camera.CameraFactory import CameraFactory import cv2 from osc.OSCMessage import OSCMessageType, OSCMessage from utils.misc_utils import PlaySound, SND_FILENAME, SND_ASYNC, resource_path @@ -104,14 +105,16 @@ class CameraWidget: ) self.camera_status_queue = Queue() - self.camera = Camera( - self.config, - 0, - self.cancellation_event, - self.capture_event, - self.camera_status_queue, - self.capture_queue, - ) + + if not self.config.capture_source is None: + self.camera = CameraFactory.get_camera_from_string_type(self.config.capture_source)( + self.config, + 0, + self.cancellation_event, + self.capture_event, + self.camera_status_queue, + self.capture_queue, + ) self.hover = None