mirror of
https://github.com/EyeTrackVR/EyeTrackVR.git
synced 2025-11-04 14:39:42 +08:00
fix: identify and mitigate latency issues
This commit is contained in:
parent
b3f444ee3a
commit
295f10476d
@ -18,6 +18,11 @@ class CameraState(Enum):
|
|||||||
DISCONNECTED = 2
|
DISCONNECTED = 2
|
||||||
|
|
||||||
|
|
||||||
|
# Find JPEG frames based on start and end bytes.
|
||||||
|
FRAME_JPEG_BEGIN = b'\xff\xd8'
|
||||||
|
FRAME_JPEG_END = b'\xff\xd9'
|
||||||
|
|
||||||
|
|
||||||
class Camera:
|
class Camera:
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
@ -43,7 +48,7 @@ class Camera:
|
|||||||
self.frame_number = 0
|
self.frame_number = 0
|
||||||
self.fps = 0
|
self.fps = 0
|
||||||
self.start = True
|
self.start = True
|
||||||
self.serialByteBuffer = b''
|
self.buffer = b''
|
||||||
|
|
||||||
self.error_message = "\033[93m[WARN] Capture source {} not found, retrying...\033[0m"
|
self.error_message = "\033[93m[WARN] Capture source {} not found, retrying...\033[0m"
|
||||||
|
|
||||||
@ -120,43 +125,58 @@ class Camera:
|
|||||||
self.camera_status = CameraState.DISCONNECTED
|
self.camera_status = CameraState.DISCONNECTED
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
def get_next_frame_bounds(self):
|
||||||
|
beg = -1
|
||||||
|
end = -1
|
||||||
|
# Initial read.
|
||||||
|
self.buffer += self.serial_connection.read(4096)
|
||||||
|
while beg == -1 or end == -1:
|
||||||
|
# Find the jpeg begin bytes.
|
||||||
|
if beg == -1:
|
||||||
|
beg = self.buffer.find(FRAME_JPEG_BEGIN)
|
||||||
|
continue
|
||||||
|
# Discard any data before the jpeg frame.
|
||||||
|
if beg > 0:
|
||||||
|
self.buffer = self.buffer[beg:]
|
||||||
|
beg = 0
|
||||||
|
continue
|
||||||
|
# Find the jpeg end bytes.
|
||||||
|
end = self.buffer.find(FRAME_JPEG_END)
|
||||||
|
# Read more data when no jpeg end bytes found.
|
||||||
|
if end == -1:
|
||||||
|
self.buffer += self.serial_connection.read(2048)
|
||||||
|
return beg, end
|
||||||
|
|
||||||
|
def get_next_jpeg_frame(self):
|
||||||
|
beg, end = self.get_next_frame_bounds()
|
||||||
|
# Extract jpeg from the buffer.
|
||||||
|
jpeg = self.buffer[beg:end+2]
|
||||||
|
# Remove jpeg from the buffer.
|
||||||
|
self.buffer = self.buffer[end+2:]
|
||||||
|
return jpeg
|
||||||
|
|
||||||
def get_serial_camera_picture(self, should_push):
|
def get_serial_camera_picture(self, should_push):
|
||||||
start = time.time()
|
start = time.time()
|
||||||
try:
|
try:
|
||||||
bytes = self.serialByteBuffer
|
|
||||||
if self.serial_connection.in_waiting:
|
if self.serial_connection.in_waiting:
|
||||||
bytes += self.serial_connection.read(4096) # Read in initial bytes
|
jpeg = self.get_next_jpeg_frame()
|
||||||
|
if jpeg:
|
||||||
a = bytes.find(b'\xff\xd8') # Find start byte for jpeg image
|
# Create jpeg frame from byte string
|
||||||
b = bytes.find(b'\xff\xd9') # Fine end byte for jpeg image
|
image = cv2.imdecode(np.fromstring(jpeg, dtype=np.uint8), cv2.IMREAD_UNCHANGED)
|
||||||
|
if image is None:
|
||||||
# If the first found end byte is before the start byte, keep reading in serial
|
print("image not found")
|
||||||
# data and discarding the old data until the start byte is before the end byte
|
return
|
||||||
while a > b:
|
# Discard the serial buffer. This is due to the fact that it
|
||||||
bytes = bytes[a:]
|
# may build up some outdated frames. A bit of a workaround here tbh.
|
||||||
a = bytes.find(b'\xff\xd8')
|
self.serial_connection.reset_input_buffer()
|
||||||
b = bytes.find(b'\xff\xd9')
|
self.buffer = b''
|
||||||
if a == -1 or b == -1:
|
# Calculate the fps.
|
||||||
bytes += self.serial_connection.read(2048)
|
delta_time = time.time() - start
|
||||||
|
if delta_time > 0:
|
||||||
if a != -1 and b != -1: # If there is jpeg data
|
self.fps = 1 / delta_time
|
||||||
jpg = bytes[a:b + 2] # Create the string of bytes for the current jpeg
|
self.frame_number = self.frame_number + 1
|
||||||
bytes = bytes[b + 2:] # Clear the buffer until the end of our current jpeg
|
if should_push:
|
||||||
self.serialByteBuffer = bytes
|
self.push_image_to_queue(image, self.frame_number, self.fps)
|
||||||
|
|
||||||
if jpg:
|
|
||||||
# Create jpeg frame from byte string
|
|
||||||
image = cv2.imdecode(np.fromstring(jpg, dtype=np.uint8), cv2.IMREAD_UNCHANGED)
|
|
||||||
if image is None:
|
|
||||||
print("image not found")
|
|
||||||
return
|
|
||||||
else:
|
|
||||||
self.frame_number = self.frame_number + 1
|
|
||||||
delta_time = time.time() - start
|
|
||||||
if delta_time > 0:
|
|
||||||
self.fps = 1 / delta_time
|
|
||||||
if should_push:
|
|
||||||
self.push_image_to_queue(image, self.frame_number, self.fps)
|
|
||||||
|
|
||||||
except UnboundLocalError as ex:
|
except UnboundLocalError as ex:
|
||||||
print(ex)
|
print(ex)
|
||||||
@ -179,22 +199,25 @@ class Camera:
|
|||||||
if not any(p for p in com_ports if port in p):
|
if not any(p for p in com_ports if port in p):
|
||||||
return
|
return
|
||||||
try:
|
try:
|
||||||
conn = serial.Serial()
|
conn = serial.Serial(
|
||||||
conn.baudrate = 3000000
|
baudrate = 3000000,
|
||||||
conn.port = port
|
port = port,
|
||||||
conn.setDTR(False)
|
xonxoff=False,
|
||||||
conn.setRTS(False)
|
dsrdtr=False,
|
||||||
conn.open()
|
rtscts=False)
|
||||||
|
|
||||||
|
conn.reset_input_buffer()
|
||||||
|
|
||||||
print(f"\033[94m[INFO] Serial Tracker successfully connected on {port}\033[0m")
|
print(f"\033[94m[INFO] Serial Tracker successfully connected on {port}\033[0m")
|
||||||
self.serial_connection = conn
|
self.serial_connection = conn
|
||||||
self.camera_status = CameraState.CONNECTED
|
self.camera_status = CameraState.CONNECTED
|
||||||
except Exception as e:
|
except Exception:
|
||||||
self.camera_status = CameraState.DISCONNECTED
|
self.camera_status = CameraState.DISCONNECTED
|
||||||
|
|
||||||
def push_image_to_queue(self, image, frame_number, fps):
|
def push_image_to_queue(self, image, frame_number, fps):
|
||||||
# If there's backpressure, just yell. We really shouldn't have this unless we start getting
|
# If there's backpressure, just yell. We really shouldn't have this unless we start getting
|
||||||
# some sort of capture event conflict though.
|
# some sort of capture event conflict though.
|
||||||
|
print(f"N {frame_number} FPS {fps}")
|
||||||
qsize = self.camera_output_outgoing.qsize()
|
qsize = self.camera_output_outgoing.qsize()
|
||||||
if qsize > 1:
|
if qsize > 1:
|
||||||
print(
|
print(
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user