mirror of
https://github.com/EyeTrackVR/EyeTrackVR.git
synced 2025-11-04 14:39:42 +08:00
feat: 3d calib beginnings, fix build crash, new LEAP model
This commit is contained in:
parent
7dae037dd5
commit
c4f1bb116e
41415
EyeTrackApp/Models/LEAP06212024.onnx
Normal file
41415
EyeTrackApp/Models/LEAP06212024.onnx
Normal file
File diff suppressed because it is too large
Load Diff
@ -373,6 +373,7 @@ class CameraWidget:
|
|||||||
|
|
||||||
if event == self.gui_restart_3d_calibration:
|
if event == self.gui_restart_3d_calibration:
|
||||||
self.ransac.calibration_3d_frame_counter = -621
|
self.ransac.calibration_3d_frame_counter = -621
|
||||||
|
self.settings.gui_3d_calibration = True
|
||||||
self.ransac.ibo.clear_filter()
|
self.ransac.ibo.clear_filter()
|
||||||
PlaySound(resource_path("Audio/start.wav"), SND_FILENAME | SND_ASYNC)
|
PlaySound(resource_path("Audio/start.wav"), SND_FILENAME | SND_ASYNC)
|
||||||
|
|
||||||
|
|||||||
@ -80,6 +80,8 @@ class EyeTrackSettingsConfig(BaseModel):
|
|||||||
gui_blob_maxsize: float = 25
|
gui_blob_maxsize: float = 25
|
||||||
gui_blob_minsize: float = 10
|
gui_blob_minsize: float = 10
|
||||||
gui_recenter_eyes: bool = False
|
gui_recenter_eyes: bool = False
|
||||||
|
gui_3d_calibration: bool = False
|
||||||
|
grab_3d_point: bool = False
|
||||||
tracker_single_eye: int = 0
|
tracker_single_eye: int = 0
|
||||||
gui_threshold: int = 65
|
gui_threshold: int = 65
|
||||||
gui_AHSFRACP: int = 1
|
gui_AHSFRACP: int = 1
|
||||||
|
|||||||
@ -104,6 +104,8 @@ class EyeProcessor:
|
|||||||
self.eye_id = eye_id
|
self.eye_id = eye_id
|
||||||
self.baseconfig = baseconfig
|
self.baseconfig = baseconfig
|
||||||
self.filterlist = []
|
self.filterlist = []
|
||||||
|
self.left_eye_data = [(0.351, 0.399, 1), (0.352, 0.400, 1)] # Example data
|
||||||
|
self.right_eye_data = [(0.351, 0.399, 1), (0.352, 0.400, 1)] # Example data
|
||||||
|
|
||||||
# Cross algo state
|
# Cross algo state
|
||||||
self.lkg_projected_sphere = None
|
self.lkg_projected_sphere = None
|
||||||
|
|||||||
@ -11,7 +11,7 @@ a = Analysis(
|
|||||||
pathex=[],
|
pathex=[],
|
||||||
binaries=[],
|
binaries=[],
|
||||||
datas=resources,
|
datas=resources,
|
||||||
hiddenimports=['cv2', 'numpy', 'PySimpleGui'],
|
hiddenimports=['cv2', 'numpy', 'PySimpleGui', 'pkg_resources.extern'],
|
||||||
hookspath=[],
|
hookspath=[],
|
||||||
hooksconfig={},
|
hooksconfig={},
|
||||||
runtime_hooks=[],
|
runtime_hooks=[],
|
||||||
|
|||||||
@ -51,7 +51,7 @@ class LEAP_C(object):
|
|||||||
# Config variables
|
# Config variables
|
||||||
self.num_threads = 1 # Number of python threads to use (using ~1 more than needed to achieve wanted fps yields lower cpu usage)
|
self.num_threads = 1 # Number of python threads to use (using ~1 more than needed to achieve wanted fps yields lower cpu usage)
|
||||||
self.queue_max_size = 1 # Optimize for best CPU usage, Memory, and Latency. A maxsize is needed to not create a potential memory leak.
|
self.queue_max_size = 1 # Optimize for best CPU usage, Memory, and Latency. A maxsize is needed to not create a potential memory leak.
|
||||||
self.model_path = resource_path(models / 'LEAP053024.onnx')
|
self.model_path = resource_path(models / 'LEAP06212024.onnx')
|
||||||
|
|
||||||
self.low_priority = (
|
self.low_priority = (
|
||||||
False # set process priority to low (may cause issues when unfocusing? reported by one, not reproducable)
|
False # set process priority to low (may cause issues when unfocusing? reported by one, not reproducable)
|
||||||
|
|||||||
@ -35,7 +35,7 @@ import threading
|
|||||||
import os
|
import os
|
||||||
import subprocess
|
import subprocess
|
||||||
import math
|
import math
|
||||||
|
from utils.calibration_3d import receive_calibration_data
|
||||||
|
|
||||||
class TimeoutError(RuntimeError):
|
class TimeoutError(RuntimeError):
|
||||||
pass
|
pass
|
||||||
@ -108,6 +108,9 @@ class var:
|
|||||||
single_eye = True
|
single_eye = True
|
||||||
left_enb = 0
|
left_enb = 0
|
||||||
right_enb = 0
|
right_enb = 0
|
||||||
|
eye_wait = 10
|
||||||
|
left_calib = False
|
||||||
|
right_calib = False
|
||||||
|
|
||||||
|
|
||||||
@Async
|
@Async
|
||||||
@ -152,9 +155,12 @@ def overlay_calibrate_3d(self):
|
|||||||
received_int = struct.unpack("!l", data)[0]
|
received_int = struct.unpack("!l", data)[0]
|
||||||
message = received_int
|
message = received_int
|
||||||
self.settings.gui_recenter_eyes = False
|
self.settings.gui_recenter_eyes = False
|
||||||
self.grab_3d_point = True
|
self.settings.grab_3d_point = True
|
||||||
|
|
||||||
print(message)
|
print(message)
|
||||||
|
if message == 8:
|
||||||
|
var.overlay_active = False
|
||||||
|
|
||||||
except:
|
except:
|
||||||
print("[WARN] Calibration overlay error. Make sure SteamVR is Running.")
|
print("[WARN] Calibration overlay error. Make sure SteamVR is Running.")
|
||||||
self.settings.gui_recenter_eyes = False
|
self.settings.gui_recenter_eyes = False
|
||||||
@ -177,18 +183,42 @@ class cal:
|
|||||||
flipx = self.settings.gui_flip_x_axis_right
|
flipx = self.settings.gui_flip_x_axis_right
|
||||||
else:
|
else:
|
||||||
flipx = self.settings.gui_flip_x_axis_left
|
flipx = self.settings.gui_flip_x_axis_left
|
||||||
if self.calibration_3d_frame_counter == -621:
|
if self.calibration_3d_frame_counter == -621: #or self.settings.gui_3d_calibration:
|
||||||
|
|
||||||
self.calibration_3d_frame_counter = self.calibration_3d_frame_counter - 1
|
self.calibration_3d_frame_counter = self.calibration_3d_frame_counter - 1
|
||||||
overlay_calibrate_3d(self)
|
overlay_calibrate_3d(self)
|
||||||
print("yippe")
|
|
||||||
|
|
||||||
if self.grab_3d_point:
|
# print(self.eye_id, cx, cy)
|
||||||
self.grab_3d_point = False
|
# self.settings.gui_3d_calibration = False
|
||||||
|
|
||||||
self.config.calibration_points.append((cx, cy))
|
if self.settings.grab_3d_point:
|
||||||
print(self.config.calibration_points, self.eye_id)
|
# Check if both calibrations are done
|
||||||
|
if var.left_calib and var.right_calib:
|
||||||
|
self.settings.grab_3d_point = False
|
||||||
|
var.left_calib = False
|
||||||
|
var.right_calib = False
|
||||||
|
print('end')
|
||||||
|
else:
|
||||||
|
# Check if it's the left eye and left calibration is not done yet
|
||||||
|
if self.eye_id == EyeId.LEFT and not var.left_calib:
|
||||||
|
var.left_calib = True
|
||||||
|
self.config.calibration_points.append((cx, cy, 1))
|
||||||
|
# Check if it's the right eye and right calibration is not done yet
|
||||||
|
elif self.eye_id == EyeId.RIGHT and not var.right_calib:
|
||||||
|
var.right_calib = True
|
||||||
|
self.config.calibration_points.append((cx, cy, 0))
|
||||||
|
|
||||||
# print("calib")
|
|
||||||
|
if self.eye_id == EyeId.LEFT and len(self.config.calibration_points) == 8 and var.left_calib == False:
|
||||||
|
var.left_calib = True
|
||||||
|
receive_calibration_data(self.config.calibration_points, self.eye_id)
|
||||||
|
print('SENT LEFT EYE POINTS')
|
||||||
|
|
||||||
|
if self.eye_id == EyeId.RIGHT and len(self.config.calibration_points) == 8 and var.right_calib == False:
|
||||||
|
var.right_calib = True
|
||||||
|
receive_calibration_data(self.config.calibration_points, self.eye_id)
|
||||||
|
print('SENT RIGHT EYE POINTS')
|
||||||
|
# print(len(self.config.calibration_points), self.eye_id)
|
||||||
|
|
||||||
if self.calibration_frame_counter == 0:
|
if self.calibration_frame_counter == 0:
|
||||||
self.calibration_frame_counter = None
|
self.calibration_frame_counter = None
|
||||||
@ -217,6 +247,8 @@ class cal:
|
|||||||
|
|
||||||
self.calibration_frame_counter -= 1
|
self.calibration_frame_counter -= 1
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
if self.settings.gui_recenter_eyes == True:
|
if self.settings.gui_recenter_eyes == True:
|
||||||
self.config.calib_XOFF = cx
|
self.config.calib_XOFF = cx
|
||||||
self.config.calib_YOFF = cy
|
self.config.calib_YOFF = cy
|
||||||
|
|||||||
@ -1,92 +0,0 @@
|
|||||||
import numpy as np
|
|
||||||
import socket
|
|
||||||
import threading
|
|
||||||
import os
|
|
||||||
|
|
||||||
class TimeoutError(RuntimeError):
|
|
||||||
pass
|
|
||||||
|
|
||||||
|
|
||||||
class AsyncCall(object):
|
|
||||||
def __init__(self, fnc, callback=None):
|
|
||||||
self.Callable = fnc
|
|
||||||
self.Callback = callback
|
|
||||||
|
|
||||||
def __call__(self, *args, **kwargs):
|
|
||||||
self.Thread = threading.Thread(target=self.run, name=self.Callable.__name__, args=args, kwargs=kwargs)
|
|
||||||
self.Thread.start()
|
|
||||||
return self
|
|
||||||
|
|
||||||
def wait(self, timeout=None):
|
|
||||||
self.Thread.join(timeout)
|
|
||||||
if self.Thread.isAlive():
|
|
||||||
raise TimeoutError()
|
|
||||||
else:
|
|
||||||
return self.Result
|
|
||||||
|
|
||||||
def run(self, *args, **kwargs):
|
|
||||||
self.Result = self.Callable(*args, **kwargs)
|
|
||||||
if self.Callback:
|
|
||||||
self.Callback(self.Result)
|
|
||||||
|
|
||||||
|
|
||||||
class AsyncMethod(object):
|
|
||||||
def __init__(self, fnc, callback=None):
|
|
||||||
self.Callable = fnc
|
|
||||||
self.Callback = callback
|
|
||||||
|
|
||||||
def __call__(self, *args, **kwargs):
|
|
||||||
return AsyncCall(self.Callable, self.Callback)(*args, **kwargs)
|
|
||||||
|
|
||||||
|
|
||||||
def Async(fnc=None, callback=None):
|
|
||||||
if fnc == None:
|
|
||||||
|
|
||||||
def AddAsyncCallback(fnc):
|
|
||||||
return AsyncMethod(fnc, callback)
|
|
||||||
|
|
||||||
return AddAsyncCallback
|
|
||||||
else:
|
|
||||||
return AsyncMethod(fnc, callback)
|
|
||||||
|
|
||||||
|
|
||||||
@Async
|
|
||||||
def overlay_calibrate_3d(self):
|
|
||||||
try:
|
|
||||||
if self.overlay_active != True:
|
|
||||||
dirname = os.getcwd()
|
|
||||||
overlay_path = os.path.join(dirname, "calibrate.bat") # start overlay
|
|
||||||
os.startfile(overlay_path)
|
|
||||||
self.overlay_active = True
|
|
||||||
while var.overlay_active:
|
|
||||||
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
|
|
||||||
server_address = ("localhost", 2112)
|
|
||||||
sock.bind(server_address)
|
|
||||||
data, address = sock.recvfrom(4096)
|
|
||||||
received_int = struct.unpack("!l", data)[0] # this will hold until message received
|
|
||||||
self.point_trigger = True
|
|
||||||
self.calib_step = reveived_int
|
|
||||||
|
|
||||||
print(received_int)
|
|
||||||
except:
|
|
||||||
print("[WARN] Calibration overlay error. Make sure SteamVR is Running.")
|
|
||||||
self.settings.gui_recenter_eyes = False
|
|
||||||
var.overlay_active = False
|
|
||||||
|
|
||||||
|
|
||||||
class calibrate_3d:
|
|
||||||
overlay_active = False
|
|
||||||
point_trigger = False
|
|
||||||
calib_step = 0
|
|
||||||
calib_matrix = np.zeros((9, 2))
|
|
||||||
|
|
||||||
def nine_point_3d_calib(x, y): # dummy format
|
|
||||||
|
|
||||||
if calibrate_3d.point_trigger:
|
|
||||||
calibrate_3d.calib_matrix[calibrate_3d.calib_step][0] = x
|
|
||||||
calibrate_3d.calib_matrix[calibrate_3d.calib_step][1] = y
|
|
||||||
print(calibrate_3d.calib_matrix)
|
|
||||||
calibrate_3d.point_trigger = True
|
|
||||||
|
|
||||||
|
|
||||||
calibrate_3d.nine_point_3d_calib(1, 2)
|
|
||||||
38
EyeTrackApp/utils/calibration_3d.py
Normal file
38
EyeTrackApp/utils/calibration_3d.py
Normal file
@ -0,0 +1,38 @@
|
|||||||
|
# calibration_module.py
|
||||||
|
|
||||||
|
class CalibrationProcessor:
|
||||||
|
def __init__(self):
|
||||||
|
self.left_eye_data = None
|
||||||
|
self.right_eye_data = None
|
||||||
|
|
||||||
|
def receive_calibration_data(self, eye_id, data):
|
||||||
|
if eye_id == 'left':
|
||||||
|
self.left_eye_data = data
|
||||||
|
elif eye_id == 'right':
|
||||||
|
self.right_eye_data = data
|
||||||
|
|
||||||
|
# Check if both sets of data have been received
|
||||||
|
if self.left_eye_data is not None and self.right_eye_data is not None:
|
||||||
|
self.process_calibration_data()
|
||||||
|
|
||||||
|
def process_calibration_data(self):
|
||||||
|
# Ensure both data are present
|
||||||
|
if self.left_eye_data is None or self.right_eye_data is None:
|
||||||
|
raise ValueError("Calibration data for both eyes must be provided")
|
||||||
|
|
||||||
|
# Example computation with both calibration arrays
|
||||||
|
# Replace with your actual computation logic
|
||||||
|
print("Processing calibration data for both eyes...")
|
||||||
|
print(f"Left Eye Data: {self.left_eye_data}")
|
||||||
|
print(f"Right Eye Data: {self.right_eye_data}")
|
||||||
|
|
||||||
|
# After processing, reset the data
|
||||||
|
self.left_eye_data = None
|
||||||
|
self.right_eye_data = None
|
||||||
|
|
||||||
|
# Global instance of CalibrationProcessor
|
||||||
|
calibration_processor = CalibrationProcessor()
|
||||||
|
|
||||||
|
def receive_calibration_data(eye_id, data):
|
||||||
|
global calibration_processor
|
||||||
|
calibration_processor.receive_calibration_data(eye_id, data)
|
||||||
Loading…
Reference in New Issue
Block a user