feat: 3d calib beginnings, fix build crash, new LEAP model

This commit is contained in:
Prohurtz 2024-06-21 08:42:37 -05:00
parent 7dae037dd5
commit c4f1bb116e
9 changed files with 41501 additions and 103 deletions

File diff suppressed because it is too large Load Diff

View File

@ -373,6 +373,7 @@ class CameraWidget:
if event == self.gui_restart_3d_calibration:
self.ransac.calibration_3d_frame_counter = -621
self.settings.gui_3d_calibration = True
self.ransac.ibo.clear_filter()
PlaySound(resource_path("Audio/start.wav"), SND_FILENAME | SND_ASYNC)

View File

@ -80,6 +80,8 @@ class EyeTrackSettingsConfig(BaseModel):
gui_blob_maxsize: float = 25
gui_blob_minsize: float = 10
gui_recenter_eyes: bool = False
gui_3d_calibration: bool = False
grab_3d_point: bool = False
tracker_single_eye: int = 0
gui_threshold: int = 65
gui_AHSFRACP: int = 1

View File

@ -104,6 +104,8 @@ class EyeProcessor:
self.eye_id = eye_id
self.baseconfig = baseconfig
self.filterlist = []
self.left_eye_data = [(0.351, 0.399, 1), (0.352, 0.400, 1)] # Example data
self.right_eye_data = [(0.351, 0.399, 1), (0.352, 0.400, 1)] # Example data
# Cross algo state
self.lkg_projected_sphere = None

View File

@ -11,7 +11,7 @@ a = Analysis(
pathex=[],
binaries=[],
datas=resources,
hiddenimports=['cv2', 'numpy', 'PySimpleGui'],
hiddenimports=['cv2', 'numpy', 'PySimpleGui', 'pkg_resources.extern'],
hookspath=[],
hooksconfig={},
runtime_hooks=[],

View File

@ -51,7 +51,7 @@ class LEAP_C(object):
# Config variables
self.num_threads = 1 # Number of python threads to use (using ~1 more than needed to achieve wanted fps yields lower cpu usage)
self.queue_max_size = 1 # Optimize for best CPU usage, Memory, and Latency. A maxsize is needed to not create a potential memory leak.
self.model_path = resource_path(models / 'LEAP053024.onnx')
self.model_path = resource_path(models / 'LEAP06212024.onnx')
self.low_priority = (
False # set process priority to low (may cause issues when unfocusing? reported by one, not reproducable)

View File

@ -35,7 +35,7 @@ import threading
import os
import subprocess
import math
from utils.calibration_3d import receive_calibration_data
class TimeoutError(RuntimeError):
pass
@ -108,6 +108,9 @@ class var:
single_eye = True
left_enb = 0
right_enb = 0
eye_wait = 10
left_calib = False
right_calib = False
@Async
@ -152,9 +155,12 @@ def overlay_calibrate_3d(self):
received_int = struct.unpack("!l", data)[0]
message = received_int
self.settings.gui_recenter_eyes = False
self.grab_3d_point = True
self.settings.grab_3d_point = True
print(message)
if message == 8:
var.overlay_active = False
except:
print("[WARN] Calibration overlay error. Make sure SteamVR is Running.")
self.settings.gui_recenter_eyes = False
@ -177,18 +183,42 @@ class cal:
flipx = self.settings.gui_flip_x_axis_right
else:
flipx = self.settings.gui_flip_x_axis_left
if self.calibration_3d_frame_counter == -621:
if self.calibration_3d_frame_counter == -621: #or self.settings.gui_3d_calibration:
self.calibration_3d_frame_counter = self.calibration_3d_frame_counter - 1
overlay_calibrate_3d(self)
print("yippe")
if self.grab_3d_point:
self.grab_3d_point = False
# print(self.eye_id, cx, cy)
# self.settings.gui_3d_calibration = False
self.config.calibration_points.append((cx, cy))
print(self.config.calibration_points, self.eye_id)
if self.settings.grab_3d_point:
# Check if both calibrations are done
if var.left_calib and var.right_calib:
self.settings.grab_3d_point = False
var.left_calib = False
var.right_calib = False
print('end')
else:
# Check if it's the left eye and left calibration is not done yet
if self.eye_id == EyeId.LEFT and not var.left_calib:
var.left_calib = True
self.config.calibration_points.append((cx, cy, 1))
# Check if it's the right eye and right calibration is not done yet
elif self.eye_id == EyeId.RIGHT and not var.right_calib:
var.right_calib = True
self.config.calibration_points.append((cx, cy, 0))
# print("calib")
if self.eye_id == EyeId.LEFT and len(self.config.calibration_points) == 8 and var.left_calib == False:
var.left_calib = True
receive_calibration_data(self.config.calibration_points, self.eye_id)
print('SENT LEFT EYE POINTS')
if self.eye_id == EyeId.RIGHT and len(self.config.calibration_points) == 8 and var.right_calib == False:
var.right_calib = True
receive_calibration_data(self.config.calibration_points, self.eye_id)
print('SENT RIGHT EYE POINTS')
# print(len(self.config.calibration_points), self.eye_id)
if self.calibration_frame_counter == 0:
self.calibration_frame_counter = None
@ -217,6 +247,8 @@ class cal:
self.calibration_frame_counter -= 1
if self.settings.gui_recenter_eyes == True:
self.config.calib_XOFF = cx
self.config.calib_YOFF = cy

View File

@ -1,92 +0,0 @@
import numpy as np
import socket
import threading
import os
class TimeoutError(RuntimeError):
pass
class AsyncCall(object):
def __init__(self, fnc, callback=None):
self.Callable = fnc
self.Callback = callback
def __call__(self, *args, **kwargs):
self.Thread = threading.Thread(target=self.run, name=self.Callable.__name__, args=args, kwargs=kwargs)
self.Thread.start()
return self
def wait(self, timeout=None):
self.Thread.join(timeout)
if self.Thread.isAlive():
raise TimeoutError()
else:
return self.Result
def run(self, *args, **kwargs):
self.Result = self.Callable(*args, **kwargs)
if self.Callback:
self.Callback(self.Result)
class AsyncMethod(object):
def __init__(self, fnc, callback=None):
self.Callable = fnc
self.Callback = callback
def __call__(self, *args, **kwargs):
return AsyncCall(self.Callable, self.Callback)(*args, **kwargs)
def Async(fnc=None, callback=None):
if fnc == None:
def AddAsyncCallback(fnc):
return AsyncMethod(fnc, callback)
return AddAsyncCallback
else:
return AsyncMethod(fnc, callback)
@Async
def overlay_calibrate_3d(self):
try:
if self.overlay_active != True:
dirname = os.getcwd()
overlay_path = os.path.join(dirname, "calibrate.bat") # start overlay
os.startfile(overlay_path)
self.overlay_active = True
while var.overlay_active:
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
server_address = ("localhost", 2112)
sock.bind(server_address)
data, address = sock.recvfrom(4096)
received_int = struct.unpack("!l", data)[0] # this will hold until message received
self.point_trigger = True
self.calib_step = reveived_int
print(received_int)
except:
print("[WARN] Calibration overlay error. Make sure SteamVR is Running.")
self.settings.gui_recenter_eyes = False
var.overlay_active = False
class calibrate_3d:
overlay_active = False
point_trigger = False
calib_step = 0
calib_matrix = np.zeros((9, 2))
def nine_point_3d_calib(x, y): # dummy format
if calibrate_3d.point_trigger:
calibrate_3d.calib_matrix[calibrate_3d.calib_step][0] = x
calibrate_3d.calib_matrix[calibrate_3d.calib_step][1] = y
print(calibrate_3d.calib_matrix)
calibrate_3d.point_trigger = True
calibrate_3d.nine_point_3d_calib(1, 2)

View File

@ -0,0 +1,38 @@
# calibration_module.py
class CalibrationProcessor:
def __init__(self):
self.left_eye_data = None
self.right_eye_data = None
def receive_calibration_data(self, eye_id, data):
if eye_id == 'left':
self.left_eye_data = data
elif eye_id == 'right':
self.right_eye_data = data
# Check if both sets of data have been received
if self.left_eye_data is not None and self.right_eye_data is not None:
self.process_calibration_data()
def process_calibration_data(self):
# Ensure both data are present
if self.left_eye_data is None or self.right_eye_data is None:
raise ValueError("Calibration data for both eyes must be provided")
# Example computation with both calibration arrays
# Replace with your actual computation logic
print("Processing calibration data for both eyes...")
print(f"Left Eye Data: {self.left_eye_data}")
print(f"Right Eye Data: {self.right_eye_data}")
# After processing, reset the data
self.left_eye_data = None
self.right_eye_data = None
# Global instance of CalibrationProcessor
calibration_processor = CalibrationProcessor()
def receive_calibration_data(eye_id, data):
global calibration_processor
calibration_processor.receive_calibration_data(eye_id, data)