mirror of
https://github.com/EyeTrackVR/EyeTrackVR.git
synced 2025-11-04 14:39:42 +08:00
441 lines
16 KiB
Python
441 lines
16 KiB
Python
import os
|
|
|
|
|
|
import cv2
|
|
import numpy as np
|
|
from sympy import N
|
|
import tensorflow.compat.v1 as tf
|
|
|
|
from config import config
|
|
from models import Inception
|
|
from utils import change_channel, gray_normalizer
|
|
|
|
import time
|
|
from pythonosc import udp_client
|
|
from scipy import ndimage
|
|
|
|
|
|
import pyttsx3
|
|
|
|
engine = pyttsx3.init()
|
|
|
|
|
|
tf.disable_v2_behavior()
|
|
|
|
|
|
def load_model(session, m_type, m_name):
|
|
# load the weights based on best loss
|
|
best_dir = "best_loss"
|
|
|
|
# check model dir
|
|
model_path = "models/" + m_name
|
|
path = os.path.join(model_path, best_dir)
|
|
if not os.path.exists(path):
|
|
raise FileNotFoundError
|
|
model = Inception(m_name, config)
|
|
|
|
# load the best saved weights
|
|
ckpt = tf.train.get_checkpoint_state(path)
|
|
if ckpt and tf.train.checkpoint_exists(ckpt.model_checkpoint_path):
|
|
model.restore(session, ckpt.model_checkpoint_path)
|
|
|
|
else:
|
|
raise ValueError("There is no best model with given model")
|
|
|
|
return model
|
|
|
|
|
|
def rescale(image):
|
|
"""
|
|
If the input video is other than network size, it will resize the input video
|
|
:param image: a frame form input video
|
|
:return: scaled down frame
|
|
"""
|
|
scale_side = max(image.shape)
|
|
# image width and height are equal to 192
|
|
scale_value = config["input_width"] / scale_side
|
|
|
|
# scale down or up the input image
|
|
scaled_image = cv2.resize(image, dsize=None, fx=scale_value, fy=scale_value)
|
|
|
|
# convert to numpy array
|
|
scaled_image = np.asarray(scaled_image, dtype=np.uint8)
|
|
|
|
# one of pad should be zero
|
|
w_pad = int((config["input_width"] - scaled_image.shape[1]) / 2)
|
|
h_pad = int((config["input_width"] - scaled_image.shape[0]) / 2)
|
|
|
|
# create a new image with size of: (config["image_width"], config["image_height"])
|
|
new_image = (
|
|
np.ones((config["input_width"], config["input_height"]), dtype=np.uint8) * 250
|
|
)
|
|
|
|
# put the scaled image in the middle of new image
|
|
new_image[
|
|
h_pad : h_pad + scaled_image.shape[0], w_pad : w_pad + scaled_image.shape[1]
|
|
] = scaled_image
|
|
|
|
return new_image
|
|
|
|
|
|
def writet(addressipn):
|
|
addressips = addressipn.strip().lower()
|
|
camadd = open("cam.txt","w+")
|
|
camadd.write(str(addressips))
|
|
print(addressips)
|
|
camadd.close
|
|
|
|
|
|
def main(m_type, m_name):
|
|
with tf.Session() as sess: # start a session
|
|
|
|
# load best model
|
|
model = load_model(sess, m_type, m_name) # load the best model
|
|
#cap = cv2.VideoCapture('http://192.168.0.202:81/stream') # load the camera
|
|
#cap = rotated = ndimage.rotate(capu, 45)
|
|
#while cap.isOpened():
|
|
|
|
with open("config.txt") as calibratefl:
|
|
lines = calibratefl.readlines()
|
|
vx = float(lines[0].strip())
|
|
vy = float(lines[1].strip())
|
|
vxl = float(lines[2].strip())
|
|
vyl = float(lines[3].strip())
|
|
rv = float(lines[4].strip())
|
|
calibratefl.close()
|
|
|
|
#cap = cv2.VideoCapture('http://192.168.0.202:81/stream')
|
|
#ret, frame = cap.read()
|
|
#frame = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
|
|
#try:
|
|
# frame1 = ndimage.rotate(frame, int(rv), reshape=True)
|
|
# frame1 = frame1[int(vxl): int(float(vy)), int(vyl): int(float(vx))]
|
|
|
|
# if frame1.shape[0] != 192:
|
|
# frame1 = rescale(frame1)
|
|
|
|
#image = gray_normalizer(frame1)
|
|
#image = change_channel(image, config["input_channel"])
|
|
|
|
# [p] = model.predict(sess, [image])
|
|
#@ cv2.circle(frame1, (int(p[0]), int(p[1])), int(p[2]), (0, 0, 255), 2)
|
|
#cv2.circle(frame1, (int(p[0]), int(p[1])), 1, (0, 0, 255), -1)
|
|
#print(int(p[0]), int(p[1]), int(p[2])) #int(p[2]) pupil pixel size (circ diamiter)
|
|
#xt = int(p[0])
|
|
#yt = int(p[1])
|
|
#cap.release()
|
|
#except:
|
|
# print('preoc error')
|
|
#try:
|
|
# xpercentage = (((xt - vc.xmin) * 100) / (vc.xmax - vc.xmin)) / 100 #TESTING NEEDED AM UNSURE IF VALUES NEED TO BE FLIPPED
|
|
# ypercentage = (((yt - vc.ymin) * 100) / (vc.ymax - vc.ymin)) / 100
|
|
#if vc.cfc == 1 and vc.fc != 1:
|
|
#if vc.cfc == 1:
|
|
#00 try:
|
|
|
|
# with open("eyeconfig.cfg") as eyecalib:
|
|
# lines = eyecalib.readlines()
|
|
# calibcenterx = float(lines[0].strip())
|
|
# calibcentery = float(lines[1].strip())
|
|
# calibrightx = float(lines[2].strip())
|
|
# calibleftx = float(lines[3].strip())
|
|
# calibrighty = float(lines[4].strip())
|
|
# caliblefty = float(lines[5].strip())
|
|
#calibupx = float(lines[6].strip())
|
|
#calibupy = float(lines[7].strip())
|
|
|
|
#vc.cfc = 1
|
|
#eyecalib.close()
|
|
|
|
#except:
|
|
while True:
|
|
|
|
if vc.cc == 1:
|
|
engine.say("a saved calibration file was not found.")
|
|
engine.say("Calibration starting, 3. 2. 1. please look straight forward")
|
|
engine.runAndWait()
|
|
vc.cc = 2
|
|
|
|
|
|
if vc.cc == 2:
|
|
cap = cv2.VideoCapture('http://192.168.0.202:81/stream')
|
|
ret, frame = cap.read()
|
|
frame = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
|
|
try:
|
|
frame1 = ndimage.rotate(frame, int(rv), reshape=True)
|
|
frame1 = frame1[int(vxl): int(float(vy)), int(vyl): int(float(vx))]
|
|
|
|
if frame1.shape[0] != 192:
|
|
frame1 = rescale(frame1)
|
|
|
|
image = gray_normalizer(frame1)
|
|
image = change_channel(image, config["input_channel"])
|
|
|
|
[p] = model.predict(sess, [image])
|
|
cv2.circle(frame1, (int(p[0]), int(p[1])), int(p[2]), (0, 0, 255), 2)
|
|
cv2.circle(frame1, (int(p[0]), int(p[1])), 1, (0, 0, 255), -1)
|
|
#print(int(p[0]), int(p[1]), int(p[2])) #int(p[2]) pupil pixel size (circ diamiter)
|
|
xt = int(p[0])
|
|
yt = int(p[1])
|
|
cap.release()
|
|
cv2.destroyAllWindows()
|
|
except:
|
|
print('preoc error')
|
|
|
|
calibcenterx = xt
|
|
calibcentery = yt
|
|
print(xt, yt)
|
|
engine.say("center calibration complete, please look right")
|
|
engine.runAndWait()
|
|
vc.cr = 1
|
|
vc.cc = 3
|
|
|
|
|
|
if vc.cr == 1:
|
|
engine.say("Right calibration starting")
|
|
engine.runAndWait()
|
|
vc.cr = 2
|
|
|
|
|
|
if vc.cr == 2:
|
|
cap = cv2.VideoCapture('http://192.168.0.202:81/stream')
|
|
ret, frame = cap.read()
|
|
frame = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
|
|
try:
|
|
frame1 = ndimage.rotate(frame, int(rv), reshape=True)
|
|
frame1 = frame1[int(vxl): int(float(vy)), int(vyl): int(float(vx))]
|
|
|
|
if frame1.shape[0] != 192:
|
|
frame1 = rescale(frame1)
|
|
|
|
image = gray_normalizer(frame1)
|
|
image = change_channel(image, config["input_channel"])
|
|
|
|
[p] = model.predict(sess, [image])
|
|
cv2.circle(frame1, (int(p[0]), int(p[1])), int(p[2]), (0, 0, 255), 2)
|
|
cv2.circle(frame1, (int(p[0]), int(p[1])), 1, (0, 0, 255), -1)
|
|
#print(int(p[0]), int(p[1]), int(p[2])) #int(p[2]) pupil pixel size (circ diamiter)
|
|
xt = int(p[0])
|
|
yt = int(p[1])
|
|
cap.release()
|
|
cv2.destroyAllWindows()
|
|
except:
|
|
print('preoc error')
|
|
calibrightx = xt
|
|
calibrighty = yt
|
|
print(xt, yt)
|
|
engine.say("Right calibration complete, please look left")
|
|
engine.runAndWait()
|
|
vc.cl = 1
|
|
vc.cr = 3
|
|
|
|
|
|
if vc.cl == 1:
|
|
engine.say("left calibration starting")
|
|
engine.runAndWait()
|
|
vc.cl = 2
|
|
|
|
|
|
|
|
if vc.cl == 2:
|
|
cap = cv2.VideoCapture('http://192.168.0.202:81/stream')
|
|
ret, frame = cap.read()
|
|
frame = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
|
|
try:
|
|
frame1 = ndimage.rotate(frame, int(rv), reshape=True)
|
|
frame1 = frame1[int(vxl): int(float(vy)), int(vyl): int(float(vx))]
|
|
|
|
if frame1.shape[0] != 192:
|
|
frame1 = rescale(frame1)
|
|
|
|
image = gray_normalizer(frame1)
|
|
image = change_channel(image, config["input_channel"])
|
|
|
|
[p] = model.predict(sess, [image])
|
|
cv2.circle(frame1, (int(p[0]), int(p[1])), int(p[2]), (0, 0, 255), 2)
|
|
cv2.circle(frame1, (int(p[0]), int(p[1])), 1, (0, 0, 255), -1)
|
|
#print(int(p[0]), int(p[1]), int(p[2])) #int(p[2]) pupil pixel size (circ diamiter)
|
|
xt = int(p[0])
|
|
yt = int(p[1])
|
|
cap.release()
|
|
cv2.destroyAllWindows()
|
|
except:
|
|
print('preoc error')
|
|
calibleftx = xt
|
|
caliblefty = yt
|
|
print(xt, yt)
|
|
engine.say("left calibration complete, please look up")
|
|
engine.runAndWait()
|
|
vc.cl = 3
|
|
vc.cu = 1
|
|
|
|
if vc.cu == 1:
|
|
engine.say("up calibration starting")
|
|
engine.runAndWait()
|
|
vc.cu = 2
|
|
|
|
|
|
if vc.cu == 2:
|
|
cap = cv2.VideoCapture('http://192.168.0.202:81/stream')
|
|
ret, frame = cap.read()
|
|
frame = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
|
|
try:
|
|
frame1 = ndimage.rotate(frame, int(rv), reshape=True)
|
|
frame1 = frame1[int(vxl): int(float(vy)), int(vyl): int(float(vx))]
|
|
|
|
if frame1.shape[0] != 192:
|
|
frame1 = rescale(frame1)
|
|
|
|
image = gray_normalizer(frame1)
|
|
image = change_channel(image, config["input_channel"])
|
|
|
|
[p] = model.predict(sess, [image])
|
|
cv2.circle(frame1, (int(p[0]), int(p[1])), int(p[2]), (0, 0, 255), 2)
|
|
cv2.circle(frame1, (int(p[0]), int(p[1])), 1, (0, 0, 255), -1)
|
|
#print(int(p[0]), int(p[1]), int(p[2])) #int(p[2]) pupil pixel size (circ diamiter)
|
|
xt = int(p[0])
|
|
yt = int(p[1])
|
|
cap.release()
|
|
cv2.destroyAllWindows()
|
|
except:
|
|
print('preoc error')
|
|
calibupx = xt
|
|
calibupy = yt
|
|
print(xt, yt)
|
|
engine.say("up calibration complete, please look down")
|
|
engine.runAndWait()
|
|
vc.cd = 1
|
|
vc.cu = 3
|
|
|
|
if vc.cd == 1:
|
|
engine.say("down calibration starting")
|
|
engine.runAndWait()
|
|
vc.cd = 2
|
|
|
|
|
|
if vc.cd == 2:
|
|
cap = cv2.VideoCapture('http://192.168.0.202:81/stream')
|
|
ret, frame = cap.read()
|
|
frame = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
|
|
try:
|
|
frame1 = ndimage.rotate(frame, int(rv), reshape=True)
|
|
frame1 = frame1[int(vxl): int(float(vy)), int(vyl): int(float(vx))]
|
|
|
|
if frame1.shape[0] != 192:
|
|
frame1 = rescale(frame1)
|
|
|
|
image = gray_normalizer(frame1)
|
|
image = change_channel(image, config["input_channel"])
|
|
|
|
|
|
[p] = model.predict(sess, [image])
|
|
cv2.circle(frame1, (int(p[0]), int(p[1])), int(p[2]), (0, 0, 255), 2)
|
|
cv2.circle(frame1, (int(p[0]), int(p[1])), 1, (0, 0, 255), -1)
|
|
#print(int(p[0]), int(p[1]), int(p[2])) #int(p[2]) pupil pixel size (circ diamiter)
|
|
xt = int(p[0])
|
|
yt = int(p[1])
|
|
cap.release()
|
|
cv2.destroyAllWindows()
|
|
except:
|
|
print('preoc error')
|
|
calibdownx = xt
|
|
calibdowny = yt
|
|
print(xt, yt)
|
|
engine.say("calibration complete")
|
|
engine.runAndWait()
|
|
vc.cd = 3
|
|
else:
|
|
print('CALIBCOMPLETE')
|
|
savecalibvalues(calibcenterx, calibcentery, calibrightx, calibleftx, calibupy, calibdowny)
|
|
vc.cfc = 2
|
|
vc.fc = 1
|
|
print('CALIBCOMPLETE22q2')
|
|
break
|
|
|
|
|
|
|
|
|
|
|
|
|
|
# client.send_message("/avatar/parameters/RightEyeX", xper) #sends to vr chat needs to use calibration function
|
|
#client.send_message("/avatar/parameters/LeftEyeX", xper)
|
|
# client.send_message("/Avatar/LeftEyeY", ypercentage)
|
|
#client.send_message("/Avatar/RightEyeY", ypercentage)
|
|
#client.send_message("/avatar/parameters/EyesY", yper)
|
|
#client.send_message("/avatar/parameters/RightEyeLid", 0)
|
|
#client.send_message("/avatar/parameters/LeftEyeLid", 0)
|
|
#print('X: ', xper, ' Y: ', yper)
|
|
|
|
#except:
|
|
# print('[WARN] Calculation Error: Move Eye Around or Adjust Detection Threshold.')
|
|
|
|
|
|
#xt = int(p[0])
|
|
#yt = int(p[1])
|
|
|
|
|
|
|
|
|
|
|
|
# cv2.imshow("frame", frame1)
|
|
# cv2.imshow("img", image)
|
|
# except:
|
|
# print('sussyy e rawr')
|
|
|
|
|
|
# if cv2.waitKey(1) & 0xFF == ord("q"):
|
|
#break
|
|
|
|
# cv2.destroyAllWindows()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
model_name = "3A4Bh-Ref25"
|
|
model_type = "INC"
|
|
video_path = 0
|
|
|
|
|
|
# with open("config.txt") as calibratefl:
|
|
# lines = calibratefl.readlines()
|
|
# rv = float(lines[4].strip())
|
|
# calibratefl.close()
|
|
|
|
|
|
def vc():
|
|
|
|
vc.xmax = 1
|
|
vc.xmin = 6969
|
|
vc.ymax = 1
|
|
vc.ymin = 6969
|
|
|
|
|
|
vc.cfc = 50
|
|
vc.cc = 1
|
|
vc.cu = 0
|
|
vc.cd = 0
|
|
vc.cl = 0
|
|
vc.cr = 0
|
|
vc.fc = 0
|
|
vc()
|
|
|
|
def savecalibvalues(calibcenterx, calibcentery, calibrightx, calibleftx, calibupy, calibdowny):
|
|
with open('eyeconfig.cfg', 'w+') as cw:
|
|
cw.write(str(calibcenterx))
|
|
cw.write('\n')
|
|
cw.write(str(calibcentery))
|
|
cw.write('\n')
|
|
cw.write(str(calibrightx))
|
|
cw.write('\n')
|
|
cw.write(str(calibleftx))
|
|
cw.write('\n')
|
|
cw.write(str(calibupy))
|
|
cw.write('\n')
|
|
cw.write(str(calibdowny))
|
|
cw.close()
|
|
|
|
|
|
|
|
# initial a logger
|
|
|
|
main(model_type, model_name)
|