EyeTrackVR-Docs/InceptionNet/inferno_calib.py
2022-04-21 18:42:11 -07:00

441 lines
16 KiB
Python

import os
import cv2
import numpy as np
from sympy import N
import tensorflow.compat.v1 as tf
from config import config
from models import Inception
from utils import change_channel, gray_normalizer
import time
from pythonosc import udp_client
from scipy import ndimage
import pyttsx3
engine = pyttsx3.init()
tf.disable_v2_behavior()
def load_model(session, m_type, m_name):
# load the weights based on best loss
best_dir = "best_loss"
# check model dir
model_path = "models/" + m_name
path = os.path.join(model_path, best_dir)
if not os.path.exists(path):
raise FileNotFoundError
model = Inception(m_name, config)
# load the best saved weights
ckpt = tf.train.get_checkpoint_state(path)
if ckpt and tf.train.checkpoint_exists(ckpt.model_checkpoint_path):
model.restore(session, ckpt.model_checkpoint_path)
else:
raise ValueError("There is no best model with given model")
return model
def rescale(image):
"""
If the input video is other than network size, it will resize the input video
:param image: a frame form input video
:return: scaled down frame
"""
scale_side = max(image.shape)
# image width and height are equal to 192
scale_value = config["input_width"] / scale_side
# scale down or up the input image
scaled_image = cv2.resize(image, dsize=None, fx=scale_value, fy=scale_value)
# convert to numpy array
scaled_image = np.asarray(scaled_image, dtype=np.uint8)
# one of pad should be zero
w_pad = int((config["input_width"] - scaled_image.shape[1]) / 2)
h_pad = int((config["input_width"] - scaled_image.shape[0]) / 2)
# create a new image with size of: (config["image_width"], config["image_height"])
new_image = (
np.ones((config["input_width"], config["input_height"]), dtype=np.uint8) * 250
)
# put the scaled image in the middle of new image
new_image[
h_pad : h_pad + scaled_image.shape[0], w_pad : w_pad + scaled_image.shape[1]
] = scaled_image
return new_image
def writet(addressipn):
addressips = addressipn.strip().lower()
camadd = open("cam.txt","w+")
camadd.write(str(addressips))
print(addressips)
camadd.close
def main(m_type, m_name):
with tf.Session() as sess: # start a session
# load best model
model = load_model(sess, m_type, m_name) # load the best model
#cap = cv2.VideoCapture('http://192.168.0.202:81/stream') # load the camera
#cap = rotated = ndimage.rotate(capu, 45)
#while cap.isOpened():
with open("config.txt") as calibratefl:
lines = calibratefl.readlines()
vx = float(lines[0].strip())
vy = float(lines[1].strip())
vxl = float(lines[2].strip())
vyl = float(lines[3].strip())
rv = float(lines[4].strip())
calibratefl.close()
#cap = cv2.VideoCapture('http://192.168.0.202:81/stream')
#ret, frame = cap.read()
#frame = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
#try:
# frame1 = ndimage.rotate(frame, int(rv), reshape=True)
# frame1 = frame1[int(vxl): int(float(vy)), int(vyl): int(float(vx))]
# if frame1.shape[0] != 192:
# frame1 = rescale(frame1)
#image = gray_normalizer(frame1)
#image = change_channel(image, config["input_channel"])
# [p] = model.predict(sess, [image])
#@ cv2.circle(frame1, (int(p[0]), int(p[1])), int(p[2]), (0, 0, 255), 2)
#cv2.circle(frame1, (int(p[0]), int(p[1])), 1, (0, 0, 255), -1)
#print(int(p[0]), int(p[1]), int(p[2])) #int(p[2]) pupil pixel size (circ diamiter)
#xt = int(p[0])
#yt = int(p[1])
#cap.release()
#except:
# print('preoc error')
#try:
# xpercentage = (((xt - vc.xmin) * 100) / (vc.xmax - vc.xmin)) / 100 #TESTING NEEDED AM UNSURE IF VALUES NEED TO BE FLIPPED
# ypercentage = (((yt - vc.ymin) * 100) / (vc.ymax - vc.ymin)) / 100
#if vc.cfc == 1 and vc.fc != 1:
#if vc.cfc == 1:
#00 try:
# with open("eyeconfig.cfg") as eyecalib:
# lines = eyecalib.readlines()
# calibcenterx = float(lines[0].strip())
# calibcentery = float(lines[1].strip())
# calibrightx = float(lines[2].strip())
# calibleftx = float(lines[3].strip())
# calibrighty = float(lines[4].strip())
# caliblefty = float(lines[5].strip())
#calibupx = float(lines[6].strip())
#calibupy = float(lines[7].strip())
#vc.cfc = 1
#eyecalib.close()
#except:
while True:
if vc.cc == 1:
engine.say("a saved calibration file was not found.")
engine.say("Calibration starting, 3. 2. 1. please look straight forward")
engine.runAndWait()
vc.cc = 2
if vc.cc == 2:
cap = cv2.VideoCapture('http://192.168.0.202:81/stream')
ret, frame = cap.read()
frame = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
try:
frame1 = ndimage.rotate(frame, int(rv), reshape=True)
frame1 = frame1[int(vxl): int(float(vy)), int(vyl): int(float(vx))]
if frame1.shape[0] != 192:
frame1 = rescale(frame1)
image = gray_normalizer(frame1)
image = change_channel(image, config["input_channel"])
[p] = model.predict(sess, [image])
cv2.circle(frame1, (int(p[0]), int(p[1])), int(p[2]), (0, 0, 255), 2)
cv2.circle(frame1, (int(p[0]), int(p[1])), 1, (0, 0, 255), -1)
#print(int(p[0]), int(p[1]), int(p[2])) #int(p[2]) pupil pixel size (circ diamiter)
xt = int(p[0])
yt = int(p[1])
cap.release()
cv2.destroyAllWindows()
except:
print('preoc error')
calibcenterx = xt
calibcentery = yt
print(xt, yt)
engine.say("center calibration complete, please look right")
engine.runAndWait()
vc.cr = 1
vc.cc = 3
if vc.cr == 1:
engine.say("Right calibration starting")
engine.runAndWait()
vc.cr = 2
if vc.cr == 2:
cap = cv2.VideoCapture('http://192.168.0.202:81/stream')
ret, frame = cap.read()
frame = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
try:
frame1 = ndimage.rotate(frame, int(rv), reshape=True)
frame1 = frame1[int(vxl): int(float(vy)), int(vyl): int(float(vx))]
if frame1.shape[0] != 192:
frame1 = rescale(frame1)
image = gray_normalizer(frame1)
image = change_channel(image, config["input_channel"])
[p] = model.predict(sess, [image])
cv2.circle(frame1, (int(p[0]), int(p[1])), int(p[2]), (0, 0, 255), 2)
cv2.circle(frame1, (int(p[0]), int(p[1])), 1, (0, 0, 255), -1)
#print(int(p[0]), int(p[1]), int(p[2])) #int(p[2]) pupil pixel size (circ diamiter)
xt = int(p[0])
yt = int(p[1])
cap.release()
cv2.destroyAllWindows()
except:
print('preoc error')
calibrightx = xt
calibrighty = yt
print(xt, yt)
engine.say("Right calibration complete, please look left")
engine.runAndWait()
vc.cl = 1
vc.cr = 3
if vc.cl == 1:
engine.say("left calibration starting")
engine.runAndWait()
vc.cl = 2
if vc.cl == 2:
cap = cv2.VideoCapture('http://192.168.0.202:81/stream')
ret, frame = cap.read()
frame = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
try:
frame1 = ndimage.rotate(frame, int(rv), reshape=True)
frame1 = frame1[int(vxl): int(float(vy)), int(vyl): int(float(vx))]
if frame1.shape[0] != 192:
frame1 = rescale(frame1)
image = gray_normalizer(frame1)
image = change_channel(image, config["input_channel"])
[p] = model.predict(sess, [image])
cv2.circle(frame1, (int(p[0]), int(p[1])), int(p[2]), (0, 0, 255), 2)
cv2.circle(frame1, (int(p[0]), int(p[1])), 1, (0, 0, 255), -1)
#print(int(p[0]), int(p[1]), int(p[2])) #int(p[2]) pupil pixel size (circ diamiter)
xt = int(p[0])
yt = int(p[1])
cap.release()
cv2.destroyAllWindows()
except:
print('preoc error')
calibleftx = xt
caliblefty = yt
print(xt, yt)
engine.say("left calibration complete, please look up")
engine.runAndWait()
vc.cl = 3
vc.cu = 1
if vc.cu == 1:
engine.say("up calibration starting")
engine.runAndWait()
vc.cu = 2
if vc.cu == 2:
cap = cv2.VideoCapture('http://192.168.0.202:81/stream')
ret, frame = cap.read()
frame = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
try:
frame1 = ndimage.rotate(frame, int(rv), reshape=True)
frame1 = frame1[int(vxl): int(float(vy)), int(vyl): int(float(vx))]
if frame1.shape[0] != 192:
frame1 = rescale(frame1)
image = gray_normalizer(frame1)
image = change_channel(image, config["input_channel"])
[p] = model.predict(sess, [image])
cv2.circle(frame1, (int(p[0]), int(p[1])), int(p[2]), (0, 0, 255), 2)
cv2.circle(frame1, (int(p[0]), int(p[1])), 1, (0, 0, 255), -1)
#print(int(p[0]), int(p[1]), int(p[2])) #int(p[2]) pupil pixel size (circ diamiter)
xt = int(p[0])
yt = int(p[1])
cap.release()
cv2.destroyAllWindows()
except:
print('preoc error')
calibupx = xt
calibupy = yt
print(xt, yt)
engine.say("up calibration complete, please look down")
engine.runAndWait()
vc.cd = 1
vc.cu = 3
if vc.cd == 1:
engine.say("down calibration starting")
engine.runAndWait()
vc.cd = 2
if vc.cd == 2:
cap = cv2.VideoCapture('http://192.168.0.202:81/stream')
ret, frame = cap.read()
frame = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
try:
frame1 = ndimage.rotate(frame, int(rv), reshape=True)
frame1 = frame1[int(vxl): int(float(vy)), int(vyl): int(float(vx))]
if frame1.shape[0] != 192:
frame1 = rescale(frame1)
image = gray_normalizer(frame1)
image = change_channel(image, config["input_channel"])
[p] = model.predict(sess, [image])
cv2.circle(frame1, (int(p[0]), int(p[1])), int(p[2]), (0, 0, 255), 2)
cv2.circle(frame1, (int(p[0]), int(p[1])), 1, (0, 0, 255), -1)
#print(int(p[0]), int(p[1]), int(p[2])) #int(p[2]) pupil pixel size (circ diamiter)
xt = int(p[0])
yt = int(p[1])
cap.release()
cv2.destroyAllWindows()
except:
print('preoc error')
calibdownx = xt
calibdowny = yt
print(xt, yt)
engine.say("calibration complete")
engine.runAndWait()
vc.cd = 3
else:
print('CALIBCOMPLETE')
savecalibvalues(calibcenterx, calibcentery, calibrightx, calibleftx, calibupy, calibdowny)
vc.cfc = 2
vc.fc = 1
print('CALIBCOMPLETE22q2')
break
# client.send_message("/avatar/parameters/RightEyeX", xper) #sends to vr chat needs to use calibration function
#client.send_message("/avatar/parameters/LeftEyeX", xper)
# client.send_message("/Avatar/LeftEyeY", ypercentage)
#client.send_message("/Avatar/RightEyeY", ypercentage)
#client.send_message("/avatar/parameters/EyesY", yper)
#client.send_message("/avatar/parameters/RightEyeLid", 0)
#client.send_message("/avatar/parameters/LeftEyeLid", 0)
#print('X: ', xper, ' Y: ', yper)
#except:
# print('[WARN] Calculation Error: Move Eye Around or Adjust Detection Threshold.')
#xt = int(p[0])
#yt = int(p[1])
# cv2.imshow("frame", frame1)
# cv2.imshow("img", image)
# except:
# print('sussyy e rawr')
# if cv2.waitKey(1) & 0xFF == ord("q"):
#break
# cv2.destroyAllWindows()
if __name__ == "__main__":
model_name = "3A4Bh-Ref25"
model_type = "INC"
video_path = 0
# with open("config.txt") as calibratefl:
# lines = calibratefl.readlines()
# rv = float(lines[4].strip())
# calibratefl.close()
def vc():
vc.xmax = 1
vc.xmin = 6969
vc.ymax = 1
vc.ymin = 6969
vc.cfc = 50
vc.cc = 1
vc.cu = 0
vc.cd = 0
vc.cl = 0
vc.cr = 0
vc.fc = 0
vc()
def savecalibvalues(calibcenterx, calibcentery, calibrightx, calibleftx, calibupy, calibdowny):
with open('eyeconfig.cfg', 'w+') as cw:
cw.write(str(calibcenterx))
cw.write('\n')
cw.write(str(calibcentery))
cw.write('\n')
cw.write(str(calibrightx))
cw.write('\n')
cw.write(str(calibleftx))
cw.write('\n')
cw.write(str(calibupy))
cw.write('\n')
cw.write(str(calibdowny))
cw.close()
# initial a logger
main(model_type, model_name)