From 2e48bedf551dc9ccb672a2756e1a9059d92abf56 Mon Sep 17 00:00:00 2001 From: Jason Date: Wed, 29 Aug 2018 13:58:03 +0900 Subject: [PATCH] Added drift correction code and instructions --- main/main.cpp | 307 +++++++++++++++++++++++++++++++------------------- 1 file changed, 192 insertions(+), 115 deletions(-) diff --git a/main/main.cpp b/main/main.cpp index 2488b70..b18c38a 100644 --- a/main/main.cpp +++ b/main/main.cpp @@ -4,67 +4,60 @@ **/ - #include #include #include #include #include #include - - +#include #include "ubitrack_util.h" // claibration file handlers #include #include #include #include #include - #include "opencv2/opencv.hpp" #include #include #include #include #include - - #include "pupilFitter.h" // 2D pupil detector #include "timer.h" - #include "eye_model_updater.h" // 3D model builder #include "eye_cameras.h" // Camera interfaces - - +#include "pupil_stereo_cameras.h" //interface to pupil stereo cameras on a single USB +#include namespace { -enum InputMode { CAMERA, CAMERA_MONO, VIDEO, IMAGE }; +enum InputMode { CAMERA, CAMERA_MONO, CAMERA_PUPIL, VIDEO, IMAGE }; } - int main(int argc, char *argv[]){ + InputMode input_mode = + //InputMode::VIDEO; // Set a video as a video source + //InputMode::CAMERA; // Set two cameras (separate USB buses) as video sources + InputMode::CAMERA_PUPIL; //Pupil stereo cameras (on a single cable, uses libuvc) + // InputMode::CAMERA_MONO; // Set a single camera as a video source + // InputMode::IMAGE;// Set an image as a the source + + string folderPath = "C:\\Storage\\Research\\Eye Tracking\\coordinates.txt"; + // Variables for FPS eye_tracker::FrameRateCounter frame_rate_counter; - bool kVisualization = false; kVisualization = true; singleeyefitter::EyeModelFitter::Circle curr_circle; - - InputMode input_mode = - //InputMode::VIDEO; // Set a video as a video source - // InputMode::CAMERA; // Set two cameras as video sources - InputMode::CAMERA_MONO; // Set a camera as video sources - // InputMode::IMAGE;// Set an image as a video source - - ////// Command line opitions ///////////// std::string kDir = "C:/Users/Yuta/Dropbox/work/Projects/20150427_Alex_EyeTracker/"; std::string media_file; std::string media_file_stem; - //std::string kOutputDataDirectory(kDir + "out/"); // Data output directroy + if (argc > 2) { boost::filesystem::path file_name = std::string(argv[2]); kDir = std::string(argv[1]); @@ -106,7 +99,7 @@ int main(int argc, char *argv[]){ std::string calib_path="../../docs/cameraintrinsics_eye.txt"; eye_tracker::UbitrackTextReader ubitrack_calib_text_reader; if (ubitrack_calib_text_reader.read(calib_path) == false){ - std::cout << "Calibration file onpen error: " << calib_path << std::endl; + std::cout << "Calibration file open error: " << calib_path << std::endl; return -1; } cv::Mat K; // Camera intrinsic matrix in OpenCV format @@ -116,7 +109,6 @@ int main(int argc, char *argv[]){ // Focal distance used in the 3D eye model fitter double focal_length = (K.at(0,0)+K.at(1,1))*0.5; // Required for the 3D model fitting - // Set mode parameters size_t kCameraNums; switch (input_mode) @@ -129,6 +121,9 @@ int main(int argc, char *argv[]){ case InputMode::CAMERA: kCameraNums = 2; break; + case InputMode::CAMERA_PUPIL: + kCameraNums = 2; + break; default: break; } @@ -164,7 +159,7 @@ int main(int argc, char *argv[]){ break; case InputMode::CAMERA: camera_indices[0] = 0; - camera_indices[1] = 2; + camera_indices[1] = 1; #if 0 // OpenCV HighGUI frame grabber eyecams[0] = std::make_unique(camera_indices[0], false); @@ -181,6 +176,21 @@ int main(int argc, char *argv[]){ window_names = { "Cam0", "Cam1" }; file_stems = { "cam0", "cam1" }; break; + case InputMode::CAMERA_PUPIL: + { + camera_indices[0] = 0; + camera_indices[1] = 1; + eye_model_updaters[0] = std::make_unique(focal_length, 5, 0.5); + eye_model_updaters[1] = std::make_unique(focal_length, 5, 0.5); + camera_undistorters[0] = std::make_unique(K, distCoeffs); + camera_undistorters[1] = std::make_unique(K, distCoeffs); + window_names = { "Cam0", "Cam1" }; + file_stems = { "cam0", "cam1" }; + initialize(); + manager->setExposureTime(0, .035); + manager->setExposureTime(1, .035); + break; + } case InputMode::CAMERA_MONO: eyecams[0] = std::make_unique("Pupil Cam1 ID1"); // eye_model_updaters[0] = std::make_unique(focal_length, 5, 0.5); @@ -192,7 +202,7 @@ int main(int argc, char *argv[]){ break; } } - catch (char *c){ + catch (char *c) { std::cout << "Exception: "; std::cout << c << std::endl; return 0; @@ -204,17 +214,21 @@ int main(int argc, char *argv[]){ PupilFitter pupilFitter; pupilFitter.setDebug(false); ///////////////////////// - - //std::getchar(); + //For running a video //VideoCapture inputVideo1("C:\\Documents\\Osaka\\Research\\Eye Tracking\\Benchmark Videos\\eyetracking4.avi"); // Open input + + //for video writing + /* VideoWriter outputVideo1; outputVideo1.open("C:\\Documents\\Osaka\\Research\\Eye Tracking\\Benchmark Videos\\outSaccade.avi", CV_FOURCC('W', 'M', 'V', '2'), - 20, + 20, cv::Size(640,480), true); + Mat frame1; + */ // Main loop const char kTerminate = 27;//Escape 0x1b @@ -228,21 +242,33 @@ int main(int argc, char *argv[]){ vector xData; //corresponding x eye rotations for N frames vector yData; //corresponding y eye rotations for N frames vector intensityData; //holds average intensity of last N frames - vector eyes; //holds a vector of spheres for the eye model filter + vector eyes[2]; //holds a vector of spheres for the eye model filter (cam 0) + singleeyefitter::EyeModelFitter::Sphere lastGoodEyes[2]; + singleeyefitter::EyeModelFitter::Sphere originalModels[2]; + double originalEyeSizes[2] = { 0, 0 }; //stores original 2D radii + int medianTotal = 120; + double camSpheres[6] = { 0, 0, 0, 0, 0, 0 }; //holds left cam (0-2) and right cam (3-5) sphere centers + //double cam1Sphere[3] = { 0, 0, 0 }; + //print instructions + cout << endl; + cout << "********** This is the beta version of our open source eye tracker **********" << endl; + cout << endl; + cout << "Keyboard options (with either of the eye camera windows selected) include: " << endl; + cout << "r: Resets the original eye models - do this if the initial model creation results in a poor fit. " << endl; + cout << "x: Cleanly exists the stream - if you accidentally close the window using the mouse, you may need to unplug and replug your pupil labs USB cable" << endl; + cout << endl; + cout << "Edit the 'input_mode' variable to select the type of camera you are using (default is the stereo pupil labs rig on a single USB)." << endl; + cout << "Edit the 'folderPath' variable to output coordinates to a text file in real time." << endl; + cout << "The current 'folderPath' is: " << folderPath << endl; while (is_run) { - //inputVideo1 >> frame1;//for video //if (frame1.empty()) {//for video // break; //} - //imshow("test", frame1);//for video - //waitKey(0); - - // Fetch key input char kKEY = 0; if (kVisualization) { @@ -254,14 +280,19 @@ int main(int argc, char *argv[]){ break; } - - // Fetch images for (size_t cam = 0; cam < kCameraNums; cam++) { - - eyecams[cam] -> fetchFrame(images[cam]); - + if (InputMode::CAMERA_PUPIL) { //stereo on single/dual bus + fetchFrame(images[cam], cam); + } + else { //any other camera solution + eyecams[cam]->fetchFrame(images[cam]); + } } + + //for writing data to file + stringstream eyeVector[2]; + // Process each camera images for (size_t cam = 0; cam < kCameraNums; cam++) { @@ -269,6 +300,10 @@ int main(int argc, char *argv[]){ //img = frame1; //for video //imshow("test", img); //waitKey(1); + + if (cam == 0) { + flip(images[cam], img, -1); + } if (img.empty()) { //is_run = false; @@ -282,6 +317,7 @@ int main(int argc, char *argv[]){ cv::Mat img_rgb_debug = img.clone(); cv::Mat img_grey; + switch (kKEY) { case 'r': eye_model_updaters[cam]->reset(); @@ -292,15 +328,15 @@ int main(int argc, char *argv[]){ case 'q': is_run = false; break; - case 'z': - eye_model_updaters[cam]->rm_oldest_observation(); - break; + case 'x': + is_run = false; + manager->stopStream(0); + manager->stopStream(1); + exit(0); default: break; } - - const clock_t begin_time = clock(); // 2D ellipse detection @@ -308,11 +344,8 @@ int main(int argc, char *argv[]){ cv::cvtColor(img, img_grey, CV_RGB2GRAY); cv::RotatedRect rr_pf; - - //imshow("test", img_grey); - - bool is_pupil_found = pupilFitter.pupilAreaFitRR(img_grey, rr_pf, inlier_pts, 15, 0, 0, 20, 30, 250, 6); - + bool is_pupil_found = pupilFitter.pupilAreaFitRR(img_grey, rr_pf, inlier_pts, 15, 0, 0, 15, 35, 240, 6); + is_pupil_found = pupilFitter.badEllipseFilter(rr_pf, 250); //cout << "pupil fitter time: " << float(clock() - begin_time) / CLOCKS_PER_SEC << endl; const clock_t begin_time2 = clock(); @@ -321,42 +354,46 @@ int main(int argc, char *argv[]){ //cout << "singleeyefitter time: " << float(clock() - begin_time2) / CLOCKS_PER_SEC << endl; - // 3D eye pose estimation bool is_reliable = false; bool is_added = false; const bool force_add = false; - const double kReliabilityThreshold = 0.0;//0.96; + const double kReliabilityThreshold = 0;//0.96; double ellipse_reliability = 0.0; /// Reliability of a detected 2D ellipse based on 3D eye model + if (is_pupil_found) { if (eye_model_updaters[cam]->is_model_built()) { ellipse_reliability = eye_model_updaters[cam]->compute_reliability(img, el, inlier_pts); is_reliable = (ellipse_reliability > kReliabilityThreshold); - // is_reliable = true; + is_reliable = true; - + //remove oldest observation, add new, and rebuild model (drift correction) eye_model_updaters[cam]->rm_oldest_observation(); eye_model_updaters[cam]->add_observation(img_grey, el, inlier_pts, false); eye_model_updaters[cam]->force_rebuild_model(); + } - else { - cout << "oops" << endl; + else { is_added = eye_model_updaters[cam]->add_observation(img_grey, el, inlier_pts, force_add); + if (eye_model_updaters[cam]->is_model_built()) { + // happens once when model is built for the first time to establish eye-box + originalModels[cam] = eye_model_updaters[cam]->getEye(); + originalEyeSizes[cam] = eye_tracker::toImgCoord(singleeyefitter::toRotatedRect( + singleeyefitter::project(eye_model_updaters[cam]->getEye(), focal_length)), img, 1.0f).size.height; + } } - //TODO test in Unity to see how well this works } // Visualize results - if (cam == 0 && kVisualization) { + if (kVisualization) { // 2D pupil if (is_pupil_found) { cv::ellipse(img_rgb_debug, rr_pf, cv::Vec3b(255, 128, 0), 1); } - // 3D eye ball if (eye_model_updaters[cam]->is_model_built()) { - cv::putText(img, "Reliability: " + std::to_string(ellipse_reliability), cv::Point(30, 440), cv::FONT_HERSHEY_SIMPLEX, 1.0, cv::Scalar(0, 128, 255), 1); + if (is_reliable) { singleeyefitter::Sphere medianCircle; @@ -365,84 +402,124 @@ int main(int argc, char *argv[]){ // medianCircle = eye_model_updaters[cam]->eyeModelFilter(curr_circle, eyes); // useDriftCorrection = true; //} - - //std::cout << "after filter: " << curr_circle.radius << std::endl; eye_model_updaters[cam]->render(img_rgb_debug, el, inlier_pts); - eye_model_updaters[cam]->set_fitter_max_count(130); //manually sets max count - + eye_model_updaters[cam]->set_fitter_max_count(50); //manually sets max count //3D filtered eye model - medianCircle = eye_model_updaters[cam]->eyeModelFilter(eye_model_updaters[cam]->fitter().eye, eyes, 500); - if (medianCircle.radius < 10) { - medianCircle.radius = 10; - } - eye_model_updaters[cam]->setEye(medianCircle); curr_circle = eye_model_updaters[cam]->unproject(img, el, inlier_pts); // 3D pupil (relative to filtered eye model) - singleeyefitter::Ellipse2D pupil_el(singleeyefitter::project(curr_circle, focal_length)); - cv::RotatedRect rr_pupil = eye_tracker::toImgCoord(singleeyefitter::toRotatedRect(pupil_el), img, 1.0f); - singleeyefitter::EyeModelFitter::Sphere filteredEye(medianCircle.centre, medianCircle.radius); + singleeyefitter::Ellipse2D pupil_elTest(singleeyefitter::project(curr_circle, focal_length)); + cv::RotatedRect rr_pupilTest = eye_tracker::toImgCoord(singleeyefitter::toRotatedRect(pupil_elTest), img, 1.0f); - cout << "radius was " << medianCircle.radius << endl; - cv::RotatedRect rr_eye = eye_tracker::toImgCoord(singleeyefitter::toRotatedRect( - singleeyefitter::project(filteredEye, focal_length)), img, 1.0f); - cv::ellipse(img_rgb_debug, rr_eye, cv::Vec3b(255, 222, 222), 2, CV_AA); - cv::circle(img_rgb_debug, rr_eye.center, 3, cv::Vec3b(255, 32, 32), 2); // Eyeball center projection - singleeyefitter::EyeModelFitter::Circle c_end = curr_circle; - c_end.centre = curr_circle.centre + (10.0)*curr_circle.normal; + bool ignoreNewEye = true; + if (rr_pupilTest.center.x > 0 || rr_pupilTest.center.y > 0 ) { + ignoreNewEye = false; //ignore eyes with 0 or negative origins + } - cv::line(img_rgb_debug, rr_eye.center, rr_pupil.center, cv::Vec3b(25, 22, 222), 3, CV_AA); + if (eye_model_updaters[cam]->fitter().eye) { //ensure eye model exists + //insert current eye into filter and return a filtered model (very important for new model accuracy) + singleeyefitter::Sphere tempCircle = + eye_model_updaters[cam]->eyeModelFilter(eye_model_updaters[cam]->fitter().eye, eyes[cam], medianTotal, ignoreNewEye, originalModels[cam]); + + singleeyefitter::EyeModelFitter::Sphere filteredEye(tempCircle.centre, tempCircle.radius); + cv::RotatedRect rr_eye = eye_tracker::toImgCoord(singleeyefitter::toRotatedRect( + singleeyefitter::project(filteredEye, focal_length)), img, 1.0f); + cv::RotatedRect originalRadius = eye_tracker::toImgCoord(singleeyefitter::toRotatedRect( + singleeyefitter::project(originalModels[cam], focal_length)), img, 1.0f); //projection of last 3D radius into 2D coordinates + cv::RotatedRect newRadius = eye_tracker::toImgCoord(singleeyefitter::toRotatedRect( + singleeyefitter::project(tempCircle, focal_length)), img, 1.0f); //projection of new 3D radius into 2D coordinates - //update time, xdata, and ydata vectors for input into saccade detector - dataAdd(curr_circle.centre(0), 5, xData); - dataAdd(curr_circle.centre(1), 5, yData); - dataAdd(clock(), 5, timeData); - float intensity = 0; - - //to-Unity write - //std::ofstream myfile("C:\\Users\\O\\Documents\\Visual Studio 2013\\Projects\\EyeTrackerRealTime\\coordinates.txt"); - //std::ofstream myfile; - //myfile.open("C:\\Documents\\Osaka\\Research\\Presence 2017\\testcoordinates.txt", std::ios_base::app); - //myfile << "" << c_end.centre.x() << "," << c_end.centre.y() << "," << c_end.centre.z() - // << "," << filteredEye.centre[0] << "," << filteredEye.centre[1] << "," << filteredEye.centre[2] << - // std::endl; - //myfile.close(); + //note: radius of 0 returned from filter if model was bad + if (eyes[cam].size() > 0 && tempCircle.radius != 0 && std::abs(originalRadius.size.height - newRadius.size.height) < 30){ + //2D projections of radii were non-zero and did not differ significantly from originals + medianCircle = tempCircle; //pass on for model update + lastGoodEyes[cam] = tempCircle; //update last good eye (for possible use in next frame) + } + else if(lastGoodEyes[cam].radius != 0){ //filter returned 0 + medianCircle = lastGoodEyes[cam]; //use last known good eye model (last frame) + } + } + else if (lastGoodEyes[cam].radius != 0){ //new model not built + medianCircle = lastGoodEyes[cam]; //use last known good eye model + } + if (medianCircle.radius > 0) { // filtered eye was good -> render to screen and output coordinates + + eye_model_updaters[cam]->render_status(img_rgb_debug); + eye_model_updaters[cam]->setEye(medianCircle); //set eye model to + + curr_circle = eye_model_updaters[cam]->unproject(img, el, inlier_pts); + // 3D pupil (relative to filtered eye model) + singleeyefitter::Ellipse2D pupil_el(singleeyefitter::project(curr_circle, focal_length)); + cv::RotatedRect rr_pupil = eye_tracker::toImgCoord(singleeyefitter::toRotatedRect(pupil_el), img, 1.0f); + + singleeyefitter::EyeModelFitter::Sphere filteredEye(medianCircle.centre, medianCircle.radius); + cv::RotatedRect rr_eye = eye_tracker::toImgCoord(singleeyefitter::toRotatedRect( + singleeyefitter::project(filteredEye, focal_length)), img, 1.0f); + + cv::ellipse(img_rgb_debug, rr_eye, cv::Vec3b(255, 255, 255), 2, CV_AA); + cv::circle(img_rgb_debug, rr_eye.center, 3, cv::Vec3b(255, 32, 32), 2); // Eyeball center projection + singleeyefitter::EyeModelFitter::Circle c_end = curr_circle; + c_end.centre = curr_circle.centre + (10.0)*curr_circle.normal; + cv::line(img_rgb_debug, rr_eye.center, rr_pupil.center, cv::Vec3b(25, 22, 222), 3, CV_AA); + + //External code + //cout << "cam: " << cam << ", center x: " << rr_eye.center.x << ", center y: " << rr_eye.center.x << endl; + //update time, xdata, and ydata vectors for input into saccade detector + //dataAdd(curr_circle.centre(0), 5, xData); + //dataAdd(curr_circle.centre(1), 5, yData); + //dataAdd(clock(), 5, timeData); + //float intensity = 0; + +// if (cam == 0) {//append eye 0 model data to output string (3D pupil center in c_end; 3D eye center in filteredEye) + eyeVector[cam] << "" << c_end.centre.x() << "," << c_end.centre.y() << "," << c_end.centre.z() + << "," << filteredEye.centre[0] << "," << filteredEye.centre[1] << "," << filteredEye.centre[2]; + camSpheres[0 + cam * 3] = filteredEye.centre[0]; + camSpheres[1 + cam * 3] = filteredEye.centre[1]; + camSpheres[2 + cam * 3] = filteredEye.centre[2]; + }//end if: returned model radius was 0 } - }else{ + }else{//model not built, just render current ellipses eye_model_updaters[cam]->render_status(img_rgb_debug); cv::putText(img, "Sample #: " + std::to_string(eye_model_updaters[cam]->fitter_count()) + "/" + std::to_string(eye_model_updaters[cam]->fitter_end_count()), cv::Point(30, 440), cv::FONT_HERSHEY_SIMPLEX, 1.0, cv::Scalar(0, 128, 255), 2); } - float confidence = 0; - //outputVideo1 << img_rgb_debug; + //outputVideo1 << img_rgb_debug; //custom video write + + //show eye camera windows cv::imshow(window_names[cam], img_rgb_debug); - - } // Visualization + } // end loop for each camera - } // For each cameras - - // Compute FPS - frame_rate_counter.count(); - // Print current frame data - static int ss = 0; - if (ss++ > 100) { - std::cout << "Frame #" << frame_rate_counter.frame_count() << ", FPS=" << frame_rate_counter.fps() << std::endl; - ss = 0; + //OUTPUT DATA IS HERE******* + //each eyeVector string contains six comma separated values: + // eye center 0 (x, y, z); pupil center 0 (x, y, z); eye center 1 (x, y, z); pupil center 1 (x, y, z) + if (eyeVector[0].str().length() > 0 && eyeVector[1].str().length() > 0) {//check to ensure strings both have data (both eyes found) + std::ofstream myfile(folderPath); //to-Unity write + //std::ofstream myfile; //for recording experiment data + //myfile.open("C:\\Storage\\Research\\Eye Tracking\\testcoordinates.txt", std::ios_base::app); + myfile << "" << eyeVector[0].str() << "," << eyeVector[1].str() << endl; + myfile.close(); } - singleeyefitter::EyeModelFitter::Circle curr_circle; - singleeyefitter::EyeModelFitter::Circle c_end = curr_circle; - c_end.centre = curr_circle.centre + (10.0)*curr_circle.normal; // Unit: mm + // Compute FPS + //frame_rate_counter.count(); + //// Print current frame data + //static int ss = 0; + //if (ss++ > 100) { + // std::cout << "Frame #" << frame_rate_counter.frame_count() << ", FPS=" << frame_rate_counter.fps() << std::endl; + // ss = 0; + //} - - - + //cout << "dist: " << pupilFitter.getInterpupillaryDifference(cam0Sphere, cam1Sphere) << endl; + + //singleeyefitter::EyeModelFitter::Circle curr_circle; + //singleeyefitter::EyeModelFitter::Circle c_end = curr_circle; + //c_end.centre = curr_circle.centre + (10.0)*curr_circle.normal; // Unit: mm }// Main capture loop - outputVideo1.release(); - return 0; + //outputVideo1.release(); // custom video + return 0; }