mirror of
https://github.com/EyeTrackVR/OpenIris.git
synced 2025-11-04 15:39:42 +08:00
Add initial camera support and some high level documentation
This commit is contained in:
commit
3509317804
4
OpenEyeTracking/.obsidian/app.json
vendored
Normal file
4
OpenEyeTracking/.obsidian/app.json
vendored
Normal file
@ -0,0 +1,4 @@
|
||||
{
|
||||
"legacyEditor": false,
|
||||
"livePreview": true
|
||||
}
|
||||
3
OpenEyeTracking/.obsidian/appearance.json
vendored
Normal file
3
OpenEyeTracking/.obsidian/appearance.json
vendored
Normal file
@ -0,0 +1,3 @@
|
||||
{
|
||||
"baseFontSize": 16
|
||||
}
|
||||
15
OpenEyeTracking/.obsidian/core-plugins.json
vendored
Normal file
15
OpenEyeTracking/.obsidian/core-plugins.json
vendored
Normal file
@ -0,0 +1,15 @@
|
||||
[
|
||||
"file-explorer",
|
||||
"global-search",
|
||||
"switcher",
|
||||
"graph",
|
||||
"backlink",
|
||||
"page-preview",
|
||||
"note-composer",
|
||||
"command-palette",
|
||||
"editor-status",
|
||||
"markdown-importer",
|
||||
"word-count",
|
||||
"open-with-default-app",
|
||||
"file-recovery"
|
||||
]
|
||||
1
OpenEyeTracking/.obsidian/hotkeys.json
vendored
Normal file
1
OpenEyeTracking/.obsidian/hotkeys.json
vendored
Normal file
@ -0,0 +1 @@
|
||||
{}
|
||||
96
OpenEyeTracking/.obsidian/workspace
vendored
Normal file
96
OpenEyeTracking/.obsidian/workspace
vendored
Normal file
@ -0,0 +1,96 @@
|
||||
{
|
||||
"main": {
|
||||
"id": "76582a91acd97409",
|
||||
"type": "split",
|
||||
"children": [
|
||||
{
|
||||
"id": "9df6eb5a463bf556",
|
||||
"type": "leaf",
|
||||
"state": {
|
||||
"type": "markdown",
|
||||
"state": {
|
||||
"file": "Software.md",
|
||||
"mode": "source",
|
||||
"source": false
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"direction": "vertical"
|
||||
},
|
||||
"left": {
|
||||
"id": "aba372dbf4edb5fa",
|
||||
"type": "split",
|
||||
"children": [
|
||||
{
|
||||
"id": "e0497a81d44e0782",
|
||||
"type": "tabs",
|
||||
"children": [
|
||||
{
|
||||
"id": "a1f109ecf55429e3",
|
||||
"type": "leaf",
|
||||
"state": {
|
||||
"type": "file-explorer",
|
||||
"state": {}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "98fad3c11c00a054",
|
||||
"type": "leaf",
|
||||
"state": {
|
||||
"type": "search",
|
||||
"state": {
|
||||
"query": "",
|
||||
"matchingCase": false,
|
||||
"explainSearch": false,
|
||||
"collapseAll": false,
|
||||
"extraContext": false,
|
||||
"sortOrder": "alphabetical"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"direction": "horizontal",
|
||||
"width": 300
|
||||
},
|
||||
"right": {
|
||||
"id": "8576fab9e625b9b2",
|
||||
"type": "split",
|
||||
"children": [
|
||||
{
|
||||
"id": "29df0dc0be92d7ce",
|
||||
"type": "tabs",
|
||||
"children": [
|
||||
{
|
||||
"id": "391f68634caa74bb",
|
||||
"type": "leaf",
|
||||
"state": {
|
||||
"type": "backlink",
|
||||
"state": {
|
||||
"file": "Software.md",
|
||||
"collapseAll": false,
|
||||
"extraContext": false,
|
||||
"sortOrder": "alphabetical",
|
||||
"showSearch": false,
|
||||
"searchQuery": "",
|
||||
"backlinkCollapsed": false,
|
||||
"unlinkedCollapsed": true
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"direction": "horizontal",
|
||||
"width": 300,
|
||||
"collapsed": true
|
||||
},
|
||||
"active": "9df6eb5a463bf556",
|
||||
"lastOpenFiles": [
|
||||
"Software.md",
|
||||
"Goal and project description.md",
|
||||
"Hardware.md"
|
||||
]
|
||||
}
|
||||
7
OpenEyeTracking/Goal and project description.md
Normal file
7
OpenEyeTracking/Goal and project description.md
Normal file
@ -0,0 +1,7 @@
|
||||
The goal of this project is to bring an easy to set up hardware and software wise, fast and accurate eye and/or mouth tracking solution for any headset that allows for mounting the eye tracking cameras and/or mouth tracking camera in or under the headset (in case of mouth tracking).
|
||||
|
||||
It is also open source and open hardware meaning anyone can build it for them selfes and use it free of charge.
|
||||
|
||||
This project is highly inspired by the excelent Full Body Trakcing solution - [SlimeVR](https://docs.slimevr.dev)
|
||||
and is aimging to do the same - make the eye and mouth tracking technology independted of the headset so that anyone can use it with anything they want and also make it cheaper.
|
||||
|
||||
38
OpenEyeTracking/Hardware.md
Normal file
38
OpenEyeTracking/Hardware.md
Normal file
@ -0,0 +1,38 @@
|
||||
### Hardware is supposed to be:
|
||||
- Cheap
|
||||
- Off the shelf and readly available
|
||||
- Easy to work with
|
||||
- Fast enough for send a live stream
|
||||
|
||||
|
||||
### So far all the boxes have been check by:
|
||||
##### Base components:
|
||||
- esp32-cam
|
||||
- ov2640 IR with 160 degree FOV
|
||||
- esp-32 programmer
|
||||
|
||||
#### Alternative possibility:
|
||||
- raspberry pi zero
|
||||
- ov2640 IR with 160 FOV
|
||||
|
||||
#### Powering it on:
|
||||
Just like a [slimevr trakcer](https://docs.slimevr.dev/diy/components-guide.html)
|
||||
|
||||
- 2x1N5817 SCHOTTKY diodes
|
||||
- TP4056 USB-C charging board
|
||||
- a bipositional switch
|
||||
- A 800mAh Li-Po battery
|
||||
|
||||
This will make the eye tracker last for about 9 - 10h of playtime.
|
||||
|
||||
One could also power it though a USB port as the esp requires only 5V and 300-500ma
|
||||
|
||||
A usb battery or two lipos with a 5.1v power regulator should also do the job.
|
||||
|
||||
#### Radical alternatives for a camera
|
||||
|
||||
Shave off the IR filter off of any camera module:
|
||||
https://marksbench.com/electronics/removing-ir-filter-from-esp32-cam/
|
||||
|
||||
#### Ommited because of safety
|
||||
I'm ommiting any IR emmiters here purposfully - I have no idea if any of them are safe to use for prolonged periods of time and thus I leave them out.
|
||||
26
OpenEyeTracking/Software.md
Normal file
26
OpenEyeTracking/Software.md
Normal file
@ -0,0 +1,26 @@
|
||||
Aight, so this is not decided at all and serves as a "throw it all in and figure it as you go" document, so here it goes.
|
||||
|
||||
The goal is to:
|
||||
|
||||
- detect pupil position send that data though websockets / OSC back to Neos/Vrchat.
|
||||
- detect eye lids position and send that data though websockets / OSC back to Neos/Vrchat
|
||||
|
||||
Now, having that out of the way, how should we process the data? It's being sent to us as an uncompressed stream from the ESP.
|
||||
|
||||
We could use the OpenCV for image processing but what out detection?
|
||||
|
||||
DLib has a very good eye / face detection pre-trained model
|
||||
There are some ready-to-go solutions for openCV but won't they be too heavy?
|
||||
|
||||
How about training out own CNN model based on yey data sets?
|
||||
They would require labeling but they were used with great success by others.
|
||||
|
||||
Datasets - https://datagen.tech/blog/eye-datasets/
|
||||
TowardsDataScience showcasing DLib used for eye tracking (no eye lids) - https://towardsdatascience.com/real-time-eye-tracking-using-opencv-and-dlib-b504ca724ac6
|
||||
some more research using dlib - https://pyimagesearch.com/2017/04/24/eye-blink-detection-opencv-python-dlib/
|
||||
eyeloop, looks intereg - https://github.com/simonarvin/eyeloop
|
||||
|
||||
Now for some papers:
|
||||
http://stanford.edu/class/ee267/Spring2018/report_griffin_ramirez.pdf - eye tracing using CNN
|
||||
|
||||
For a stupid version, we could blink the eyes if there are no pupils detected? But that means no stupid faces :c
|
||||
46
main.py
Normal file
46
main.py
Normal file
@ -0,0 +1,46 @@
|
||||
import cv2 as cv
|
||||
import threading
|
||||
import numpy as np
|
||||
|
||||
|
||||
class ThreadedCamera:
|
||||
def __init__(self, camera_index=0):
|
||||
self.cam = cv.VideoCapture(camera_index)
|
||||
self.status = False
|
||||
self.frame = None
|
||||
|
||||
if not self.cam.isOpened():
|
||||
raise Exception("Could not connect to a camera")
|
||||
|
||||
self.cam.set(cv.CAP_PROP_BUFFERSIZE, 3)
|
||||
self.camera_thread = threading.Thread(target=self.update, args=(), daemon=True)
|
||||
self.camera_thread.start()
|
||||
|
||||
def update(self):
|
||||
while True:
|
||||
ret, frame = self.cam.read()
|
||||
if not ret:
|
||||
print("something went wrong with reading frame, exiting")
|
||||
break
|
||||
|
||||
self.status, self.frame = ret, frame
|
||||
|
||||
def display_frame(self):
|
||||
if self.frame is not None:
|
||||
image = cv.cvtColor(self.frame, cv.COLOR_BGR2GRAY)
|
||||
image = cv.resize(image, dsize=(int(self.frame.shape[1]/3), int(self.frame.shape[0]/3)))
|
||||
cv.imshow("frame", image)
|
||||
fps = self.cam.get(cv.CAP_PROP_FPS)
|
||||
print("Frames per second using video.get(cv2.CAP_PROP_FPS) : {0}".format(fps))
|
||||
if cv.waitKey(1) == ord("q"):
|
||||
exit()
|
||||
|
||||
|
||||
def main():
|
||||
camera = ThreadedCamera(0)
|
||||
while True:
|
||||
camera.display_frame()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
60
poetry.lock
generated
Normal file
60
poetry.lock
generated
Normal file
@ -0,0 +1,60 @@
|
||||
[[package]]
|
||||
name = "numpy"
|
||||
version = "1.22.2"
|
||||
description = "NumPy is the fundamental package for array computing with Python."
|
||||
category = "main"
|
||||
optional = false
|
||||
python-versions = ">=3.8"
|
||||
|
||||
[[package]]
|
||||
name = "opencv-python"
|
||||
version = "4.5.5.62"
|
||||
description = "Wrapper package for OpenCV python bindings."
|
||||
category = "main"
|
||||
optional = false
|
||||
python-versions = ">=3.6"
|
||||
|
||||
[package.dependencies]
|
||||
numpy = [
|
||||
{version = ">=1.21.2", markers = "python_version >= \"3.10\" or python_version >= \"3.6\" and platform_system == \"Darwin\" and platform_machine == \"arm64\""},
|
||||
{version = ">=1.19.3", markers = "python_version >= \"3.6\" and platform_system == \"Linux\" and platform_machine == \"aarch64\" or python_version >= \"3.9\""},
|
||||
{version = ">=1.14.5", markers = "python_version >= \"3.7\""},
|
||||
{version = ">=1.17.3", markers = "python_version >= \"3.8\""},
|
||||
]
|
||||
|
||||
[metadata]
|
||||
lock-version = "1.1"
|
||||
python-versions = "^3.9"
|
||||
content-hash = "326adfe81f905ea983945fc8d76917b75526d3082b50b43d26e378e00666bae2"
|
||||
|
||||
[metadata.files]
|
||||
numpy = [
|
||||
{file = "numpy-1.22.2-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:515a8b6edbb904594685da6e176ac9fbea8f73a5ebae947281de6613e27f1956"},
|
||||
{file = "numpy-1.22.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:76a4f9bce0278becc2da7da3b8ef854bed41a991f4226911a24a9711baad672c"},
|
||||
{file = "numpy-1.22.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:168259b1b184aa83a514f307352c25c56af111c269ffc109d9704e81f72e764b"},
|
||||
{file = "numpy-1.22.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3556c5550de40027d3121ebbb170f61bbe19eb639c7ad0c7b482cd9b560cd23b"},
|
||||
{file = "numpy-1.22.2-cp310-cp310-win_amd64.whl", hash = "sha256:aafa46b5a39a27aca566198d3312fb3bde95ce9677085efd02c86f7ef6be4ec7"},
|
||||
{file = "numpy-1.22.2-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:55535c7c2f61e2b2fc817c5cbe1af7cb907c7f011e46ae0a52caa4be1f19afe2"},
|
||||
{file = "numpy-1.22.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:60cb8e5933193a3cc2912ee29ca331e9c15b2da034f76159b7abc520b3d1233a"},
|
||||
{file = "numpy-1.22.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0b536b6840e84c1c6a410f3a5aa727821e6108f3454d81a5cd5900999ef04f89"},
|
||||
{file = "numpy-1.22.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2638389562bda1635b564490d76713695ff497242a83d9b684d27bb4a6cc9d7a"},
|
||||
{file = "numpy-1.22.2-cp38-cp38-win32.whl", hash = "sha256:6767ad399e9327bfdbaa40871be4254d1995f4a3ca3806127f10cec778bd9896"},
|
||||
{file = "numpy-1.22.2-cp38-cp38-win_amd64.whl", hash = "sha256:03ae5850619abb34a879d5f2d4bb4dcd025d6d8fb72f5e461dae84edccfe129f"},
|
||||
{file = "numpy-1.22.2-cp39-cp39-macosx_10_14_x86_64.whl", hash = "sha256:d76a26c5118c4d96e264acc9e3242d72e1a2b92e739807b3b69d8d47684b6677"},
|
||||
{file = "numpy-1.22.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:15efb7b93806d438e3bc590ca8ef2f953b0ce4f86f337ef4559d31ec6cf9d7dd"},
|
||||
{file = "numpy-1.22.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:badca914580eb46385e7f7e4e426fea6de0a37b9e06bec252e481ae7ec287082"},
|
||||
{file = "numpy-1.22.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:94dd11d9f13ea1be17bac39c1942f527cbf7065f94953cf62dfe805653da2f8f"},
|
||||
{file = "numpy-1.22.2-cp39-cp39-win32.whl", hash = "sha256:8cf33634b60c9cef346663a222d9841d3bbbc0a2f00221d6bcfd0d993d5543f6"},
|
||||
{file = "numpy-1.22.2-cp39-cp39-win_amd64.whl", hash = "sha256:59153979d60f5bfe9e4c00e401e24dfe0469ef8da6d68247439d3278f30a180f"},
|
||||
{file = "numpy-1.22.2-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4a176959b6e7e00b5a0d6f549a479f869829bfd8150282c590deee6d099bbb6e"},
|
||||
{file = "numpy-1.22.2.zip", hash = "sha256:076aee5a3763d41da6bef9565fdf3cb987606f567cd8b104aded2b38b7b47abf"},
|
||||
]
|
||||
opencv-python = [
|
||||
{file = "opencv-python-4.5.5.62.tar.gz", hash = "sha256:3efe232b32d5e1327e7c82bc6d61230737821c5190ce5c783e64a1bc8d514e18"},
|
||||
{file = "opencv_python-4.5.5.62-cp36-abi3-macosx_10_15_x86_64.whl", hash = "sha256:2601388def0d6b957cc30dd88f8ff74a5651ae6940dd9e488241608cfa2b15c7"},
|
||||
{file = "opencv_python-4.5.5.62-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:71fdc49df412b102d97f14927321309043c79c4a3582cce1dc803370ff9c39c0"},
|
||||
{file = "opencv_python-4.5.5.62-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:130cc75d56b29aa3c5de8b6ac438242dd2574ba6eaa8bccdffdcfd6b78632f7f"},
|
||||
{file = "opencv_python-4.5.5.62-cp36-abi3-win32.whl", hash = "sha256:3a75c7ad45b032eea0c72e389aac6dd435f5c87e87f60237095c083400bc23aa"},
|
||||
{file = "opencv_python-4.5.5.62-cp36-abi3-win_amd64.whl", hash = "sha256:c463d2276d8662b972d20ca9644702188507de200ca5405b89e1fe71c5c99989"},
|
||||
{file = "opencv_python-4.5.5.62-cp37-abi3-macosx_11_0_arm64.whl", hash = "sha256:ac92e743e22681f30001942d78512c1e39bce53dbffc504e5645fdc45c0f2c47"},
|
||||
]
|
||||
16
pyproject.toml
Normal file
16
pyproject.toml
Normal file
@ -0,0 +1,16 @@
|
||||
[tool.poetry]
|
||||
name = "diyeyetrackingtest"
|
||||
version = "0.1.0"
|
||||
description = ""
|
||||
authors = ["Zdzislaw Goik <z.goik@rynekpierwotny.pl>"]
|
||||
|
||||
[tool.poetry.dependencies]
|
||||
python = "^3.9"
|
||||
numpy = "^1.22.2"
|
||||
opencv-python = "^4.5.5"
|
||||
|
||||
[tool.poetry.dev-dependencies]
|
||||
|
||||
[build-system]
|
||||
requires = ["poetry-core>=1.0.0"]
|
||||
build-backend = "poetry.core.masonry.api"
|
||||
Loading…
Reference in New Issue
Block a user