🎉 initial commit

Author: pperle

.gitignore

@@ -0,0 +1,253 @@
+
+# Created by https://www.toptal.com/developers/gitignore/api/python,pycharm+all,visualstudiocode
+# Edit at https://www.toptal.com/developers/gitignore?templates=python,pycharm+all,visualstudiocode
+
+### PyCharm+all ###
+# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider
+# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
+
+# User-specific stuff
+.idea/**/workspace.xml
+.idea/**/tasks.xml
+.idea/**/usage.statistics.xml
+.idea/**/dictionaries
+.idea/**/shelf
+
+# AWS User-specific
+.idea/**/aws.xml
+
+# Generated files
+.idea/**/contentModel.xml
+
+# Sensitive or high-churn files
+.idea/**/dataSources/
+.idea/**/dataSources.ids
+.idea/**/dataSources.local.xml
+.idea/**/sqlDataSources.xml
+.idea/**/dynamic.xml
+.idea/**/uiDesigner.xml
+.idea/**/dbnavigator.xml
+
+# Gradle
+.idea/**/gradle.xml
+.idea/**/libraries
+
+# Gradle and Maven with auto-import
+# When using Gradle or Maven with auto-import, you should exclude module files,
+# since they will be recreated, and may cause churn.  Uncomment if using
+# auto-import.
+# .idea/artifacts
+# .idea/compiler.xml
+# .idea/jarRepositories.xml
+# .idea/modules.xml
+# .idea/*.iml
+# .idea/modules
+# *.iml
+# *.ipr
+
+# CMake
+cmake-build-*/
+
+# Mongo Explorer plugin
+.idea/**/mongoSettings.xml
+
+# File-based project format
+*.iws
+
+# IntelliJ
+out/
+
+# mpeltonen/sbt-idea plugin
+.idea_modules/
+
+# JIRA plugin
+atlassian-ide-plugin.xml
+
+# Cursive Clojure plugin
+.idea/replstate.xml
+
+# Crashlytics plugin (for Android Studio and IntelliJ)
+com_crashlytics_export_strings.xml
+crashlytics.properties
+crashlytics-build.properties
+fabric.properties
+
+# Editor-based Rest Client
+.idea/httpRequests
+
+# Android studio 3.1+ serialized cache file
+.idea/caches/build_file_checksums.ser
+
+### PyCharm+all Patch ###
+# Ignores the whole .idea folder and all .iml files
+# See https://github.com/joeblau/gitignore.io/issues/186 and https://github.com/joeblau/gitignore.io/issues/360
+
+.idea/
+
+# Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-249601023
+
+*.iml
+modules.xml
+.idea/misc.xml
+*.ipr
+
+# Sonarlint plugin
+.idea/sonarlint
+
+### Python ###
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+### VisualStudioCode ###
+.vscode/*
+!.vscode/settings.json
+!.vscode/tasks.json
+!.vscode/launch.json
+!.vscode/extensions.json
+*.code-workspace
+
+# Local History for Visual Studio Code
+.history/
+
+### VisualStudioCode Patch ###
+# Ignore all local history of files
+.history
+.ionide
+
+# End of https://www.toptal.com/developers/gitignore/api/python,pycharm+all,visualstudiocode

README.md

@@ -0,0 +1,22 @@
+# Collect Training and Calibration Data for Gaze Tracking
+
+This tool allows collecting gaze data necessary for personal calibration or training of eye-tracking models. It was developed as part of my master's thesis on [eye tracking with a monocular webcam](https://github.com/pperle/gaze-tracking).
+The [framework for the full gaze tracking pipeline](https://github.com/pperle/gaze-tracking-pipeline) is also available.
+
+The output is a folder with a CSV file containing the target that the person is looking at in pixels and the file name of the associated webcam image. For good calibration results, it is recommended to take at least 9 calibration images, the more, the better.
+
+## How to run
+
+1. `pip install -r requirements.txt`
+2. If necessary, calibrate the camera using the provided interactive script `python calibrate_camera.py`, see [Camera Calibration by OpenCV](https://docs.opencv.org/4.5.3/dc/dbb/tutorial_py_calibration.html).
+3. For higher accuracy, it is also advisable to calibrate the position of the screen as described by [Takahashiet al.](https://doi.org/10.2197/ipsjtcva.8.11), which provide an [OpenCV and matlab implementation](https://github.com/computer-vision/takahashi2012cvpr).
+4. `python main.py --base_path=./data/p00`
+   1. This was only tested on Ubuntu 20.10 and Ubuntu 21.04. If you are using macOS or Windows, you might have to supply the monitor parameters manually, e.g., `--monitor_mm=750,420 --monitor_pixels=1920,1080`, and adjust the `TargetOrientation` values in `utils.py`.
+5. Look at the screen and press the corresponding arrow key where the letter `E` is pointing at when the letter color changes from blue to orange. Please press the arrow key several times because sometimes OpenCV doesn't register the click the first time.
+6. Press the `q` key when the data collection is complete.
+
+![data collection example](./docs/demo.gif)
+
+7. Visualize the recorded data, image by image by running `python visualization.py --base_path=./data/p00`.
+
+![visualization example](./docs/3d_plot.gif)

camera_calibration.py

@@ -0,0 +1,106 @@
+import glob
+from datetime import datetime
+
+import cv2
+import numpy as np
+import yaml
+
+from webcam import WebcamSource
+
+
+def record_video(width: int, height: int, fps: int) -> None:
+    """
+    Create a mp4 video file with `width`x`height` and `fps` frames per second.
+    Shows a preview of the recording every 5 frames.
+
+    :param width: width of the video
+    :param height: height of the video
+    :param fps: frames per second
+    :return: None
+    """
+
+    source = WebcamSource(width=width, height=height, fps=fps, buffer_size=10)
+    video_writer = cv2.VideoWriter(f'{datetime.now().strftime("%Y-%m-%d_%H:%M:%S")}.mp4', cv2.VideoWriter_fourcc(*'MP4V'), fps, (width, height))
+    for idx, frame in enumerate(source):
+        video_writer.write(frame)
+        source.show(frame, only_print=idx % 5 != 0)
+
+
+def calibration(image_path, every_nth: int = 1, debug: bool = False, chessboard_grid_size=(7, 7)):
+    """
+    Perform camera calibration on the previously collected images.
+    Creates `calibration_matrix.yaml` with the camera intrinsic matrix and the distortion coefficients.
+
+    :param image_path: path to all png images
+    :param every_nth: only use every n_th image
+    :param debug: preview the matched chess patterns
+    :param chessboard_grid_size: size of chess pattern
+    :return:
+    """
+
+    x, y = chessboard_grid_size
+
+    # termination criteria
+    criteria = (cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER, 30, 0.001)
+
+    # prepare object points, like (0,0,0), (1,0,0), (2,0,0) ....,(6,5,0)
+    objp = np.zeros((y * x, 3), np.float32)
+    objp[:, :2] = np.mgrid[0:x, 0:y].T.reshape(-1, 2)
+
+    # Arrays to store object points and image points from all the images.
+    objpoints = []  # 3d point in real world space
+    imgpoints = []  # 2d points in image plane.
+
+    images = glob.glob(f'{image_path}/*.png')[::every_nth]
+
+    found = 0
+    for fname in images:
+        img = cv2.imread(fname)  # Capture frame-by-frame
+        gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
+
+        # Find the chess board corners
+        ret, corners = cv2.findChessboardCorners(gray, (x, y), None)
+
+        # If found, add object points, image points (after refining them)
+        if ret == True:
+            objpoints.append(objp)  # Certainly, every loop objp is the same, in 3D.
+            corners2 = cv2.cornerSubPix(gray, corners, (11, 11), (-1, -1), criteria)
+            imgpoints.append(corners2)
+
+            found += 1
+
+            if debug:
+                # Draw and display the corners
+                img = cv2.drawChessboardCorners(img, chessboard_grid_size, corners2, ret)
+                cv2.imshow('img', img)
+                cv2.waitKey(100)
+
+    print("Number of images used for calibration: ", found)
+
+    # When everything done, release the capture
+    cv2.destroyAllWindows()
+
+    # calibration
+    rms, mtx, dist, rvecs, tvecs = cv2.calibrateCamera(objpoints, imgpoints, gray.shape[::-1], None, None)
+    print('rms', rms)
+
+    # transform the matrix and distortion coefficients to writable lists
+    data = {
+        'rms': np.asarray(rms).tolist(),
+        'camera_matrix': np.asarray(mtx).tolist(),
+        'dist_coeff': np.asarray(dist).tolist()
+    }
+
+    # and save it to a file
+    with open("calibration_matrix.yaml", "w") as f:
+        yaml.dump(data, f)
+
+    print(data)
+
+
+if __name__ == '__main__':
+    # 1. record video
+    record_video(width=1280, height=720, fps=30)
+    # 2. split video into frames e.g. `ffmpeg -i 2021-10-15_10:30:00.mp4 -f image2 frames/video_01-%07d.png` and delete blurry images
+    # 3. run calibration on images
+    calibration('./frames', 30, debug=True)