diff --git a/Assets/HoloLensWithOpenCVForUnityExample/HoloLensArUcoExample.meta b/Assets/HoloLensWithOpenCVForUnityExample/HLArUcoExample.meta
similarity index 100%
rename from Assets/HoloLensWithOpenCVForUnityExample/HoloLensArUcoExample.meta
rename to Assets/HoloLensWithOpenCVForUnityExample/HLArUcoExample.meta
diff --git a/Assets/HoloLensWithOpenCVForUnityExample/HoloLensArUcoExample/CameraParameters.cs b/Assets/HoloLensWithOpenCVForUnityExample/HLArUcoExample/CameraParameters.cs
similarity index 96%
rename from Assets/HoloLensWithOpenCVForUnityExample/HoloLensArUcoExample/CameraParameters.cs
rename to Assets/HoloLensWithOpenCVForUnityExample/HLArUcoExample/CameraParameters.cs
index 4efe299..f0b69c2 100644
--- a/Assets/HoloLensWithOpenCVForUnityExample/HoloLensArUcoExample/CameraParameters.cs
+++ b/Assets/HoloLensWithOpenCVForUnityExample/HLArUcoExample/CameraParameters.cs
@@ -1,62 +1,62 @@
-using System;
-using OpenCVForUnity.CoreModule;
-
-namespace HoloLensWithOpenCVForUnityExample
-{
- [System.Serializable]
- public struct CameraParameters
- {
- public string calibration_date;
- public int frames_count;
- public int image_width;
- public int image_height;
- public int calibration_flags;
- public double[] camera_matrix;
- public double[] distortion_coefficients;
- public double avg_reprojection_error;
-
- public CameraParameters(int frames_count, int image_width, int image_height, int calibration_flags, double[] camera_matrix, double[] distortion_coefficients, double avg_reprojection_error)
- {
- this.calibration_date = DateTime.Now.ToString();
- this.frames_count = frames_count;
- this.image_width = image_width;
- this.image_height = image_height;
- this.calibration_flags = calibration_flags;
- this.camera_matrix = camera_matrix;
- this.distortion_coefficients = distortion_coefficients;
- this.avg_reprojection_error = avg_reprojection_error;
- }
-
- public CameraParameters(int frames_count, int image_width, int image_height, int calibration_flags, Mat camera_matrix, Mat distortion_coefficients, double avg_reprojection_error)
- {
- double[] camera_matrixArr = new double[camera_matrix.total()];
- camera_matrix.get(0, 0, camera_matrixArr);
-
- double[] distortion_coefficientsArr = new double[distortion_coefficients.total()];
- distortion_coefficients.get(0, 0, distortion_coefficientsArr);
-
- this.calibration_date = DateTime.Now.ToString();
- this.frames_count = frames_count;
- this.image_width = image_width;
- this.image_height = image_height;
- this.calibration_flags = calibration_flags;
- this.camera_matrix = camera_matrixArr;
- this.distortion_coefficients = distortion_coefficientsArr;
- this.avg_reprojection_error = avg_reprojection_error;
- }
-
- public Mat GetCameraMatrix()
- {
- Mat m = new Mat(3, 3, CvType.CV_64FC1);
- m.put(0, 0, camera_matrix);
- return m;
- }
-
- public Mat GetDistortionCoefficients()
- {
- Mat m = new Mat(distortion_coefficients.Length, 1, CvType.CV_64FC1);
- m.put(0, 0, distortion_coefficients);
- return m;
- }
- }
-}
+using System;
+using OpenCVForUnity.CoreModule;
+
+namespace HoloLensWithOpenCVForUnityExample
+{
+ [System.Serializable]
+ public struct CameraParameters
+ {
+ public string calibration_date;
+ public int frames_count;
+ public int image_width;
+ public int image_height;
+ public int calibration_flags;
+ public double[] camera_matrix;
+ public double[] distortion_coefficients;
+ public double avg_reprojection_error;
+
+ public CameraParameters(int frames_count, int image_width, int image_height, int calibration_flags, double[] camera_matrix, double[] distortion_coefficients, double avg_reprojection_error)
+ {
+ this.calibration_date = DateTime.Now.ToString();
+ this.frames_count = frames_count;
+ this.image_width = image_width;
+ this.image_height = image_height;
+ this.calibration_flags = calibration_flags;
+ this.camera_matrix = camera_matrix;
+ this.distortion_coefficients = distortion_coefficients;
+ this.avg_reprojection_error = avg_reprojection_error;
+ }
+
+ public CameraParameters(int frames_count, int image_width, int image_height, int calibration_flags, Mat camera_matrix, Mat distortion_coefficients, double avg_reprojection_error)
+ {
+ double[] camera_matrixArr = new double[camera_matrix.total()];
+ camera_matrix.get(0, 0, camera_matrixArr);
+
+ double[] distortion_coefficientsArr = new double[distortion_coefficients.total()];
+ distortion_coefficients.get(0, 0, distortion_coefficientsArr);
+
+ this.calibration_date = DateTime.Now.ToString();
+ this.frames_count = frames_count;
+ this.image_width = image_width;
+ this.image_height = image_height;
+ this.calibration_flags = calibration_flags;
+ this.camera_matrix = camera_matrixArr;
+ this.distortion_coefficients = distortion_coefficientsArr;
+ this.avg_reprojection_error = avg_reprojection_error;
+ }
+
+ public Mat GetCameraMatrix()
+ {
+ Mat m = new Mat(3, 3, CvType.CV_64FC1);
+ m.put(0, 0, camera_matrix);
+ return m;
+ }
+
+ public Mat GetDistortionCoefficients()
+ {
+ Mat m = new Mat(distortion_coefficients.Length, 1, CvType.CV_64FC1);
+ m.put(0, 0, distortion_coefficients);
+ return m;
+ }
+ }
+}
diff --git a/Assets/HoloLensWithOpenCVForUnityExample/HoloLensArUcoExample/CameraParameters.cs.meta b/Assets/HoloLensWithOpenCVForUnityExample/HLArUcoExample/CameraParameters.cs.meta
similarity index 100%
rename from Assets/HoloLensWithOpenCVForUnityExample/HoloLensArUcoExample/CameraParameters.cs.meta
rename to Assets/HoloLensWithOpenCVForUnityExample/HLArUcoExample/CameraParameters.cs.meta
diff --git a/Assets/HoloLensWithOpenCVForUnityExample/HoloLensArUcoExample/HoloLensArUcoCameraCalibrationExample.cs b/Assets/HoloLensWithOpenCVForUnityExample/HLArUcoExample/HLArUcoCameraCalibrationExample.cs
similarity index 96%
rename from Assets/HoloLensWithOpenCVForUnityExample/HoloLensArUcoExample/HoloLensArUcoCameraCalibrationExample.cs
rename to Assets/HoloLensWithOpenCVForUnityExample/HLArUcoExample/HLArUcoCameraCalibrationExample.cs
index 1359ce8..388dc9b 100644
--- a/Assets/HoloLensWithOpenCVForUnityExample/HoloLensArUcoExample/HoloLensArUcoCameraCalibrationExample.cs
+++ b/Assets/HoloLensWithOpenCVForUnityExample/HLArUcoExample/HLArUcoCameraCalibrationExample.cs
@@ -1,1256 +1,1262 @@
-using UnityEngine;
-using UnityEngine.UI;
-using UnityEngine.SceneManagement;
-using System;
-using System.Collections;
-using System.Collections.Generic;
-using System.Xml.Serialization;
-using System.IO;
-using System.Linq;
-using OpenCVForUnity.CoreModule;
-using OpenCVForUnity.ArucoModule;
-using OpenCVForUnity.UnityUtils.Helper;
-using OpenCVForUnity.ImgprocModule;
-using OpenCVForUnity.UnityUtils;
-using OpenCVForUnity.Calib3dModule;
-using OpenCVForUnity.ImgcodecsModule;
-using HoloLensWithOpenCVForUnity.UnityUtils.Helper;
-using Microsoft.MixedReality.Toolkit.Input;
-
-namespace HoloLensWithOpenCVForUnityExample
-{
- ///
- /// HoloLens ArUco Camera Calibration Example
- /// An example of camera calibration using the aruco module.
- /// Referring to https://github.com/opencv/opencv_contrib/blob/master/modules/aruco/samples/calibrate_camera.cpp.
- /// https://github.com/opencv/opencv/blob/master/samples/cpp/tutorial_code/calib3d/camera_calibration/camera_calibration.cpp
- /// https://docs.opencv.org/3.2.0/da/d13/tutorial_aruco_calibration.html
- /// https://docs.opencv.org/3.4.0/d7/d21/tutorial_interactive_calibration.html
- ///
- [RequireComponent(typeof(HololensCameraStreamToMatHelper))]
- public class HoloLensArUcoCameraCalibrationExample : MonoBehaviour
- {
- ///
- /// The preview quad.
- ///
- public GameObject previewQuad;
-
- ///
- /// The marker type.
- ///
- public MarkerType markerType = MarkerType.ChArUcoBoard;
-
- ///
- /// The marker type dropdown.
- ///
- public Dropdown markerTypeDropdown;
-
- ///
- /// The dictionary identifier.
- ///
- public ArUcoDictionary dictionaryId = ArUcoDictionary.DICT_6X6_250;
-
- ///
- /// The dictionary id dropdown.
- ///
- public Dropdown dictionaryIdDropdown;
-
- ///
- /// Number of squares in X direction.
- ///
- public NumberOfSquaresX squaresX = NumberOfSquaresX.X_5;
-
- ///
- /// The squares X dropdown.
- ///
- public Dropdown squaresXDropdown;
-
- ///
- /// Number of squares in X direction.
- ///
- public NumberOfSquaresY squaresY = NumberOfSquaresY.Y_7;
-
- ///
- /// The squares X dropdown.
- ///
- public Dropdown squaresYDropdown;
-
- ///
- /// The save path input field.
- ///
- public InputField savePathInputField;
-
- ///
- /// Determines if refine marker detection. (only valid for ArUco boards)
- ///
- public bool refineMarkerDetection = true;
-
-
- [Header("Extra Option")]
-
- ///
- /// Determines if calibrates camera using the list of calibration images.
- ///
- [TooltipAttribute("Determines if calibrates camera using the list of calibration images.")]
- public bool isImagesInputMode = false;
-
- ///
- /// The calibration images directory path.
- /// Set a relative directory path from the starting point of the "StreamingAssets" folder. e.g. "calibration_images/".
- ///
- [TooltipAttribute("Set a relative directory path from the starting point of the \"StreamingAssets\" folder. e.g. \"calibration_images\"")]
- public string calibrationImagesDirectory = "calibration_images";
-
- ///
- /// The texture.
- ///
- Texture2D texture;
-
- ///
- /// The webcam texture to mat helper.
- ///
- HololensCameraStreamToMatHelper webCamTextureToMatHelper;
-
- ///
- /// The bgr mat.
- ///
- Mat bgrMat;
-
- ///
- /// The rgba mat.
- ///
- Mat rgbaMat;
-
- ///
- /// The cameraparam matrix.
- ///
- Mat camMatrix;
-
- ///
- /// The distortion coeffs.
- ///
- MatOfDouble distCoeffs;
-
- ///
- /// The identifiers.
- ///
- Mat ids;
-
- ///
- /// The corners.
- ///
- List corners;
-
- ///
- /// The rejected corners.
- ///
- List rejectedCorners;
-
- ///
- /// The rvecs.
- ///
- List rvecs;
-
- ///
- /// The tvecs.
- ///
- List tvecs;
-
- ///
- /// The detector parameters.
- ///
- DetectorParameters detectorParams;
-
- ///
- /// The dictionary.
- ///
- Dictionary dictionary;
-
- ///
- /// The recovered identifiers.
- ///
- Mat recoveredIdxs;
-
- const int calibrationFlags = 0;// Calib3d.CALIB_FIX_K3 | Calib3d.CALIB_FIX_K4 | Calib3d.CALIB_FIX_K5
- double repErr = 0;
- bool shouldCaptureFrame = false;
-
- // for ChArUcoBoard.
- // chessboard square side length (normally in meters)
- const float chArUcoBoradSquareLength = 0.04f;
- // marker side length (same unit than squareLength)
- const float chArUcoBoradMarkerLength = 0.02f;
- const int charucoMinMarkers = 2;
- Mat charucoCorners;
- Mat charucoIds;
- CharucoBoard charucoBoard;
- List> allCorners;
- List allIds;
- List allImgs;
-
- // for OthearMarkers.
- // square size in some user-defined units (1 by default)
- const float squareSize = 1f;
- List imagePoints;
- bool isInitialized = false;
- bool isCalibrating = false;
-
- // Use this for initialization
- IEnumerator Start()
- {
- webCamTextureToMatHelper = gameObject.GetComponent();
- webCamTextureToMatHelper.outputColorFormat = WebCamTextureToMatHelper.ColorFormat.GRAY;
-
- // fix the screen orientation.
- Screen.orientation = ScreenOrientation.LandscapeLeft;
- // wait for the screen orientation to change.
- yield return null;
-
- if (markerTypeDropdown.value != (int)markerType || dictionaryIdDropdown.value != (int)dictionaryId
- || squaresXDropdown.value != (int)squaresX - 1 || squaresYDropdown.value != (int)squaresY - 1)
- {
- markerTypeDropdown.value = (int)markerType;
- dictionaryIdDropdown.value = (int)dictionaryId;
- squaresXDropdown.value = (int)squaresX - 1;
- squaresYDropdown.value = (int)squaresY - 1;
- }
- dictionaryIdDropdown.interactable = (markerType == MarkerType.ChArUcoBoard);
-
-#if UNITY_WEBGL && !UNITY_EDITOR
- isImagesInputMode = false;
-#endif
- if (isImagesInputMode)
- {
- isImagesInputMode = InitializeImagesInputMode();
- }
-
- if (!isImagesInputMode)
- {
- webCamTextureToMatHelper.Initialize();
- }
- }
-
- ///
- /// Raises the webcam texture to mat helper initialized event.
- ///
- public void OnWebCamTextureToMatHelperInitialized()
- {
- Debug.Log("OnWebCamTextureToMatHelperInitialized");
-
- Mat webCamTextureMat = webCamTextureToMatHelper.GetMat();
-
- InitializeCalibraton(webCamTextureMat);
-
- // if WebCamera is frontFaceing, flip Mat.
- webCamTextureToMatHelper.flipHorizontal = webCamTextureToMatHelper.IsFrontFacing();
- }
-
- ///
- /// Raises the webcam texture to mat helper disposed event.
- ///
- public void OnWebCamTextureToMatHelperDisposed()
- {
- Debug.Log("OnWebCamTextureToMatHelperDisposed");
-
- DisposeCalibraton();
- }
-
- ///
- /// Raises the webcam texture to mat helper error occurred event.
- ///
- /// Error code.
- public void OnWebCamTextureToMatHelperErrorOccurred(WebCamTextureToMatHelper.ErrorCode errorCode)
- {
- Debug.Log("OnWebCamTextureToMatHelperErrorOccurred " + errorCode);
- }
-
- // Update is called once per frame
- void Update()
- {
- if (isImagesInputMode)
- return;
-
- if (webCamTextureToMatHelper.IsPlaying() && webCamTextureToMatHelper.DidUpdateThisFrame())
- {
-
- Mat grayMat = webCamTextureToMatHelper.GetMat();
-
- if (shouldCaptureFrame)
- {
- shouldCaptureFrame = false;
- Mat frameMat = grayMat.clone();
- double e = CaptureFrame(frameMat);
- if (e > 0)
- repErr = e;
- }
-
- DrawFrame(grayMat, bgrMat);
- Imgproc.cvtColor(bgrMat, rgbaMat, Imgproc.COLOR_BGR2RGBA);
-
- Utils.fastMatToTexture2D(rgbaMat, texture);
- }
- }
-
- private void InitializeCalibraton(Mat frameMat)
- {
- Debug.Log("Screen.width " + Screen.width + " Screen.height " + Screen.height + " Screen.orientation " + Screen.orientation);
-
- float width = frameMat.width();
- float height = frameMat.height();
-
- texture = new Texture2D(frameMat.cols(), frameMat.rows(), TextureFormat.RGBA32, false);
- texture.wrapMode = TextureWrapMode.Clamp;
-
- previewQuad.GetComponent().material.mainTexture = texture;
- previewQuad.transform.localScale = new Vector3(0.2f * width / height, 0.2f, 1);
-
- float imageSizeScale = 1.0f;
-
- // set cameraparam.
- camMatrix = CreateCameraMatrix(width, height);
- Debug.Log("camMatrix " + camMatrix.dump());
-
- distCoeffs = new MatOfDouble(0, 0, 0, 0, 0);
- Debug.Log("distCoeffs " + distCoeffs.dump());
-
- // calibration camera.
- Size imageSize = new Size(width * imageSizeScale, height * imageSizeScale);
- double apertureWidth = 0;
- double apertureHeight = 0;
- double[] fovx = new double[1];
- double[] fovy = new double[1];
- double[] focalLength = new double[1];
- Point principalPoint = new Point(0, 0);
- double[] aspectratio = new double[1];
-
- Calib3d.calibrationMatrixValues(camMatrix, imageSize, apertureWidth, apertureHeight, fovx, fovy, focalLength, principalPoint, aspectratio);
-
- Debug.Log("imageSize " + imageSize.ToString());
- Debug.Log("apertureWidth " + apertureWidth);
- Debug.Log("apertureHeight " + apertureHeight);
- Debug.Log("fovx " + fovx[0]);
- Debug.Log("fovy " + fovy[0]);
- Debug.Log("focalLength " + focalLength[0]);
- Debug.Log("principalPoint " + principalPoint.ToString());
- Debug.Log("aspectratio " + aspectratio[0]);
-
-
- bgrMat = new Mat(frameMat.rows(), frameMat.cols(), CvType.CV_8UC3);
- rgbaMat = new Mat(frameMat.rows(), frameMat.cols(), CvType.CV_8UC4);
- ids = new Mat();
- corners = new List();
- rejectedCorners = new List();
- rvecs = new List();
- tvecs = new List();
-
- detectorParams = DetectorParameters.create();
- detectorParams.set_cornerRefinementMethod(1);// do cornerSubPix() of OpenCV.
- dictionary = Aruco.getPredefinedDictionary((int)dictionaryId);
-
- recoveredIdxs = new Mat();
-
- charucoCorners = new Mat();
- charucoIds = new Mat();
- charucoBoard = CharucoBoard.create((int)squaresX, (int)squaresY, chArUcoBoradSquareLength, chArUcoBoradMarkerLength, dictionary);
-
-
- allCorners = new List>();
- allIds = new List();
- allImgs = new List();
-
- imagePoints = new List();
-
- isInitialized = true;
- }
-
- private void DisposeCalibraton()
- {
- ResetCalibration();
-
- if (bgrMat != null)
- bgrMat.Dispose();
- if (rgbaMat != null)
- rgbaMat.Dispose();
- if (ids != null)
- ids.Dispose();
- foreach (var item in corners)
- {
- item.Dispose();
- }
- corners.Clear();
- foreach (var item in rejectedCorners)
- {
- item.Dispose();
- }
- rejectedCorners.Clear();
- foreach (var item in rvecs)
- {
- item.Dispose();
- }
- rvecs.Clear();
- foreach (var item in tvecs)
- {
- item.Dispose();
- }
- tvecs.Clear();
-
- if (recoveredIdxs != null)
- recoveredIdxs.Dispose();
-
- if (charucoCorners != null)
- charucoCorners.Dispose();
- if (charucoIds != null)
- charucoIds.Dispose();
- if (charucoBoard != null)
- charucoBoard.Dispose();
-
- isInitialized = false;
- }
-
- private void DrawFrame(Mat grayMat, Mat bgrMat)
- {
- Imgproc.cvtColor(grayMat, bgrMat, Imgproc.COLOR_GRAY2BGR);
-
- switch (markerType)
- {
- default:
- case MarkerType.ChArUcoBoard:
- // detect markers.
- Aruco.detectMarkers(grayMat, dictionary, corners, ids, detectorParams, rejectedCorners, camMatrix, distCoeffs);
-
- // refine marker detection.
- if (refineMarkerDetection)
- {
- Aruco.refineDetectedMarkers(grayMat, charucoBoard, corners, ids, rejectedCorners, camMatrix, distCoeffs, 10f, 3f, true, recoveredIdxs, detectorParams);
- }
-
- // if at least one marker detected
- if (ids.total() > 0)
- {
- Aruco.interpolateCornersCharuco(corners, ids, grayMat, charucoBoard, charucoCorners, charucoIds, camMatrix, distCoeffs, charucoMinMarkers);
-
- // draw markers.
- Aruco.drawDetectedMarkers(bgrMat, corners, ids, new Scalar(0, 255, 0, 255));
- // if at least one charuco corner detected
- if (charucoIds.total() > 0)
- {
- Aruco.drawDetectedCornersCharuco(bgrMat, charucoCorners, charucoIds, new Scalar(0, 0, 255, 255));
- }
- }
- break;
- case MarkerType.ChessBoard:
- case MarkerType.CirclesGlid:
- case MarkerType.AsymmetricCirclesGlid:
- // detect markers.
- MatOfPoint2f points = new MatOfPoint2f();
- bool found = false;
-
- switch (markerType)
- {
- default:
- case MarkerType.ChessBoard:
- found = Calib3d.findChessboardCorners(grayMat, new Size((int)squaresX, (int)squaresY), points, Calib3d.CALIB_CB_ADAPTIVE_THRESH | Calib3d.CALIB_CB_FAST_CHECK | Calib3d.CALIB_CB_NORMALIZE_IMAGE);
- break;
- case MarkerType.CirclesGlid:
- found = Calib3d.findCirclesGrid(grayMat, new Size((int)squaresX, (int)squaresY), points, Calib3d.CALIB_CB_SYMMETRIC_GRID);
- break;
- case MarkerType.AsymmetricCirclesGlid:
- found = Calib3d.findCirclesGrid(grayMat, new Size((int)squaresX, (int)squaresY), points, Calib3d.CALIB_CB_ASYMMETRIC_GRID);
- break;
- }
-
- if (found)
- {
- if (markerType == MarkerType.ChessBoard)
- Imgproc.cornerSubPix(grayMat, points, new Size(5, 5), new Size(-1, -1), new TermCriteria(TermCriteria.EPS + TermCriteria.COUNT, 30, 0.1));
-
- // draw markers.
- Calib3d.drawChessboardCorners(bgrMat, new Size((int)squaresX, (int)squaresY), points, found);
- }
- break;
- }
-
- double[] camMatrixArr = new double[(int)camMatrix.total()];
- camMatrix.get(0, 0, camMatrixArr);
- double[] distCoeffsArr = new double[(int)distCoeffs.total()];
- distCoeffs.get(0, 0, distCoeffsArr);
-
- int ff = Imgproc.FONT_HERSHEY_SIMPLEX;
- double fs = 0.4;
- Scalar c = new Scalar(255, 255, 255, 255);
- int t = 0;
- int lt = Imgproc.LINE_AA;
- bool blo = false;
- int frameCount = (markerType == MarkerType.ChArUcoBoard) ? allCorners.Count : imagePoints.Count;
- Imgproc.putText(bgrMat, frameCount + " FRAME CAPTURED", new Point(bgrMat.cols() - 300, 20), ff, fs, c, t, lt, blo);
- Imgproc.putText(bgrMat, "IMAGE_WIDTH: " + bgrMat.width(), new Point(bgrMat.cols() - 300, 40), ff, fs, c, t, lt, blo);
- Imgproc.putText(bgrMat, "IMAGE_HEIGHT: " + bgrMat.height(), new Point(bgrMat.cols() - 300, 60), ff, fs, c, t, lt, blo);
- Imgproc.putText(bgrMat, "CALIBRATION_FLAGS: " + calibrationFlags, new Point(bgrMat.cols() - 300, 80), ff, fs, c, t, lt, blo);
-
- Imgproc.putText(bgrMat, "CAMERA_MATRIX: ", new Point(bgrMat.cols() - 300, 100), ff, fs, c, t, lt, blo);
- for (int i = 0; i < camMatrixArr.Length; i = i + 3)
- {
- Imgproc.putText(bgrMat, " " + camMatrixArr[i] + ", " + camMatrixArr[i + 1] + ", " + camMatrixArr[i + 2] + ",", new Point(bgrMat.cols() - 300, 120 + 20 * i / 3), ff, fs, c, t, lt, blo);
- }
- Imgproc.putText(bgrMat, "DISTORTION_COEFFICIENTS: ", new Point(bgrMat.cols() - 300, 180), ff, fs, c, t, lt, blo);
- for (int i = 0; i < distCoeffsArr.Length; ++i)
- {
- Imgproc.putText(bgrMat, " " + distCoeffsArr[i] + ",", new Point(bgrMat.cols() - 300, 200 + 20 * i), ff, fs, c, t, lt, blo);
- }
- Imgproc.putText(bgrMat, "AVG_REPROJECTION_ERROR: " + repErr, new Point(bgrMat.cols() - 300, 300), ff, fs, c, t, lt, blo);
-
- if (frameCount == 0)
- Imgproc.putText(bgrMat, "To calibration start, please press the calibration button or do air tap gesture!", new Point(5, bgrMat.rows() - 10), Imgproc.FONT_HERSHEY_SIMPLEX, 0.5, new Scalar(255, 255, 255, 255), 1, Imgproc.LINE_AA, false);
- }
-
- private double CaptureFrame(Mat frameMat)
- {
- double repErr = -1;
-
- switch (markerType)
- {
- default:
- case MarkerType.ChArUcoBoard:
- List corners = new List();
- Mat ids = new Mat();
- Aruco.detectMarkers(frameMat, dictionary, corners, ids, detectorParams, rejectedCorners, camMatrix, distCoeffs);
-
- if (refineMarkerDetection)
- {
- Aruco.refineDetectedMarkers(frameMat, charucoBoard, corners, ids, rejectedCorners, camMatrix, distCoeffs, 10f, 3f, true, recoveredIdxs, detectorParams);
- }
-
- if (ids.total() > 0)
- {
- Debug.Log("Frame captured.");
-
- allCorners.Add(corners);
- allIds.Add(ids);
- allImgs.Add(frameMat);
- }
- else
- {
-
- Debug.Log("Invalid frame.");
-
- frameMat.Dispose();
- if (ids != null)
- ids.Dispose();
- foreach (var item in corners)
- {
- item.Dispose();
- }
- corners.Clear();
-
- return -1;
- }
-
- // calibrate camera using aruco markers
- //double arucoRepErr = CalibrateCameraAruco (allCorners, allIds, charucoBoard, frameMat.size(), camMatrix, distCoeffs, rvecs, tvecs, calibrationFlags);
- //Debug.Log ("arucoRepErr: " + arucoRepErr);
-
- // calibrate camera using charuco
- repErr = CalibrateCameraCharuco(allCorners, allIds, charucoBoard, frameMat.size(), camMatrix, distCoeffs, rvecs, tvecs, calibrationFlags, calibrationFlags);
-
- break;
- case MarkerType.ChessBoard:
- case MarkerType.CirclesGlid:
- case MarkerType.AsymmetricCirclesGlid:
-
- MatOfPoint2f points = new MatOfPoint2f();
- Size patternSize = new Size((int)squaresX, (int)squaresY);
-
- bool found = false;
- switch (markerType)
- {
- default:
- case MarkerType.ChessBoard:
- found = Calib3d.findChessboardCorners(frameMat, patternSize, points, Calib3d.CALIB_CB_ADAPTIVE_THRESH | Calib3d.CALIB_CB_FAST_CHECK | Calib3d.CALIB_CB_NORMALIZE_IMAGE);
- break;
- case MarkerType.CirclesGlid:
- found = Calib3d.findCirclesGrid(frameMat, patternSize, points, Calib3d.CALIB_CB_SYMMETRIC_GRID);
- break;
- case MarkerType.AsymmetricCirclesGlid:
- found = Calib3d.findCirclesGrid(frameMat, patternSize, points, Calib3d.CALIB_CB_ASYMMETRIC_GRID);
- break;
- }
-
- if (found)
- {
- Debug.Log("Frame captured.");
- if (markerType == MarkerType.ChessBoard)
- Imgproc.cornerSubPix(frameMat, points, new Size(5, 5), new Size(-1, -1), new TermCriteria(TermCriteria.EPS + TermCriteria.COUNT, 30, 0.1));
-
- imagePoints.Add(points);
- allImgs.Add(frameMat);
- }
- else
- {
- Debug.Log("Invalid frame.");
- frameMat.Dispose();
- if (points != null)
- points.Dispose();
- return -1;
- }
-
- if (imagePoints.Count < 1)
- {
- Debug.Log("Not enough points for calibration.");
- repErr = -1;
- }
- else
- {
-
- MatOfPoint3f objectPoint = new MatOfPoint3f(new Mat(imagePoints[0].rows(), 1, CvType.CV_32FC3));
- CalcChessboardCorners(patternSize, squareSize, objectPoint, markerType);
-
- List objectPoints = new List();
- for (int i = 0; i < imagePoints.Count; ++i)
- {
- objectPoints.Add(objectPoint);
- }
-
- repErr = Calib3d.calibrateCamera(objectPoints, imagePoints, frameMat.size(), camMatrix, distCoeffs, rvecs, tvecs, calibrationFlags);
- objectPoint.Dispose();
- }
-
- break;
- }
-
- Debug.Log("repErr: " + repErr);
- Debug.Log("camMatrix: " + camMatrix.dump());
- Debug.Log("distCoeffs: " + distCoeffs.dump());
-
- return repErr;
- }
-
- private double CalibrateCameraAruco(List> allCorners, List allIds, CharucoBoard board, Size imageSize, Mat cameraMatrix, Mat distCoeffs, List rvecs = null, List tvecs = null, int calibrationFlags = 0)
- {
- // prepare data for calibration
- int nFrames = allCorners.Count;
- int allLen = 0;
- int[] markerCounterPerFrameArr = new int[allCorners.Count];
- for (int i = 0; i < nFrames; ++i)
- {
- markerCounterPerFrameArr[i] = allCorners[i].Count;
- allLen += allCorners[i].Count;
- }
-
- int[] allIdsConcatenatedArr = new int[allLen];
- int index = 0;
- for (int j = 0; j < allIds.Count; ++j)
- {
- int[] idsArr = new int[(int)allIds[j].total()];
- allIds[j].get(0, 0, idsArr);
-
- for (int k = 0; k < idsArr.Length; ++k)
- {
- allIdsConcatenatedArr[index + k] = (int)idsArr[k];
- }
- index += idsArr.Length;
- }
-
- using (Mat allIdsConcatenated = new Mat(1, allLen, CvType.CV_32SC1))
- using (Mat markerCounterPerFrame = new Mat(1, nFrames, CvType.CV_32SC1))
- {
-
- List allCornersConcatenated = new List();
- foreach (var c in allCorners)
- {
- foreach (var m in c)
- {
- allCornersConcatenated.Add(m);
- }
- }
-
- allIdsConcatenated.put(0, 0, allIdsConcatenatedArr);
- markerCounterPerFrame.put(0, 0, markerCounterPerFrameArr);
-
- if (rvecs == null) rvecs = new List();
- if (tvecs == null) tvecs = new List();
-
- return Aruco.calibrateCameraAruco(allCornersConcatenated, allIdsConcatenated, markerCounterPerFrame, board, imageSize, cameraMatrix, distCoeffs, rvecs, tvecs, calibrationFlags);
- }
- }
-
- private double CalibrateCameraCharuco(List> allCorners, List allIds, CharucoBoard board, Size imageSize, Mat cameraMatrix, Mat distCoeffs, List rvecs = null, List tvecs = null, int calibrationFlags = 0, int minMarkers = 2)
- {
- // prepare data for charuco calibration
- int nFrames = allCorners.Count;
- List allCharucoCorners = new List();
- List allCharucoIds = new List();
- List filteredImages = new List();
-
- for (int i = 0; i < nFrames; ++i)
- {
- // interpolate using camera parameters
- Mat currentCharucoCorners = new Mat();
- Mat currentCharucoIds = new Mat();
-
- Aruco.interpolateCornersCharuco(allCorners[i], allIds[i], allImgs[i], board, currentCharucoCorners, currentCharucoIds, cameraMatrix, distCoeffs, minMarkers);
-
- if (charucoIds.total() > 0)
- {
- allCharucoCorners.Add(currentCharucoCorners);
- allCharucoIds.Add(currentCharucoIds);
- filteredImages.Add(allImgs[i]);
- }
- else
- {
- currentCharucoCorners.Dispose();
- currentCharucoIds.Dispose();
- }
- }
-
- if (allCharucoCorners.Count < 1)
- {
- Debug.Log("Not enough corners for calibration.");
- return -1;
- }
-
- if (rvecs == null) rvecs = new List();
- if (tvecs == null) tvecs = new List();
-
- return Aruco.calibrateCameraCharuco(allCharucoCorners, allCharucoIds, board, imageSize, cameraMatrix, distCoeffs, rvecs, tvecs, calibrationFlags);
- }
-
- private void ResetCalibration()
- {
- foreach (var corners in allCorners)
- {
- foreach (var item in corners)
- {
- item.Dispose();
- }
- }
- allCorners.Clear();
-
- foreach (var item in allIds)
- {
- item.Dispose();
- }
- allIds.Clear();
-
- foreach (var item in allImgs)
- {
- item.Dispose();
- }
- allImgs.Clear();
-
- repErr = 0;
- camMatrix = CreateCameraMatrix(bgrMat.width(), bgrMat.height());
- distCoeffs = new MatOfDouble(0, 0, 0, 0, 0);
-
- foreach (var item in imagePoints)
- {
- item.Dispose();
- }
- imagePoints.Clear();
- }
-
- private Mat CreateCameraMatrix(float width, float height)
- {
- int max_d = (int)Mathf.Max(width, height);
- double fx = max_d;
- double fy = max_d;
- double cx = width / 2.0f;
- double cy = height / 2.0f;
-
- Mat camMatrix = new Mat(3, 3, CvType.CV_64FC1);
- camMatrix.put(0, 0, fx);
- camMatrix.put(0, 1, 0);
- camMatrix.put(0, 2, cx);
- camMatrix.put(1, 0, 0);
- camMatrix.put(1, 1, fy);
- camMatrix.put(1, 2, cy);
- camMatrix.put(2, 0, 0);
- camMatrix.put(2, 1, 0);
- camMatrix.put(2, 2, 1.0f);
-
- return camMatrix;
- }
-
- private void CalcChessboardCorners(Size patternSize, float squareSize, MatOfPoint3f corners, MarkerType markerType)
- {
- if ((int)(patternSize.width * patternSize.height) != corners.rows())
- {
- Debug.Log("Invalid corners size.");
- corners.create((int)(patternSize.width * patternSize.height), 1, CvType.CV_32FC3);
- }
-
- const int cn = 3;
- float[] cornersArr = new float[corners.rows() * cn];
- int width = (int)patternSize.width;
- int height = (int)patternSize.height;
-
- switch (markerType)
- {
- default:
- case MarkerType.ChessBoard:
- case MarkerType.CirclesGlid:
- for (int i = 0; i < height; ++i)
- {
- for (int j = 0; j < width; ++j)
- {
- cornersArr[(i * width * cn) + (j * cn)] = j * squareSize;
- cornersArr[(i * width * cn) + (j * cn) + 1] = i * squareSize;
- cornersArr[(i * width * cn) + (j * cn) + 2] = 0;
- }
- }
- corners.put(0, 0, cornersArr);
-
- break;
- case MarkerType.AsymmetricCirclesGlid:
- for (int i = 0; i < height; ++i)
- {
- for (int j = 0; j < width; ++j)
- {
- cornersArr[(i * width * cn) + (j * cn)] = (2 * j + i % 2) * squareSize;
- cornersArr[(i * width * cn) + (j * cn) + 1] = i * squareSize;
- cornersArr[(i * width * cn) + (j * cn) + 2] = 0;
- }
- }
- corners.put(0, 0, cornersArr);
-
- break;
- }
- }
-
- private bool InitializeImagesInputMode()
- {
- if (isInitialized)
- DisposeCalibraton();
-
- if (String.IsNullOrEmpty(calibrationImagesDirectory))
- {
- Debug.LogWarning("When using the images input mode, please set a calibration images directory path.");
- return false;
- }
-
- string dirPath = Path.Combine(Application.streamingAssetsPath, calibrationImagesDirectory);
- if (!Directory.Exists(dirPath))
- {
- Debug.LogWarning("The directory does not exist.");
- return false;
- }
- string[] imageFiles = GetImageFilesInDirectory(dirPath);
- if (imageFiles.Length < 1)
- {
- Debug.LogWarning("The image file does not exist.");
- return false;
- }
-
- Uri rootPath = new Uri(Application.streamingAssetsPath + System.IO.Path.AltDirectorySeparatorChar);
- Uri fullPath = new Uri(imageFiles[0]);
- string relativePath = rootPath.MakeRelativeUri(fullPath).ToString();
-
- using (Mat gray = Imgcodecs.imread(Utils.getFilePath(relativePath), Imgcodecs.IMREAD_GRAYSCALE))
- {
-
- if (gray.total() == 0)
- {
- Debug.LogWarning("Invalid image file.");
- return false;
- }
-
- using (Mat bgr = new Mat(gray.size(), CvType.CV_8UC3))
- using (Mat bgra = new Mat(gray.size(), CvType.CV_8UC4))
- {
- InitializeCalibraton(gray);
-
- DrawFrame(gray, bgr);
- Imgproc.cvtColor(bgr, bgra, Imgproc.COLOR_BGR2RGBA);
- Utils.fastMatToTexture2D(bgra, texture);
- }
- }
- return true;
- }
-
- private IEnumerator CalibrateCameraUsingImages()
- {
- string dirPath = Path.Combine(Application.streamingAssetsPath, calibrationImagesDirectory);
- string[] imageFiles = GetImageFilesInDirectory(dirPath);
- if (imageFiles.Length < 1)
- yield break;
-
- isCalibrating = true;
- markerTypeDropdown.interactable = dictionaryIdDropdown.interactable = squaresXDropdown.interactable = squaresYDropdown.interactable = false;
-
- Uri rootPath = new Uri(Application.streamingAssetsPath + System.IO.Path.AltDirectorySeparatorChar);
-
- foreach (var path in imageFiles)
- {
-
- Uri fullPath = new Uri(path);
- string relativePath = rootPath.MakeRelativeUri(fullPath).ToString();
-
- using (Mat gray = Imgcodecs.imread(Utils.getFilePath(relativePath), Imgcodecs.IMREAD_GRAYSCALE))
- {
-
- if (gray.width() != bgrMat.width() || gray.height() != bgrMat.height())
- continue;
-
- Mat frameMat = gray.clone();
- double e = CaptureFrame(frameMat);
- if (e > 0)
- repErr = e;
-
- DrawFrame(gray, bgrMat);
- Imgproc.cvtColor(bgrMat, rgbaMat, Imgproc.COLOR_BGR2RGBA);
-
- Utils.matToTexture2D(rgbaMat, texture);
- }
- yield return new WaitForSeconds(0.5f);
- }
-
- isCalibrating = false;
- markerTypeDropdown.interactable = dictionaryIdDropdown.interactable = squaresXDropdown.interactable = squaresYDropdown.interactable = true;
- }
-
- private string[] GetImageFilesInDirectory(string dirPath)
- {
- if (Directory.Exists(dirPath))
- {
- string[] files = Directory.GetFiles(dirPath, "*.jpg");
- files = files.Concat(Directory.GetFiles(dirPath, "*.jpeg")).ToArray();
- files = files.Concat(Directory.GetFiles(dirPath, "*.png")).ToArray();
- files = files.Concat(Directory.GetFiles(dirPath, "*.tiff")).ToArray();
- files = files.Concat(Directory.GetFiles(dirPath, "*.tif")).ToArray();
- return files;
- }
- return new string[0];
- }
-
- ///
- /// Raises the destroy event.
- ///
- void OnDestroy()
- {
- if (isImagesInputMode)
- {
- DisposeCalibraton();
- }
- else
- {
- webCamTextureToMatHelper.Dispose();
- }
-
- Screen.orientation = ScreenOrientation.AutoRotation;
- }
-
- ///
- /// Raises the back button click event.
- ///
- public void OnBackButtonClick()
- {
- SceneManager.LoadScene("HoloLensWithOpenCVForUnityExample");
- }
-
- ///
- /// Raises the play button click event.
- ///
- public void OnPlayButtonClick()
- {
- if (isImagesInputMode) return;
-
- webCamTextureToMatHelper.Play();
- }
-
- ///
- /// Raises the pause button click event.
- ///
- public void OnPauseButtonClick()
- {
- if (isImagesInputMode) return;
-
- webCamTextureToMatHelper.Pause();
- }
-
- ///
- /// Raises the stop button click event.
- ///
- public void OnStopButtonClick()
- {
- if (isImagesInputMode) return;
-
- webCamTextureToMatHelper.Stop();
- }
-
- ///
- /// Raises the change camera button click event.
- ///
- public void OnChangeCameraButtonClick()
- {
- if (isImagesInputMode) return;
-
- webCamTextureToMatHelper.requestedIsFrontFacing = !webCamTextureToMatHelper.IsFrontFacing();
- }
-
- ///
- /// Raises the marker type dropdown value changed event.
- ///
- public void OnMarkerTypeDropdownValueChanged(int result)
- {
- if ((int)markerType != result)
- {
- markerType = (MarkerType)result;
-
- dictionaryIdDropdown.interactable = (markerType == MarkerType.ChArUcoBoard);
-
- if (isImagesInputMode)
- {
- InitializeImagesInputMode();
- }
- else
- {
- if (webCamTextureToMatHelper.IsInitialized())
- webCamTextureToMatHelper.Initialize();
- }
- }
- }
-
- ///
- /// Raises the dictionary id dropdown value changed event.
- ///
- public void OnDictionaryIdDropdownValueChanged(int result)
- {
- if ((int)dictionaryId != result)
- {
- dictionaryId = (ArUcoDictionary)result;
- dictionary = Aruco.getPredefinedDictionary((int)dictionaryId);
-
- if (isImagesInputMode)
- {
- InitializeImagesInputMode();
- }
- else
- {
- if (webCamTextureToMatHelper.IsInitialized())
- webCamTextureToMatHelper.Initialize();
- }
- }
- }
-
- ///
- /// Raises the squares X dropdown value changed event.
- ///
- public void OnSquaresXDropdownValueChanged(int result)
- {
- if ((int)squaresX != result + 1)
- {
- squaresX = (NumberOfSquaresX)(result + 1);
-
- if (isImagesInputMode)
- {
- InitializeImagesInputMode();
- }
- else
- {
- if (webCamTextureToMatHelper.IsInitialized())
- webCamTextureToMatHelper.Initialize();
- }
- }
- }
-
- ///
- /// Raises the squares Y dropdown value changed event.
- ///
- public void OnSquaresYDropdownValueChanged(int result)
- {
- if ((int)squaresY != result + 1)
- {
- squaresY = (NumberOfSquaresY)(result + 1);
-
- if (isImagesInputMode)
- {
- InitializeImagesInputMode();
- }
- else
- {
- if (webCamTextureToMatHelper.IsInitialized())
- webCamTextureToMatHelper.Initialize();
- }
- }
- }
-
- ///
- /// Raises the capture button click event.
- ///
- public void OnCaptureButtonClick()
- {
- if (isImagesInputMode)
- {
- if (!isCalibrating)
- InitializeImagesInputMode();
- StartCoroutine("CalibrateCameraUsingImages");
- }
- else
- {
- shouldCaptureFrame = true;
- }
- }
-
- ///
- /// Raises the reset button click event.
- ///
- public void OnResetButtonClick()
- {
- if (isImagesInputMode)
- {
- if (!isCalibrating)
- InitializeImagesInputMode();
- }
- else
- {
- ResetCalibration();
- }
- }
-
- ///
- /// Raises the save button click event.
- ///
- public void OnSaveButtonClick()
- {
- string saveDirectoryPath = Path.Combine(Application.persistentDataPath, "HoloLensArUcoCameraCalibrationExample");
-
- if (!Directory.Exists(saveDirectoryPath))
- {
- Directory.CreateDirectory(saveDirectoryPath);
- }
-
- string calibratonDirectoryName = "camera_parameters" + bgrMat.width() + "x" + bgrMat.height();
- string saveCalibratonFileDirectoryPath = Path.Combine(saveDirectoryPath, calibratonDirectoryName);
-
- // Clean up old files.
- if (Directory.Exists(saveCalibratonFileDirectoryPath))
- {
- DirectoryInfo directoryInfo = new DirectoryInfo(saveCalibratonFileDirectoryPath);
- foreach (FileInfo fileInfo in directoryInfo.GetFiles())
- {
- if ((fileInfo.Attributes & FileAttributes.ReadOnly) == FileAttributes.ReadOnly)
- {
- fileInfo.Attributes = FileAttributes.Normal;
- }
- }
- if ((directoryInfo.Attributes & FileAttributes.ReadOnly) == FileAttributes.ReadOnly)
- {
- directoryInfo.Attributes = FileAttributes.Directory;
- }
- directoryInfo.Delete(true);
- }
- Directory.CreateDirectory(saveCalibratonFileDirectoryPath);
-
- // save the calibraton file.
- string savePath = Path.Combine(saveCalibratonFileDirectoryPath, calibratonDirectoryName + ".xml");
- int frameCount = (markerType == MarkerType.ChArUcoBoard) ? allCorners.Count : imagePoints.Count;
- CameraParameters param = new CameraParameters(frameCount, bgrMat.width(), bgrMat.height(), calibrationFlags, camMatrix, distCoeffs, repErr);
- XmlSerializer serializer = new XmlSerializer(typeof(CameraParameters));
- using (var stream = new FileStream(savePath, FileMode.Create))
- {
- serializer.Serialize(stream, param);
- }
-
- // save the calibration images.
-#if UNITY_WEBGL && !UNITY_EDITOR
- string format = "jpg";
- MatOfInt compressionParams = new MatOfInt(Imgcodecs.IMWRITE_JPEG_QUALITY, 100);
-#else
- string format = "png";
- MatOfInt compressionParams = new MatOfInt(Imgcodecs.IMWRITE_PNG_COMPRESSION, 0);
-#endif
- for (int i = 0; i < allImgs.Count; ++i)
- {
- Imgcodecs.imwrite(Path.Combine(saveCalibratonFileDirectoryPath, calibratonDirectoryName + "_" + i.ToString("00") + "." + format), allImgs[i], compressionParams);
- }
-
- savePathInputField.text = savePath;
- Debug.Log("Saved the CameraParameters to disk in XML file format.");
- Debug.Log("savePath: " + savePath);
- }
-
- ///
- /// Raises the tapped event.
- ///
- public void OnTapped(MixedRealityPointerEventData eventData)
- {
- Debug.Log("OnTapped!");
-
- // Determine if a Gaze pointer is over a GUI.
- if (eventData.selectedObject != null && (eventData.selectedObject.GetComponent