Skip to content

Commit

Permalink
Facemesh cleanup (#202)
Browse files Browse the repository at this point in the history 
* Update file header

* refactor config schema

* update comments and function jsdoc
  • Loading branch information
@ziyuan-linn
ziyuan-linn authored Sep 16, 2024
1 parent 41be997 commit 9aff062
Showing 1 changed file with 130 additions and 94 deletions.
224 changes: 130 additions & 94 deletions src/FaceMesh/index.js
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,21 @@
// Copyright (c) 2020-2023 ml5
//
// This software is released under the MIT License.
// https://opensource.org/licenses/MIT
/**
* @license
* Copyright (c) 2020-2024 ml5
* This software is released under the ml5.js License.
* https://github.com/ml5js/ml5-next-gen/blob/main/LICENSE.md
*/

/*
* FaceMesh: Face landmarks tracking in the browser
* Ported and integrated from all the hard work by: https://github.com/tensorflow/tfjs-models/tree/master/face-landmarks-detection
/**
* @file HandPose
*
* The file contains the main source code of FaceMesh, a pretrained face landmark
* estimation model that detects and tracks faces and facial features with landmark points.
* The FaceMesh model is built on top of the face detection model of TensorFlow.
*
* TensorFlow Face Detection repo:
* https://github.com/tensorflow/tfjs-models/tree/master/face-detection
* ml5.js BodyPose reference documentation:
* https://docs.ml5js.org/#/reference/facemesh
*/

import * as tf from "@tensorflow/tfjs";
Expand All @@ -16,24 +26,75 @@ import { mediaReady } from "../utils/imageUtilities";
import handleOptions from "../utils/handleOptions";
import { handleModelName } from "../utils/handleOptions";

class FaceMesh {
/**
* An options object to configure FaceMesh settings
* @typedef {Object} configOptions
* @property {number} maxFacess - The maximum number of faces to detect. Defaults to 2.
* @property {boolean} refineLandmarks - Refine the ladmarks. Defaults to false.
* @property {boolean} flipHorizontal - Flip the result horizontally. Defaults to false.
* @property {string} runtime - The runtime to use. "mediapipe"(default) or "tfjs".
*
* // For using custom or offline models
* @property {string} solutionPath - The file path or URL to the model.
*/
/**
* User provided options object for FaceMesh. See config schema below for default and available values.
* @typedef {Object} configOptions
* @property {number} [maxFaces] - The maximum number of faces to detect.
* @property {boolean} [refineLandmarks] - Whether to refine the landmarks.
* @property {boolean} [flipHorizontal] - Whether to mirror the results.
* @property {string} [runtime] - The runtime to use.
* @property {string} [solutionPath] - The file path or URL to the MediaPipe solution. Only
* for `mediapipe` runtime.
* @property {string} [detectorModelUrl] - The file path or URL to the detector model. Only for
* `tfjs` runtime.
* @property {string} [landmarkModelUrl] - The file path or URL to the landmark model. Only for
* `tfjs` runtime.
*/

/**
* Schema for initialization options, used by `handleOptions` to
* validate the user's options object.
*/
const configSchema = {
runtime: {
type: "enum",
enums: ["mediapipe", "tfjs"],
default: "tfjs",
},
maxFaces: {
type: "number",
min: 1,
default: 1,
},
refineLandmarks: {
type: "boolean",
default: false,
},
solutionPath: {
type: "string",
default: "https://cdn.jsdelivr.net/npm/@mediapipe/face_mesh",
ignore: (config) => config.runtime !== "mediapipe",
},
detectorModelUrl: {
type: "string",
default: undefined,
ignore: (config) => config.runtime !== "tfjs",
},
landmarkModelUrl: {
type: "string",
default: undefined,
ignore: (config) => config.runtime !== "tfjs",
},
};

/**
* Schema for runtime options, used by `handleOptions` to
* validate the user's options object.
*/
const runtimeSchema = {
flipHorizontal: {
type: "boolean",
alias: "flipped",
default: false,
},
};

class FaceMesh {
/**
* Create FaceMesh.
* Creates an instance of FaceMesh.
* @param {string} [modelName] - The name of the model to use.
* @param {configOptions} options - An object with options.
* @param {function} callback - A callback to be called when the model is ready.
*
* @private
*/
constructor(modelName, options, callback) {
Expand All @@ -43,77 +104,46 @@ class FaceMesh {
"FaceMesh",
"faceMesh"
);
/** The underlying TensorFlow.js detector instance.*/
this.model = null;
this.config = options;
/** The user provided options object. */
this.userOptions = options;
/** The config passed to underlying detector instance during inference. */
this.runtimeConfig = {};
/** The media source being continuously detected. Only used in continuous mode. */
this.detectMedia = null;
/** The callback function for detection results. Only used in continuous mode. */
this.detectCallback = null;

// flags for detectStart() and detectStop()
this.detecting = false; // true when detection loop is running
this.signalStop = false; // true when detectStop() is called and detecting is true
this.prevCall = ""; // "start" or "stop", used for giving warning messages with detectStart() is called twice in a row

/** A flag for continuous mode, set to true when detection loop is running.*/
this.detecting = false;
/** A flag to signal stop to the detection loop.*/
this.signalStop = false;
/** A flag to track the previous call to`detectStart` and `detectStop`. */
this.prevCall = "";
/** A promise that resolves when the model is ready. */
this.ready = callCallback(this.loadModel(), callback);
}

/**
* Load the model and set it to this.model
* Load the FaceMesh instance.
* @return {this} the FaceMesh model.
*
* @private
*/
async loadModel() {
const pipeline = faceLandmarksDetection.SupportedModels.MediaPipeFaceMesh;
// filter out model config options
// Filter out model config options
const modelConfig = handleOptions(
this.config,
{
runtime: {
type: "enum",
enums: ["mediapipe", "tfjs"],
default: "tfjs",
},
maxFaces: {
type: "number",
min: 1,
default: 1,
},
refineLandmarks: {
type: "boolean",
default: false,
},
solutionPath: {
type: "string",
default: "https://cdn.jsdelivr.net/npm/@mediapipe/face_mesh",
ignore: (config) => config.runtime !== "mediapipe",
},
detectorModelUrl: {
type: "string",
default: undefined,
ignore: (config) => config.runtime !== "tfjs",
},
landmarkModelUrl: {
type: "string",
default: undefined,
ignore: (config) => config.runtime !== "tfjs",
},
},
this.userOptions,
configSchema,
"faceMesh"
);

this.runtimeConfig = handleOptions(
this.config,
{
flipHorizontal: {
type: "boolean",
alias: "flipped",
default: false,
},
},
this.userOptions,
runtimeSchema,
"faceMesh"
);

// Load the model once tfjs is ready
await tf.ready();
this.model = await faceLandmarksDetection.createDetector(
pipeline,
Expand All @@ -124,20 +154,21 @@ class FaceMesh {
}

/**
* Asynchronously output a single face prediction result when called
* @param {*} [media] - An HMTL or p5.js image, video, or canvas element to run the prediction on.
* @param {function} [callback] - A callback function to handle the predictions.
* @returns {Promise<Array>} an array of predictions.
* Asynchronously outputs a single face prediction result when called.
* @param {any} media - An HTML or p5.js image, video, or canvas element to run the prediction on.
* @param {function} [callback] - A callback function to handle the detection result.
* @returns {Promise<Array>} an array of predicted faces.
* @public
*/
async detect(...inputs) {
// Parse out the input parameters
// Parse the input parameters
const argumentObject = handleArguments(...inputs);
argumentObject.require(
"image",
"An html or p5.js image, video, or canvas element argument is required for detect()."
);
const { image, callback } = argumentObject;

// Run the prediction
await mediaReady(image, false);
const predictions = await this.model.estimateFaces(
image,
Expand All @@ -150,13 +181,13 @@ class FaceMesh {
}

/**
* Repeatedly output face predictions through a callback function
* @param {*} [media] - An HMTL or p5.js image, video, or canvas element to run the prediction on.
* @param {function} [callback] - A callback function to handle the predictions.
* @returns {Promise<Array>} an array of predictions.
* Repeatedly outputs face predictions through a callback function.
* @param {any} media - An HTML or p5.js image, video, or canvas element to run the prediction on.
* @param {function} [callback] - A callback function to handle the prediction results.
* @public
*/
detectStart(...inputs) {
// Parse out the input parameters
// Parse the input parameters
const argumentObject = handleArguments(...inputs);
argumentObject.require(
"image",
Expand All @@ -169,6 +200,7 @@ class FaceMesh {
this.detectMedia = argumentObject.image;
this.detectCallback = argumentObject.callback;

// Set the flags and call the detection loop
this.signalStop = false;
if (!this.detecting) {
this.detecting = true;
Expand All @@ -183,17 +215,17 @@ class FaceMesh {
}

/**
* Stop the detection loop before next detection loop runs.
* Stop the continuous detection before next detection loop runs.
* @public
*/
detectStop() {
if (this.detecting) this.signalStop = true;
this.prevCall = "stop";
}

/**
* Internal function to call estimateFaces in a loop
* Can be started by detectStart() and terminated by detectStop()
*
* Calls estimateFaces in a loop.
* Can be started by `detectStart` and terminated by `detectStop`.
* @private
*/
async detectLoop() {
Expand All @@ -214,21 +246,22 @@ class FaceMesh {
}

/**
* Return a new array of results with named keypoints added
* @param {Array} faces - the original detection results
* @return {Array} the detection results with named keypoints added
*
* Return a new array of results with named features added.
* The keypoints in each named feature is sorted the order of the contour.
* @param {Array} faces - The original detection results.
* @return {Array} - The detection results with named keypoints added.
* @private
*/
addKeypoints(faces) {
const contours = faceLandmarksDetection.util.getKeypointIndexByContour(
faceLandmarksDetection.SupportedModels.MediaPipeFaceMesh
);
// Add the missing keypoint to the lips contour
// Remove the following line when the tfjs fix the lips issue
// https://github.com/tensorflow/tfjs/issues/8221
if (contours.lips[20] !== 291) contours.lips.splice(20, 0, 291);
for (let face of faces) {
// Remove the following line when the tfjs fix the lips issue
// https://github.com/tensorflow/tfjs/issues/8221
face.keypoints[291].name = "lips";
for (let contourLabel in contours) {
for (let keypointIndex of contours[contourLabel]) {
Expand Down Expand Up @@ -293,8 +326,11 @@ class FaceMesh {
}

/**
* Factory function that returns a FaceMesh instance
* @returns {Object} A new faceMesh instance
* Factory function that returns a FaceMesh instance.
* @param {string} [modelName] - The name of the model to use.
* @param {configOptions} [options] - A user-defined options object.
* @param {function} [callback] - A callback to be called when the model is ready.
* @returns {Object} A new faceMesh instance.
*/
const faceMesh = (...inputs) => {
const { string, options = {}, callback } = handleArguments(...inputs);
Expand Down

0 comments on commit 9aff062

Please sign in to comment.