From ae5734246c47185ebbd0fea0db37f00464f9a844 Mon Sep 17 00:00:00 2001 From: DESIREDDY MOHITH REDDY Date: Mon, 10 Aug 2026 09:05:13 +0530 Subject: [PATCH] feat: implement AI-driven behavioral video sentiment analyzer (#1744) --- frontend/package-lock.json | 73 ++++++++++++ frontend/package.json | 1 + .../src/components/VideoSentimentAnalyzer.jsx | 107 ++++++++++++++++++ 3 files changed, 181 insertions(+) create mode 100644 frontend/src/components/VideoSentimentAnalyzer.jsx diff --git a/frontend/package-lock.json b/frontend/package-lock.json index ecce5e58..24c61066 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -11,6 +11,7 @@ "@monaco-editor/react": "^4.7.0", "@tailwindcss/vite": "^4.1.12", "axios": "^1.11.0", + "face-api.js": "^0.22.2", "framer-motion": "^12.42.2", "html2pdf.js": "^0.14.0", "lucide-react": "^0.542.0", @@ -1896,6 +1897,23 @@ "node": ">= 10" } }, + "node_modules/@tensorflow/tfjs-core": { + "version": "1.7.0", + "resolved": "https://registry.npmjs.org/@tensorflow/tfjs-core/-/tfjs-core-1.7.0.tgz", + "integrity": "sha512-uwQdiklNjqBnHPeseOdG0sGxrI3+d6lybaKu2+ou3ajVeKdPEwpWbgqA6iHjq1iylnOGkgkbbnQ6r2lwkiIIHw==", + "license": "Apache-2.0", + "dependencies": { + "@types/offscreencanvas": "~2019.3.0", + "@types/seedrandom": "2.4.27", + "@types/webgl-ext": "0.0.30", + "@types/webgl2": "0.0.4", + "node-fetch": "~2.1.2", + "seedrandom": "2.4.3" + }, + "engines": { + "yarn": ">= 1.3.2" + } + }, "node_modules/@types/babel__core": { "version": "7.20.5", "resolved": "https://registry.npmjs.org/@types/babel__core/-/babel__core-7.20.5.tgz", @@ -2077,6 +2095,12 @@ "integrity": "sha512-GsCCIZDE/p3i96vtEqx+7dBUGXrc7zeSK3wwPHIaRThS+9OhWIXRqzs4d6k1SVU8g91DrNRWxWUGhp5KXQb2VA==", "license": "MIT" }, + "node_modules/@types/offscreencanvas": { + "version": "2019.3.0", + "resolved": "https://registry.npmjs.org/@types/offscreencanvas/-/offscreencanvas-2019.3.0.tgz", + "integrity": "sha512-esIJx9bQg+QYF0ra8GnvfianIY8qWB0GBx54PK5Eps6m+xTj86KLavHv6qDhzKcu5UUOgNfJ2pWaIIV7TRUd9Q==", + "license": "MIT" + }, "node_modules/@types/pako": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/@types/pako/-/pako-2.0.4.tgz", @@ -2116,6 +2140,12 @@ "@types/react": "^18.0.0" } }, + "node_modules/@types/seedrandom": { + "version": "2.4.27", + "resolved": "https://registry.npmjs.org/@types/seedrandom/-/seedrandom-2.4.27.tgz", + "integrity": "sha512-YvMLqFak/7rt//lPBtEHv3M4sRNA+HGxrhFZ+DQs9K2IkYJbNwVIb8avtJfhDiuaUBX/AW0jnjv48FV8h3u9bQ==", + "license": "MIT" + }, "node_modules/@types/trusted-types": { "version": "2.0.7", "resolved": "https://registry.npmjs.org/@types/trusted-types/-/trusted-types-2.0.7.tgz", @@ -2129,6 +2159,18 @@ "integrity": "sha512-CmBKiL6NNo/OqgmMn95Fk9Whlp2mtvIv+KNpQKN2F4SjvrEesubTRWGYSg+BnWZOnlCaSTU1sMpsBOzgbYhnsA==", "license": "MIT" }, + "node_modules/@types/webgl-ext": { + "version": "0.0.30", + "resolved": "https://registry.npmjs.org/@types/webgl-ext/-/webgl-ext-0.0.30.tgz", + "integrity": "sha512-LKVgNmBxN0BbljJrVUwkxwRYqzsAEPcZOe6S2T6ZaBDIrFp0qu4FNlpc5sM1tGbXUYFgdVQIoeLk1Y1UoblyEg==", + "license": "MIT" + }, + "node_modules/@types/webgl2": { + "version": "0.0.4", + "resolved": "https://registry.npmjs.org/@types/webgl2/-/webgl2-0.0.4.tgz", + "integrity": "sha512-PACt1xdErJbMUOUweSrbVM7gSIYm1vTncW2hF6Os/EeWi6TXYAYMPp+8v6rzHmypE5gHrxaxZNXgMkJVIdZpHw==", + "license": "MIT" + }, "node_modules/@ungap/structured-clone": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/@ungap/structured-clone/-/structured-clone-1.3.0.tgz", @@ -3424,6 +3466,22 @@ "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==", "license": "MIT" }, + "node_modules/face-api.js": { + "version": "0.22.2", + "resolved": "https://registry.npmjs.org/face-api.js/-/face-api.js-0.22.2.tgz", + "integrity": "sha512-9Bbv/yaBRTKCXjiDqzryeKhYxmgSjJ7ukvOvEBy6krA0Ah/vNBlsf7iBNfJljWiPA8Tys1/MnB3lyP2Hfmsuyw==", + "license": "MIT", + "dependencies": { + "@tensorflow/tfjs-core": "1.7.0", + "tslib": "^1.11.1" + } + }, + "node_modules/face-api.js/node_modules/tslib": { + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-1.14.1.tgz", + "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==", + "license": "0BSD" + }, "node_modules/fast-deep-equal": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", @@ -5894,6 +5952,15 @@ "dev": true, "license": "MIT" }, + "node_modules/node-fetch": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.1.2.tgz", + "integrity": "sha512-IHLHYskTc2arMYsHZH82PVX8CSKT5lzb7AXeyO06QnjGDKtkv+pv3mEki6S7reB/x1QPo+YPxQRNEVgR5V/w3Q==", + "license": "MIT", + "engines": { + "node": "4.x || >=6.0.0" + } + }, "node_modules/node-releases": { "version": "2.0.19", "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.19.tgz", @@ -6629,6 +6696,12 @@ "loose-envify": "^1.1.0" } }, + "node_modules/seedrandom": { + "version": "2.4.3", + "resolved": "https://registry.npmjs.org/seedrandom/-/seedrandom-2.4.3.tgz", + "integrity": "sha512-2CkZ9Wn2dS4mMUWQaXLsOAfGD+irMlLEeSP3cMxpGbgyOOzJGFa+MWCOMTOCMyZinHRPxyOj/S/C57li/1to6Q==", + "license": "MIT" + }, "node_modules/semver": { "version": "6.3.1", "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", diff --git a/frontend/package.json b/frontend/package.json index 6a38a3d2..870ac249 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -15,6 +15,7 @@ "@monaco-editor/react": "^4.7.0", "@tailwindcss/vite": "^4.1.12", "axios": "^1.11.0", + "face-api.js": "^0.22.2", "framer-motion": "^12.42.2", "html2pdf.js": "^0.14.0", "lucide-react": "^0.542.0", diff --git a/frontend/src/components/VideoSentimentAnalyzer.jsx b/frontend/src/components/VideoSentimentAnalyzer.jsx new file mode 100644 index 00000000..9e78eb5a --- /dev/null +++ b/frontend/src/components/VideoSentimentAnalyzer.jsx @@ -0,0 +1,107 @@ +import React, { useEffect, useRef, useState } from 'react'; +import * as faceapi from 'face-api.js'; + +const VideoSentimentAnalyzer = ({ onAnalysisComplete }) => { + const videoRef = useRef(); + const [isModelLoaded, setIsModelLoaded] = useState(false); + const [expressions, setExpressions] = useState({}); + const [isAnalyzing, setIsAnalyzing] = useState(false); + + useEffect(() => { + const loadModels = async () => { + try { + await faceapi.nets.tinyFaceDetector.loadFromUri('/models'); + await faceapi.nets.faceExpressionNet.loadFromUri('/models'); + setIsModelLoaded(true); + } catch (err) { + console.error("Error loading face-api models", err); + } + }; + loadModels(); + }, []); + + const startVideo = () => { + navigator.mediaDevices.getUserMedia({ video: true }) + .then((stream) => { + videoRef.current.srcObject = stream; + setIsAnalyzing(true); + }) + .catch((err) => console.error(err)); + }; + + const stopVideo = () => { + const stream = videoRef.current?.srcObject; + if (stream) { + stream.getTracks().forEach(track => track.stop()); + } + setIsAnalyzing(false); + if (onAnalysisComplete) { + onAnalysisComplete(expressions); + } + }; + + const handleVideoPlay = () => { + setInterval(async () => { + if (videoRef.current && isAnalyzing) { + const detections = await faceapi.detectAllFaces( + videoRef.current, + new faceapi.TinyFaceDetectorOptions() + ).withFaceExpressions(); + + if (detections.length > 0) { + setExpressions(detections[0].expressions); + } + } + }, 1000); + }; + + return ( +
+

AI Behavioral Sentiment Analyzer

+ + {!isModelLoaded ? ( +

Loading AI Models...

+ ) : ( +
+
+
+ +
+ {!isAnalyzing ? ( + + ) : ( + + )} +
+ + {Object.keys(expressions).length > 0 && ( +
+

Real-time Analysis

+
+ {Object.entries(expressions).map(([exp, val]) => ( +
+ {exp} + {(val * 100).toFixed(1)}% +
+ ))} +
+
+ )} +
+ )} +
+ ); +}; + +export default VideoSentimentAnalyzer;