Add smiling expression tracking support for LookAtMe 2.0, current iss…

…ue is easily adding the status for the android app developer to access
Pradyuman7 · May 15, 2019 · 18ff20c · 18ff20c
1 parent 84ad25a
commit 18ff20c
Show file tree

Hide file tree

Showing 9 changed files with 105 additions and 22 deletions.
diff --git a/.idea/caches/build_file_checksums.ser b/.idea/caches/build_file_checksums.ser
diff --git a/.idea/gradle.xml b/.idea/gradle.xml
diff --git a/.idea/misc.xml b/.idea/misc.xml
diff --git a/.idea/vcs.xml b/.idea/vcs.xml
diff --git a/LookAtMe/src/main/java/com/pd/lookatme/LookAtMe.java b/LookAtMe/src/main/java/com/pd/lookatme/LookAtMe.java
@@ -25,8 +25,10 @@
 public class LookAtMe extends VideoView {
 
     private String status = "";
+    private String smilingStatus = "";
     private CameraSource cameraSource;
     private Context activityContext;
+    private int timesSmiled = 0;
 
     public void init(Context activityContext) {
         this.activityContext = activityContext;
@@ -43,6 +45,11 @@ public void init(Context activityContext, String mode){
         createCameraSource(mode);
     }
 
+    public void initWithSmilingStatus(Context activityContext){
+        this.activityContext = activityContext;
+        createCameraSourceWithSmilingStatus();
+    }
+
     public void resume(){
         if (cameraSource != null) {
             try {
@@ -51,7 +58,6 @@ public void resume(){
                     Toast.makeText(activityContext, "Grant Permission and restart app", Toast.LENGTH_SHORT).show();
                 }
                 cameraSource.start();
-                //Log.d("some8","Starting first");
             }
             catch (IOException e) {
                 e.printStackTrace();
@@ -62,11 +68,10 @@ public void resume(){
     public void paused(){
         if (cameraSource != null) {
             cameraSource.stop();
-            //Log.d("some6","stopped here");
         }
+
         if (this.isPlaying()) {
             this.pause();
-            //Log.d("some7","Paused() here");
         }
     }
 
@@ -114,17 +119,23 @@ public String getStatus(){
         return status;
     }
 
-    public void setLookMe(){
-        //Log.d("this is found","camerasource is " + cameraSource);
+    public String getSmilingStatus(){
+        return smilingStatus;
+    }
+
+    public int getTimesSmiled(){
+        return timesSmiled;
+    }
 
+    public void setLookMe(){
         try {
             if (ActivityCompat.checkSelfPermission(activityContext, Manifest.permission.CAMERA) != PackageManager.PERMISSION_GRANTED) {
                 ActivityCompat.requestPermissions((Activity) activityContext, new String[]{Manifest.permission.CAMERA}, 1);
 
                 Toast.makeText(activityContext, "Grant Permission and restart app", Toast.LENGTH_SHORT).show();
             }
             cameraSource.start();
-            Log.d("ReadThis", "camerasource started, outside");
+            Log.d("ReadThis", "camera-source started, outside");
         }
         catch (IOException e) {
             e.printStackTrace();
@@ -142,17 +153,14 @@ private EyesTracker() {
         @Override
         public void onUpdate(Detector.Detections<Face> detections, Face face) {
             if (face.getIsLeftEyeOpenProbability() > THRESHOLD || face.getIsRightEyeOpenProbability() > THRESHOLD) {
-               // Log.d("some9", "Eyes Detecting");
 
                 if (!isPlaying())
                     start();
 
                 status = "Eyes Detected and open, so video continues";
-                //Log.d("some3", "Started due to eyes detected");
             }
             else {
                 if (isPlaying()){
-                   // Log.d("some4","Paused due to eyes closed");
                     pause();
                 }
 
@@ -165,7 +173,43 @@ public void onMissing(Detector.Detections<Face> detections) {
             super.onMissing(detections);
             status = "Face Not Detected!";
             pause();
-            //Log.d("some5","Face is Missing");
+        }
+
+        @Override
+        public void onDone() {
+            super.onDone();
+        }
+    }
+
+    private class FaceTracker extends Tracker<Face> {
+        private final float THRESHOLD = 0.75f;
+
+        private FaceTracker(){
+
+        }
+
+        @Override
+        public void onUpdate(Detector.Detections<Face> detections, Face face) {
+            if(face.getIsSmilingProbability() > THRESHOLD) {
+                smilingStatus = "smiling";
+                timesSmiled++;
+                Log.d("smile", String.valueOf(getTimesSmiled()));
+
+                if(timesSmiled > 100){
+                    timesSmiled -= 100;
+
+                    Log.d("Smiling","You smiled for 100 frames!");
+
+                }
+            }
+            else{
+                smilingStatus = "Not smiling";
+            }
+        }
+
+        @Override
+        public void onMissing(Detector.Detections<Face> detections) {
+            super.onMissing(detections);
         }
 
         @Override
@@ -188,7 +232,27 @@ private void createCameraSource() {
             }
             else
                 cameraSource.start();
-            //Log.d("ReadThis", "camerasource started, inside");
+        }
+        catch (IOException e) {
+            e.printStackTrace();
+        }
+    }
+
+    private void createCameraSourceWithSmilingStatus() {
+        FaceDetector detector = new FaceDetector.Builder(activityContext).setTrackingEnabled(true).setClassificationType(FaceDetector.ALL_CLASSIFICATIONS).setMode(FaceDetector.FAST_MODE).build();
+
+        FaceTracker faceTracker = new FaceTracker();
+        detector.setProcessor(new LargestFaceFocusingProcessor(detector, faceTracker));
+
+        cameraSource = new CameraSource.Builder(activityContext, detector).setRequestedPreviewSize(1024, 768).setFacing(CameraSource.CAMERA_FACING_FRONT).setRequestedFps(30.0f).build();
+
+        try {
+            if (ActivityCompat.checkSelfPermission(activityContext, Manifest.permission.CAMERA) != PackageManager.PERMISSION_GRANTED) {
+                ActivityCompat.requestPermissions((Activity) activityContext, new String[]{Manifest.permission.CAMERA}, 1);
+                Toast.makeText(activityContext, "Grant Permission and restart app", Toast.LENGTH_SHORT).show();
+            }
+            else
+                cameraSource.start();
         }
         catch (IOException e) {
             e.printStackTrace();
@@ -217,7 +281,6 @@ private void createCameraSource(String mode) {
             }
             else
                 cameraSource.start();
-            //Log.d("ReadThis", "camerasource started, inside");
         }
         catch (IOException e) {
             e.printStackTrace();
@@ -226,7 +289,7 @@ private void createCameraSource(String mode) {
 
     private void createCameraSource(String mode, String cameraFace) {
 
-        // Let the mode be fast or accurate
+        // Let the mode be fast or accurate and
         // Let the cameraFace be front or back
 
         FaceDetector detector;
@@ -250,7 +313,6 @@ private void createCameraSource(String mode, String cameraFace) {
             }
             else
                 cameraSource.start();
-            //Log.d("ReadThis", "camerasource started, inside");
         }
         catch (IOException e) {
             e.printStackTrace();

diff --git a/app/src/main/java/com/pd/lookme/MainActivity.java b/app/src/main/java/com/pd/lookme/MainActivity.java
@@ -3,7 +3,9 @@
 import android.net.Uri;
 import android.support.v7.app.AppCompatActivity;
 import android.os.Bundle;
+import android.util.Log;
 import android.widget.TextView;
+import android.widget.Toast;
 import android.widget.VideoView;
 
 import com.pd.lookatme.LookAtMe;
@@ -21,12 +23,14 @@ protected void onCreate(Bundle savedInstanceState) {
         lookAtMe = findViewById(R.id.lookme);
         textView = findViewById(R.id.textView2);
 
-        lookAtMe.init(this);
+        //lookAtMe.init(this); for eye tracking status
+        lookAtMe.initWithSmilingStatus(this); // for smiling status of the user
         lookAtMe.setVideoURI(Uri.parse("android.resource://" + getPackageName() + "/" + R.raw.videoplayback));
 
         lookAtMe.start();
         lookAtMe.setLookMe();
 
+
         this.showStatus();
     }
 

diff --git a/build.gradle b/build.gradle
@@ -7,7 +7,7 @@ buildscript {
         jcenter()
     }
     dependencies {
-        classpath 'com.android.tools.build:gradle:3.1.4'
+        classpath 'com.android.tools.build:gradle:3.3.2'
 
 
         // NOTE: Do not place your application dependencies here; they belong

diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties
@@ -1,6 +1,6 @@
-#Sat Mar 02 07:55:19 CET 2019
+#Mon May 13 10:18:57 CEST 2019
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-4.4-all.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-4.10.1-all.zip
diff --git a/settings.gradle b/settings.gradle
@@ -1 +1 @@
-include ':app', ':LookAtMe'
+include ':app', ':LookAtMe', ':lookatad'