From 1abab9f92dedf8847c62be6422a29fc83c086805 Mon Sep 17 00:00:00 2001 From: ButterscotchVanilla Date: Tue, 10 Aug 2021 12:23:30 -0400 Subject: [PATCH 01/60] AutoBone: Add basic skeleton initialization --- .../io/eiren/gui/autobone/SimpleSkeleton.java | 110 ++++++++++++++++++ 1 file changed, 110 insertions(+) create mode 100644 src/main/java/io/eiren/gui/autobone/SimpleSkeleton.java diff --git a/src/main/java/io/eiren/gui/autobone/SimpleSkeleton.java b/src/main/java/io/eiren/gui/autobone/SimpleSkeleton.java new file mode 100644 index 0000000000..60d1c46722 --- /dev/null +++ b/src/main/java/io/eiren/gui/autobone/SimpleSkeleton.java @@ -0,0 +1,110 @@ +package io.eiren.gui.autobone; + +import io.eiren.vr.VRServer; +import io.eiren.vr.processor.HumanSkeletonWithLegs; +import io.eiren.vr.processor.HumanSkeletonWithWaist; +import io.eiren.vr.processor.TransformNode; + +public class SimpleSkeleton { + + protected final VRServer server; + + // Waist + protected final TransformNode hmdNode = new TransformNode("HMD", false); + protected final TransformNode headNode = new TransformNode("Head", false); + protected final TransformNode neckNode = new TransformNode("Neck", false); + protected final TransformNode waistNode = new TransformNode("Waist", false); + protected final TransformNode chestNode = new TransformNode("Chest", false); + + protected float chestDistance = 0.42f; + /** + * Distance from eyes to waist + */ + protected float waistDistance = 0.85f; + /** + * Distacne from eyes to the base of the neck + */ + protected float neckLength = HumanSkeletonWithWaist.NECK_LENGTH_DEFAULT; + /** + * Distance from eyes to ear + */ + protected float headShift = HumanSkeletonWithWaist.HEAD_SHIFT_DEFAULT; + + // Legs + protected final TransformNode leftHipNode = new TransformNode("Left-Hip", false); + protected final TransformNode leftKneeNode = new TransformNode("Left-Knee", false); + protected final TransformNode leftAnkleNode = new TransformNode("Left-Ankle", false); + protected final TransformNode leftFootNode = new TransformNode("Left-Foot", false); + protected final TransformNode rightHipNode = new TransformNode("Right-Hip", false); + protected final TransformNode rightKneeNode = new TransformNode("Right-Knee", false); + protected final TransformNode rightAnkleNode = new TransformNode("Right-Ankle", false); + protected final TransformNode rightFootNode = new TransformNode("Right-Foot", false); + + /** + * Distance between centers of both hips + */ + protected float hipsWidth = HumanSkeletonWithLegs.HIPS_WIDTH_DEFAULT; + /** + * Length from waist to knees + */ + protected float kneeHeight = 0.42f; + /** + * Distance from waist to ankle + */ + protected float legsLength = 0.84f; + protected float footLength = HumanSkeletonWithLegs.FOOT_LENGTH_DEFAULT; + + public SimpleSkeleton(VRServer server) { + this.server = server; + + // Load waist configs + headShift = server.config.getFloat("body.headShift", headShift); + neckLength = server.config.getFloat("body.neckLength", neckLength); + chestDistance = server.config.getFloat("body.chestDistance", chestDistance); + waistDistance = server.config.getFloat("body.waistDistance", waistDistance); + + // Load leg configs + hipsWidth = server.config.getFloat("body.hipsWidth", hipsWidth); + kneeHeight = server.config.getFloat("body.kneeHeight", kneeHeight); + legsLength = server.config.getFloat("body.legsLength", legsLength); + footLength = server.config.getFloat("body.footLength", footLength); + + // Assemble skeleton to waist + hmdNode.attachChild(headNode); + headNode.localTransform.setTranslation(0, 0, headShift); + + headNode.attachChild(neckNode); + neckNode.localTransform.setTranslation(0, -neckLength, 0); + + neckNode.attachChild(chestNode); + chestNode.localTransform.setTranslation(0, -chestDistance, 0); + + chestNode.attachChild(waistNode); + waistNode.localTransform.setTranslation(0, -(waistDistance - chestDistance), 0); + + // Assemble skeleton to feet + waistNode.attachChild(leftHipNode); + leftHipNode.localTransform.setTranslation(-hipsWidth / 2, 0, 0); + + waistNode.attachChild(rightHipNode); + rightHipNode.localTransform.setTranslation(hipsWidth / 2, 0, 0); + + leftHipNode.attachChild(leftKneeNode); + leftKneeNode.localTransform.setTranslation(0, -(legsLength - kneeHeight), 0); + + rightHipNode.attachChild(rightKneeNode); + rightKneeNode.localTransform.setTranslation(0, -(legsLength - kneeHeight), 0); + + leftKneeNode.attachChild(leftAnkleNode); + leftAnkleNode.localTransform.setTranslation(0, -kneeHeight, 0); + + rightKneeNode.attachChild(rightAnkleNode); + rightAnkleNode.localTransform.setTranslation(0, -kneeHeight, 0); + + leftAnkleNode.attachChild(leftFootNode); + leftFootNode.localTransform.setTranslation(0, 0, -footLength); + + rightAnkleNode.attachChild(rightFootNode); + rightFootNode.localTransform.setTranslation(0, 0, -footLength); + } +} From fc6f7d30049e8462de220c822087c221c6647ce8 Mon Sep 17 00:00:00 2001 From: ButterscotchVanilla Date: Tue, 10 Aug 2021 13:40:21 -0400 Subject: [PATCH 02/60] AutoBone: Add config setting/saving --- .../io/eiren/gui/autobone/SimpleSkeleton.java | 58 +++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/src/main/java/io/eiren/gui/autobone/SimpleSkeleton.java b/src/main/java/io/eiren/gui/autobone/SimpleSkeleton.java index 60d1c46722..58033a4a8e 100644 --- a/src/main/java/io/eiren/gui/autobone/SimpleSkeleton.java +++ b/src/main/java/io/eiren/gui/autobone/SimpleSkeleton.java @@ -107,4 +107,62 @@ public SimpleSkeleton(VRServer server) { rightAnkleNode.attachChild(rightFootNode); rightFootNode.localTransform.setTranslation(0, 0, -footLength); } + + public void setSkeletonConfig(String joint, float newLength) { + switch(joint) { + case "Head": + headShift = newLength; + headNode.localTransform.setTranslation(0, 0, headShift); + break; + case "Neck": + neckLength = newLength; + neckNode.localTransform.setTranslation(0, -neckLength, 0); + break; + case "Waist": + waistDistance = newLength; + waistNode.localTransform.setTranslation(0, -(waistDistance - chestDistance), 0); + break; + case "Chest": + chestDistance = newLength; + chestNode.localTransform.setTranslation(0, -chestDistance, 0); + waistNode.localTransform.setTranslation(0, -(waistDistance - chestDistance), 0); + break; + case "Hips width": + hipsWidth = newLength; + leftHipNode.localTransform.setTranslation(-hipsWidth / 2, 0, 0); + rightHipNode.localTransform.setTranslation(hipsWidth / 2, 0, 0); + break; + case "Knee height": + kneeHeight = newLength; + leftAnkleNode.localTransform.setTranslation(0, -kneeHeight, 0); + rightAnkleNode.localTransform.setTranslation(0, -kneeHeight, 0); + leftKneeNode.localTransform.setTranslation(0, -(legsLength - kneeHeight), 0); + rightKneeNode.localTransform.setTranslation(0, -(legsLength - kneeHeight), 0); + break; + case "Legs length": + legsLength = newLength; + leftKneeNode.localTransform.setTranslation(0, -(legsLength - kneeHeight), 0); + rightKneeNode.localTransform.setTranslation(0, -(legsLength - kneeHeight), 0); + break; + case "Foot length": + footLength = newLength; + leftFootNode.localTransform.setTranslation(0, 0, -footLength); + rightFootNode.localTransform.setTranslation(0, 0, -footLength); + break; + } + } + + public void saveConfigs() { + // Save waist configs + server.config.setProperty("body.headShift", headShift); + server.config.setProperty("body.neckLength", neckLength); + server.config.setProperty("body.waistDistance", waistDistance); + server.config.setProperty("body.chestDistance", chestDistance); + + // Save leg configs + server.config.setProperty("body.hipsWidth", hipsWidth); + server.config.setProperty("body.kneeHeight", kneeHeight); + server.config.setProperty("body.legsLength", legsLength); + server.config.setProperty("body.footLength", footLength); + } } From 8b209eaf273445bc257bd7499aab06d8c9d67b7b Mon Sep 17 00:00:00 2001 From: ButterscotchVanilla Date: Tue, 10 Aug 2021 14:19:10 -0400 Subject: [PATCH 03/60] AutoBone: Add node HashMap --- .../io/eiren/gui/autobone/SimpleSkeleton.java | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/main/java/io/eiren/gui/autobone/SimpleSkeleton.java b/src/main/java/io/eiren/gui/autobone/SimpleSkeleton.java index 58033a4a8e..13744b8edf 100644 --- a/src/main/java/io/eiren/gui/autobone/SimpleSkeleton.java +++ b/src/main/java/io/eiren/gui/autobone/SimpleSkeleton.java @@ -1,5 +1,7 @@ package io.eiren.gui.autobone; +import java.util.HashMap; + import io.eiren.vr.VRServer; import io.eiren.vr.processor.HumanSkeletonWithLegs; import io.eiren.vr.processor.HumanSkeletonWithWaist; @@ -54,6 +56,8 @@ public class SimpleSkeleton { protected float legsLength = 0.84f; protected float footLength = HumanSkeletonWithLegs.FOOT_LENGTH_DEFAULT; + HashMap nodes = new HashMap(); + public SimpleSkeleton(VRServer server) { this.server = server; @@ -106,6 +110,22 @@ public SimpleSkeleton(VRServer server) { rightAnkleNode.attachChild(rightFootNode); rightFootNode.localTransform.setTranslation(0, 0, -footLength); + + // Set up a HashMap to get nodes by name easily + hmdNode.depthFirstTraversal(visitor -> { + nodes.put(visitor.getName(), visitor); + }); + } + + public void poseFromSkeleton(HumanSkeletonWithLegs humanSkeleton) { + humanSkeleton.getRootNode().depthFirstTraversal(visitor -> { + TransformNode targetNode = nodes.get(visitor.getName()); + + // Handle unexpected nodes gracefully + if (targetNode != null) { + targetNode.localTransform.setRotation(visitor.localTransform.getRotation()); + } + }); } public void setSkeletonConfig(String joint, float newLength) { From 19a1101b43b665d2d7fad37cd01c4905ab4e3b57 Mon Sep 17 00:00:00 2001 From: ButterscotchVanilla Date: Tue, 10 Aug 2021 17:33:12 -0400 Subject: [PATCH 04/60] AutoBone: Add AutoBone, PoseFrame, and finish implementing SimpleSkeleton --- .../java/io/eiren/gui/autobone/AutoBone.java | 13 ++++++ .../java/io/eiren/gui/autobone/PoseFrame.java | 33 ++++++++++++++ .../io/eiren/gui/autobone/SimpleSkeleton.java | 45 +++++++++++++++++-- 3 files changed, 87 insertions(+), 4 deletions(-) create mode 100644 src/main/java/io/eiren/gui/autobone/AutoBone.java create mode 100644 src/main/java/io/eiren/gui/autobone/PoseFrame.java diff --git a/src/main/java/io/eiren/gui/autobone/AutoBone.java b/src/main/java/io/eiren/gui/autobone/AutoBone.java new file mode 100644 index 0000000000..e02c91629e --- /dev/null +++ b/src/main/java/io/eiren/gui/autobone/AutoBone.java @@ -0,0 +1,13 @@ +package io.eiren.gui.autobone; + +import io.eiren.vr.VRServer; + +public class AutoBone { + + protected final VRServer server; + + public AutoBone(VRServer server) { + this.server = server; + } + +} diff --git a/src/main/java/io/eiren/gui/autobone/PoseFrame.java b/src/main/java/io/eiren/gui/autobone/PoseFrame.java new file mode 100644 index 0000000000..a7c584917e --- /dev/null +++ b/src/main/java/io/eiren/gui/autobone/PoseFrame.java @@ -0,0 +1,33 @@ +package io.eiren.gui.autobone; + +import java.util.HashMap; + +import com.jme3.math.Quaternion; +import com.jme3.math.Vector3f; + +import io.eiren.vr.processor.HumanSkeletonWithLegs; +import io.eiren.vr.processor.TransformNode; + +public final class PoseFrame { + + public final Vector3f rootPos; + public final HashMap rotations; + + public PoseFrame(Vector3f rootPos, HashMap rotations) { + this.rootPos = rootPos; + this.rotations = rotations; + } + + public PoseFrame(HumanSkeletonWithLegs skeleton) { + // Copy headset position + TransformNode rootNode = skeleton.getRootNode(); + this.rootPos = new Vector3f(rootNode.localTransform.getTranslation()); + + // Copy all rotations + this.rotations = new HashMap(); + rootNode.depthFirstTraversal(visitor -> { + // Insert a copied quaternion so it isn't changed by reference + rotations.put(visitor.getName(), new Quaternion(visitor.localTransform.getRotation())); + }); + } +} diff --git a/src/main/java/io/eiren/gui/autobone/SimpleSkeleton.java b/src/main/java/io/eiren/gui/autobone/SimpleSkeleton.java index 13744b8edf..71e44334f3 100644 --- a/src/main/java/io/eiren/gui/autobone/SimpleSkeleton.java +++ b/src/main/java/io/eiren/gui/autobone/SimpleSkeleton.java @@ -1,6 +1,10 @@ package io.eiren.gui.autobone; import java.util.HashMap; +import java.util.Map.Entry; + +import com.jme3.math.Quaternion; +import com.jme3.math.Vector3f; import io.eiren.vr.VRServer; import io.eiren.vr.processor.HumanSkeletonWithLegs; @@ -24,7 +28,7 @@ public class SimpleSkeleton { */ protected float waistDistance = 0.85f; /** - * Distacne from eyes to the base of the neck + * Distance from eyes to the base of the neck */ protected float neckLength = HumanSkeletonWithWaist.NECK_LENGTH_DEFAULT; /** @@ -56,7 +60,7 @@ public class SimpleSkeleton { protected float legsLength = 0.84f; protected float footLength = HumanSkeletonWithLegs.FOOT_LENGTH_DEFAULT; - HashMap nodes = new HashMap(); + protected final HashMap nodes = new HashMap(); public SimpleSkeleton(VRServer server) { this.server = server; @@ -117,8 +121,14 @@ public SimpleSkeleton(VRServer server) { }); } - public void poseFromSkeleton(HumanSkeletonWithLegs humanSkeleton) { - humanSkeleton.getRootNode().depthFirstTraversal(visitor -> { + public void setPoseFromSkeleton(HumanSkeletonWithLegs humanSkeleton) { + TransformNode rootNode = humanSkeleton.getRootNode(); + + // Copy headset position + hmdNode.localTransform.setTranslation(rootNode.localTransform.getTranslation()); + + // Copy all rotations + rootNode.depthFirstTraversal(visitor -> { TransformNode targetNode = nodes.get(visitor.getName()); // Handle unexpected nodes gracefully @@ -128,6 +138,21 @@ public void poseFromSkeleton(HumanSkeletonWithLegs humanSkeleton) { }); } + public void setPoseFromFrame(PoseFrame frame) { + // Copy headset position + hmdNode.localTransform.setTranslation(frame.rootPos); + + // Copy all rotations + for (Entry rotation : frame.rotations.entrySet()) { + TransformNode targetNode = nodes.get(rotation.getKey()); + + // Handle unexpected nodes gracefully + if (targetNode != null) { + targetNode.localTransform.setRotation(rotation.getValue()); + } + } + } + public void setSkeletonConfig(String joint, float newLength) { switch(joint) { case "Head": @@ -172,6 +197,18 @@ public void setSkeletonConfig(String joint, float newLength) { } } + public void updatePose() { + hmdNode.update(); + } + + public Vector3f getLeftFootPos() { + return leftAnkleNode.worldTransform.getTranslation(); + } + + public Vector3f getRightFootPos() { + return rightAnkleNode.worldTransform.getTranslation(); + } + public void saveConfigs() { // Save waist configs server.config.setProperty("body.headShift", headShift); From d35760d3a27ee3d52304bc1d8317778853f3b4d3 Mon Sep 17 00:00:00 2001 From: ButterscotchVanilla Date: Tue, 10 Aug 2021 18:11:36 -0400 Subject: [PATCH 05/60] AutoBone: Add basic PoseFrame recording and start processing loop --- .../java/io/eiren/gui/autobone/AutoBone.java | 111 ++++++++++++++++++ 1 file changed, 111 insertions(+) diff --git a/src/main/java/io/eiren/gui/autobone/AutoBone.java b/src/main/java/io/eiren/gui/autobone/AutoBone.java index e02c91629e..4a6e4a7b17 100644 --- a/src/main/java/io/eiren/gui/autobone/AutoBone.java +++ b/src/main/java/io/eiren/gui/autobone/AutoBone.java @@ -1,13 +1,124 @@ package io.eiren.gui.autobone; +import io.eiren.util.ann.ThreadSafe; import io.eiren.vr.VRServer; +import io.eiren.vr.processor.HumanSkeleton; +import io.eiren.vr.processor.HumanSkeletonWithLegs; +import io.eiren.vr.processor.HumanSkeletonWithWaist; public class AutoBone { protected final VRServer server; + protected PoseFrame[] frames = new PoseFrame[0]; + protected int frameRecordingCursor = -1; + + protected final SimpleSkeleton skeleton1; + protected final SimpleSkeleton skeleton2; + + // Waist + protected float chestDistance = 0.42f; + /** + * Distance from eyes to waist + */ + protected float waistDistance = 0.85f; + /** + * Distance from eyes to the base of the neck + */ + protected float neckLength = HumanSkeletonWithWaist.NECK_LENGTH_DEFAULT; + /** + * Distance from eyes to ear + */ + protected float headShift = HumanSkeletonWithWaist.HEAD_SHIFT_DEFAULT; + + // Legs + /** + * Distance between centers of both hips + */ + protected float hipsWidth = HumanSkeletonWithLegs.HIPS_WIDTH_DEFAULT; + /** + * Length from waist to knees + */ + protected float kneeHeight = 0.42f; + /** + * Distance from waist to ankle + */ + protected float legsLength = 0.84f; + protected float footLength = HumanSkeletonWithLegs.FOOT_LENGTH_DEFAULT; + public AutoBone(VRServer server) { this.server = server; + + reloadConfigValues(); + + this.skeleton1 = new SimpleSkeleton(server); + this.skeleton2 = new SimpleSkeleton(server); + + server.addSkeletonUpdatedCallback(this::skeletonUpdated); + } + + public void reloadConfigValues() { + // Load waist configs + headShift = server.config.getFloat("body.headShift", HumanSkeletonWithWaist.HEAD_SHIFT_DEFAULT); + neckLength = server.config.getFloat("body.neckLength", HumanSkeletonWithWaist.NECK_LENGTH_DEFAULT); + chestDistance = server.config.getFloat("body.chestDistance", 0.42f); + waistDistance = server.config.getFloat("body.waistDistance", 0.85f); + + // Load leg configs + hipsWidth = server.config.getFloat("body.hipsWidth", HumanSkeletonWithLegs.HIPS_WIDTH_DEFAULT); + kneeHeight = server.config.getFloat("body.kneeHeight", 0.42f); + legsLength = server.config.getFloat("body.legsLength", 0.84f); + footLength = server.config.getFloat("body.footLength", HumanSkeletonWithLegs.FOOT_LENGTH_DEFAULT); + } + + public void setSkeletonLengths(SimpleSkeleton skeleton) { + skeleton.setSkeletonConfig("Head", headShift); + skeleton.setSkeletonConfig("Neck", neckLength); + skeleton.setSkeletonConfig("Waist", waistDistance); + skeleton.setSkeletonConfig("Chest", chestDistance); + skeleton.setSkeletonConfig("Hips width", hipsWidth); + skeleton.setSkeletonConfig("Knee height", kneeHeight); + skeleton.setSkeletonConfig("Legs length", legsLength); + skeleton.setSkeletonConfig("Foot length", footLength); } + @ThreadSafe + public void skeletonUpdated(HumanSkeleton newSkeleton) { + if (frameRecordingCursor >= 0 && frameRecordingCursor < frames.length && newSkeleton instanceof HumanSkeletonWithLegs) { + HumanSkeletonWithLegs newLegSkeleton = (HumanSkeletonWithLegs)newSkeleton; + PoseFrame frame = new PoseFrame(newLegSkeleton); + frames[frameRecordingCursor++] = frame; + } + } + + public void startFrameRecording(int numFrames) { + frames = new PoseFrame[numFrames]; + frameRecordingCursor = 0; + } + + public void stopFrameRecording() { + // Set to end of the frame array to prevent race condition with `frameRecordingCursor++` + frameRecordingCursor = frames.length; + } + + public void processFrames() { + int cursorOffset = 1; + + for (;;) { + // Detect end of iteration + if (cursorOffset >= frames.length) + break; + + int frameCursor1 = 0; + int frameCursor2 = cursorOffset++; + + do { + PoseFrame frame1 = frames[frameCursor1]; + PoseFrame frame2 = frames[frameCursor2]; + + skeleton1.setPoseFromFrame(frame1); + skeleton2.setPoseFromFrame(frame2); + } while (++frameCursor1 < frames.length && ++frameCursor2 < frames.length); + } + } } From 2f46b3ff581ef9c73eaca0545070f8a29d0a05d0 Mon Sep 17 00:00:00 2001 From: ButterscotchVanilla Date: Tue, 10 Aug 2021 19:01:23 -0400 Subject: [PATCH 06/60] AutoBone: Move configs to HashMap and finish implementing adjustment --- .../java/io/eiren/gui/autobone/AutoBone.java | 122 +++++++++++------- 1 file changed, 77 insertions(+), 45 deletions(-) diff --git a/src/main/java/io/eiren/gui/autobone/AutoBone.java b/src/main/java/io/eiren/gui/autobone/AutoBone.java index 4a6e4a7b17..819cd8b789 100644 --- a/src/main/java/io/eiren/gui/autobone/AutoBone.java +++ b/src/main/java/io/eiren/gui/autobone/AutoBone.java @@ -1,5 +1,8 @@ package io.eiren.gui.autobone; +import java.util.HashMap; +import java.util.Map.Entry; + import io.eiren.util.ann.ThreadSafe; import io.eiren.vr.VRServer; import io.eiren.vr.processor.HumanSkeleton; @@ -16,35 +19,16 @@ public class AutoBone { protected final SimpleSkeleton skeleton1; protected final SimpleSkeleton skeleton2; - // Waist - protected float chestDistance = 0.42f; - /** - * Distance from eyes to waist - */ - protected float waistDistance = 0.85f; - /** - * Distance from eyes to the base of the neck - */ - protected float neckLength = HumanSkeletonWithWaist.NECK_LENGTH_DEFAULT; - /** - * Distance from eyes to ear - */ - protected float headShift = HumanSkeletonWithWaist.HEAD_SHIFT_DEFAULT; - - // Legs - /** - * Distance between centers of both hips - */ - protected float hipsWidth = HumanSkeletonWithLegs.HIPS_WIDTH_DEFAULT; - /** - * Length from waist to knees - */ - protected float kneeHeight = 0.42f; - /** - * Distance from waist to ankle - */ - protected float legsLength = 0.84f; - protected float footLength = HumanSkeletonWithLegs.FOOT_LENGTH_DEFAULT; + public final HashMap configs = new HashMap() {{ + put("Head", HumanSkeletonWithWaist.HEAD_SHIFT_DEFAULT); + put("Neck", HumanSkeletonWithWaist.NECK_LENGTH_DEFAULT); + put("Waist", 0.85f); + put("Chest", 0.42f); + put("Hips width", HumanSkeletonWithLegs.HIPS_WIDTH_DEFAULT); + put("Knee height", 0.42f); + put("Legs length", 0.84f); + put("Foot length", HumanSkeletonWithLegs.FOOT_LENGTH_DEFAULT); + }}; public AutoBone(VRServer server) { this.server = server; @@ -59,27 +43,22 @@ public AutoBone(VRServer server) { public void reloadConfigValues() { // Load waist configs - headShift = server.config.getFloat("body.headShift", HumanSkeletonWithWaist.HEAD_SHIFT_DEFAULT); - neckLength = server.config.getFloat("body.neckLength", HumanSkeletonWithWaist.NECK_LENGTH_DEFAULT); - chestDistance = server.config.getFloat("body.chestDistance", 0.42f); - waistDistance = server.config.getFloat("body.waistDistance", 0.85f); + configs.put("Head", server.config.getFloat("body.headShift", HumanSkeletonWithWaist.HEAD_SHIFT_DEFAULT)); + configs.put("Neck", server.config.getFloat("body.neckLength", HumanSkeletonWithWaist.NECK_LENGTH_DEFAULT)); + configs.put("Waist", server.config.getFloat("body.waistDistance", 0.85f)); + configs.put("Chest", server.config.getFloat("body.chestDistance", 0.42f)); // Load leg configs - hipsWidth = server.config.getFloat("body.hipsWidth", HumanSkeletonWithLegs.HIPS_WIDTH_DEFAULT); - kneeHeight = server.config.getFloat("body.kneeHeight", 0.42f); - legsLength = server.config.getFloat("body.legsLength", 0.84f); - footLength = server.config.getFloat("body.footLength", HumanSkeletonWithLegs.FOOT_LENGTH_DEFAULT); + configs.put("Hips width", server.config.getFloat("body.hipsWidth", HumanSkeletonWithLegs.HIPS_WIDTH_DEFAULT)); + configs.put("Knee height", server.config.getFloat("body.kneeHeight", 0.42f)); + configs.put("Legs length", server.config.getFloat("body.legsLength", 0.84f)); + configs.put("Foot length", server.config.getFloat("body.footLength", HumanSkeletonWithLegs.FOOT_LENGTH_DEFAULT)); } public void setSkeletonLengths(SimpleSkeleton skeleton) { - skeleton.setSkeletonConfig("Head", headShift); - skeleton.setSkeletonConfig("Neck", neckLength); - skeleton.setSkeletonConfig("Waist", waistDistance); - skeleton.setSkeletonConfig("Chest", chestDistance); - skeleton.setSkeletonConfig("Hips width", hipsWidth); - skeleton.setSkeletonConfig("Knee height", kneeHeight); - skeleton.setSkeletonConfig("Legs length", legsLength); - skeleton.setSkeletonConfig("Foot length", footLength); + for (Entry entry : configs.entrySet()) { + skeleton.setSkeletonConfig(entry.getKey(), entry.getValue()); + } } @ThreadSafe @@ -103,6 +82,7 @@ public void stopFrameRecording() { public void processFrames() { int cursorOffset = 1; + float adjustRate = 0.5f; for (;;) { // Detect end of iteration @@ -116,9 +96,61 @@ public void processFrames() { PoseFrame frame1 = frames[frameCursor1]; PoseFrame frame2 = frames[frameCursor2]; + setSkeletonLengths(skeleton1); + setSkeletonLengths(skeleton2); + skeleton1.setPoseFromFrame(frame1); skeleton2.setPoseFromFrame(frame2); + + skeleton1.updatePose(); + skeleton2.updatePose(); + + float error = getFootError(skeleton1, skeleton2); + float adjustVal = error * adjustRate; + + for (Entry entry : configs.entrySet()) { + float newLength = entry.getValue() + adjustVal; + updateSekeletonBoneLength(entry.getKey(), newLength); + float newError = getFootError(skeleton1, skeleton2); + + if (newError >= error) { + newLength = entry.getValue() - adjustVal; + updateSekeletonBoneLength(entry.getKey(), newLength); + newError = getFootError(skeleton1, skeleton2); + + if (newError >= error) { + // Reset value and continue without getting new error values + updateSekeletonBoneLength(entry.getKey(), entry.getValue()); + continue; + } else { + configs.put(entry.getKey(), newLength); + } + } else { + configs.put(entry.getKey(), newLength); + } + + // Update values with the new length + error = getFootError(skeleton1, skeleton2); + adjustVal = error * adjustRate; + } } while (++frameCursor1 < frames.length && ++frameCursor2 < frames.length); } } + + protected static float getFootError(SimpleSkeleton skeleton1, SimpleSkeleton skeleton2) { + float distLeft = skeleton1.getLeftFootPos().distance(skeleton2.getLeftFootPos()); + float distRight = skeleton1.getRightFootPos().distance(skeleton2.getRightFootPos()); + + float avg = (distLeft + distRight) / 2f; + + return avg * avg; + } + + protected void updateSekeletonBoneLength(String joint, float newLength) { + skeleton1.setSkeletonConfig(joint, newLength); + skeleton2.setSkeletonConfig(joint, newLength); + + skeleton1.updatePose(); + skeleton2.updatePose(); + } } From 0a39c746a36f1b6407a55a333f49d162972faa6c Mon Sep 17 00:00:00 2001 From: ButterscotchVanilla Date: Tue, 10 Aug 2021 19:05:36 -0400 Subject: [PATCH 07/60] AutoBone: Add frame recording interval --- src/main/java/io/eiren/gui/autobone/AutoBone.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/main/java/io/eiren/gui/autobone/AutoBone.java b/src/main/java/io/eiren/gui/autobone/AutoBone.java index 819cd8b789..0e60e7d9a7 100644 --- a/src/main/java/io/eiren/gui/autobone/AutoBone.java +++ b/src/main/java/io/eiren/gui/autobone/AutoBone.java @@ -15,6 +15,8 @@ public class AutoBone { protected PoseFrame[] frames = new PoseFrame[0]; protected int frameRecordingCursor = -1; + protected int frameRecordingInterval = 1; + protected int frameIntervalCounter = 0; protected final SimpleSkeleton skeleton1; protected final SimpleSkeleton skeleton2; @@ -63,15 +65,18 @@ public void setSkeletonLengths(SimpleSkeleton skeleton) { @ThreadSafe public void skeletonUpdated(HumanSkeleton newSkeleton) { - if (frameRecordingCursor >= 0 && frameRecordingCursor < frames.length && newSkeleton instanceof HumanSkeletonWithLegs) { + if (frameRecordingCursor >= 0 && frameRecordingCursor < frames.length && newSkeleton instanceof HumanSkeletonWithLegs && frameIntervalCounter++ % frameRecordingInterval == 0) { + frameIntervalCounter = 0; + HumanSkeletonWithLegs newLegSkeleton = (HumanSkeletonWithLegs)newSkeleton; PoseFrame frame = new PoseFrame(newLegSkeleton); frames[frameRecordingCursor++] = frame; } } - public void startFrameRecording(int numFrames) { + public void startFrameRecording(int numFrames, int interval) { frames = new PoseFrame[numFrames]; + frameRecordingInterval = interval; frameRecordingCursor = 0; } From c163effe605c3d7d55e1040ae2cf4a29b483b79f Mon Sep 17 00:00:00 2001 From: ButterscotchVanilla Date: Tue, 10 Aug 2021 19:38:20 -0400 Subject: [PATCH 08/60] AutoBone: Add test button --- src/main/java/io/eiren/gui/SkeletonConfig.java | 15 +++++++++++++++ src/main/java/io/eiren/gui/autobone/AutoBone.java | 3 +++ 2 files changed, 18 insertions(+) diff --git a/src/main/java/io/eiren/gui/SkeletonConfig.java b/src/main/java/io/eiren/gui/SkeletonConfig.java index fa1f518781..7537b16f3e 100644 --- a/src/main/java/io/eiren/gui/SkeletonConfig.java +++ b/src/main/java/io/eiren/gui/SkeletonConfig.java @@ -11,22 +11,26 @@ import javax.swing.JLabel; import javax.swing.event.MouseInputAdapter; +import io.eiren.gui.autobone.AutoBone; import io.eiren.util.StringUtils; import io.eiren.util.ann.ThreadSafe; import io.eiren.vr.VRServer; import io.eiren.vr.processor.HumanSkeletonWithLegs; import io.eiren.vr.processor.HumanSkeleton; +import io.eiren.util.logging.LogManager; public class SkeletonConfig extends EJBag { private final VRServer server; private final VRServerGUI gui; + private final AutoBone autoBone; private Map labels = new HashMap<>(); public SkeletonConfig(VRServer server, VRServerGUI gui) { super(); this.server = server; this.gui = gui; + this.autoBone = new AutoBone(server); setAlignmentY(TOP_ALIGNMENT); server.humanPoseProcessor.addSkeletonUpdatedCallback(this::skeletonUpdated); @@ -91,6 +95,17 @@ public void itemStateChanged(ItemEvent e) { //*/ add(new TimedResetButton("Reset All", "All"), s(c(1, row, 1), 3, 1)); + add(new JButton("Auto") {{ + addMouseListener(new MouseInputAdapter() { + @Override + public void mouseClicked(MouseEvent e) { + LogManager.log.info("[AutoBone] Recording 250 samples at a 6 frame interval"); + autoBone.startFrameRecording(250, 6); + LogManager.log.info("[AutoBone] Done recording! Processing frames..."); + autoBone.processFrames(); + } + }); + }}, s(c(4, row, 1), 3, 1)); row++; add(new JLabel("Chest"), c(0, row, 1)); diff --git a/src/main/java/io/eiren/gui/autobone/AutoBone.java b/src/main/java/io/eiren/gui/autobone/AutoBone.java index 0e60e7d9a7..8a96f9bc08 100644 --- a/src/main/java/io/eiren/gui/autobone/AutoBone.java +++ b/src/main/java/io/eiren/gui/autobone/AutoBone.java @@ -4,6 +4,8 @@ import java.util.Map.Entry; import io.eiren.util.ann.ThreadSafe; +import io.eiren.util.logging.LogManager; +import io.eiren.util.StringUtils; import io.eiren.vr.VRServer; import io.eiren.vr.processor.HumanSkeleton; import io.eiren.vr.processor.HumanSkeletonWithLegs; @@ -112,6 +114,7 @@ public void processFrames() { float error = getFootError(skeleton1, skeleton2); float adjustVal = error * adjustRate; + LogManager.log.info("[AutoBone] Current position error: " + StringUtils.prettyNumber(error)); for (Entry entry : configs.entrySet()) { float newLength = entry.getValue() + adjustVal; From 644fee2d1fc376bf637dcf648659ab01b9e08071 Mon Sep 17 00:00:00 2001 From: ButterscotchVanilla Date: Tue, 10 Aug 2021 19:59:40 -0400 Subject: [PATCH 09/60] AutoBone: Make auto-adjustment wait for recording to finish --- .../java/io/eiren/gui/SkeletonConfig.java | 86 +++++++++++++------ .../java/io/eiren/gui/autobone/AutoBone.java | 8 ++ 2 files changed, 70 insertions(+), 24 deletions(-) diff --git a/src/main/java/io/eiren/gui/SkeletonConfig.java b/src/main/java/io/eiren/gui/SkeletonConfig.java index 7537b16f3e..d66a3a6607 100644 --- a/src/main/java/io/eiren/gui/SkeletonConfig.java +++ b/src/main/java/io/eiren/gui/SkeletonConfig.java @@ -5,6 +5,7 @@ import java.awt.event.MouseEvent; import java.util.HashMap; import java.util.Map; +import java.util.Map.Entry; import javax.swing.JButton; import javax.swing.JCheckBox; @@ -20,12 +21,12 @@ import io.eiren.util.logging.LogManager; public class SkeletonConfig extends EJBag { - + private final VRServer server; private final VRServerGUI gui; private final AutoBone autoBone; private Map labels = new HashMap<>(); - + public SkeletonConfig(VRServer server, VRServerGUI gui) { super(); this.server = server; @@ -36,12 +37,12 @@ public SkeletonConfig(VRServer server, VRServerGUI gui) { server.humanPoseProcessor.addSkeletonUpdatedCallback(this::skeletonUpdated); skeletonUpdated(null); } - + @ThreadSafe public void skeletonUpdated(HumanSkeleton newSkeleton) { java.awt.EventQueue.invokeLater(() -> { removeAll(); - + int row = 0; add(new JCheckBox("Extended pelvis model") {{ @@ -67,7 +68,7 @@ public void itemStateChanged(ItemEvent e) { } }}, s(c(0, row, 1), 3, 1)); row++; - + /* add(new JCheckBox("Extended knee model") {{ addItemListener(new ItemListener() { @@ -93,28 +94,65 @@ public void itemStateChanged(ItemEvent e) { }}, s(c(0, row, 1), 3, 1)); row++; //*/ - + add(new TimedResetButton("Reset All", "All"), s(c(1, row, 1), 3, 1)); add(new JButton("Auto") {{ addMouseListener(new MouseInputAdapter() { @Override public void mouseClicked(MouseEvent e) { - LogManager.log.info("[AutoBone] Recording 250 samples at a 6 frame interval"); - autoBone.startFrameRecording(250, 6); - LogManager.log.info("[AutoBone] Done recording! Processing frames..."); - autoBone.processFrames(); + Thread thread = new Thread() { + @Override + public void run() { + try { + setText("Wait"); + LogManager.log.info("[AutoBone] Recording 250 samples at a 6 frame interval"); + autoBone.startFrameRecording(250, 6); + + while (autoBone.isRecording()) { + Thread.sleep(10); + } + + LogManager.log.info("[AutoBone] Done recording! Processing frames..."); + autoBone.processFrames(); + LogManager.log.info("[AutoBone] Done processing!"); + + boolean first = true; + StringBuilder configInfo = new StringBuilder("["); + + for (Entry entry : autoBone.configs.entrySet()) { + if (!first) { + configInfo.append(", "); + } else { + first = false; + } + + configInfo.append("{" + entry.getKey() + ": " + StringUtils.prettyNumber(entry.getValue()) + "}"); + } + + configInfo.append(']'); + + LogManager.log.info("[AutoBone] Length values: " + configInfo.toString()); + } catch (Exception e1) { + LogManager.log.severe("[AutoBone] Failed adjustment!", e1); + } finally { + setText("Auto"); + } + } + }; + + thread.start(); } }); }}, s(c(4, row, 1), 3, 1)); row++; - + add(new JLabel("Chest"), c(0, row, 1)); add(new AdjButton("+", "Chest", 0.01f), c(1, row, 1)); add(new SkeletonLabel("Chest"), c(2, row, 1)); add(new AdjButton("-", "Chest", -0.01f), c(3, row, 1)); add(new ResetButton("Reset", "Chest"), c(4, row, 1)); row++; - + add(new JLabel("Waist"), c(0, row, 1)); add(new AdjButton("+", "Waist", 0.01f), c(1, row, 1)); add(new SkeletonLabel("Waist"), c(2, row, 1)); @@ -163,14 +201,14 @@ public void mouseClicked(MouseEvent e) { add(new AdjButton("-", "Neck", -0.01f), c(3, row, 1)); add(new ResetButton("Reset", "Neck"), c(4, row, 1)); row++; - + add(new JLabel("Virtual waist"), c(0, row, 1)); add(new AdjButton("+", "Virtual waist", 0.01f), c(1, row, 1)); add(new SkeletonLabel("Virtual waist"), c(2, row, 1)); add(new AdjButton("-", "Virtual waist", -0.01f), c(3, row, 1)); add(new ResetButton("Reset", "Virtual waist"), c(4, row, 1)); row++; - + gui.refresh(); }); } @@ -183,14 +221,14 @@ public void refreshAll() { }); }); } - + private void change(String joint, float diff) { float current = server.humanPoseProcessor.getSkeletonConfig(joint); server.humanPoseProcessor.setSkeletonConfig(joint, current + diff); server.saveConfig(); labels.get(joint).setText(StringUtils.prettyNumber((current + diff) * 100, 0)); } - + private void reset(String joint) { server.humanPoseProcessor.resetSkeletonConfig(joint); server.saveConfig(); @@ -204,17 +242,17 @@ private void reset(String joint) { }); } } - + private class SkeletonLabel extends JLabel { - + public SkeletonLabel(String joint) { super(StringUtils.prettyNumber(server.humanPoseProcessor.getSkeletonConfig(joint) * 100, 0)); labels.put(joint, this); } } - + private class AdjButton extends JButton { - + public AdjButton(String text, String joint, float diff) { super(text); addMouseListener(new MouseInputAdapter() { @@ -225,9 +263,9 @@ public void mouseClicked(MouseEvent e) { }); } } - + private class ResetButton extends JButton { - + public ResetButton(String text, String joint) { super(text); addMouseListener(new MouseInputAdapter() { @@ -238,9 +276,9 @@ public void mouseClicked(MouseEvent e) { }); } } - + private class TimedResetButton extends JButton { - + public TimedResetButton(String text, String joint) { super(text); addMouseListener(new MouseInputAdapter() { diff --git a/src/main/java/io/eiren/gui/autobone/AutoBone.java b/src/main/java/io/eiren/gui/autobone/AutoBone.java index 8a96f9bc08..cb4ca080eb 100644 --- a/src/main/java/io/eiren/gui/autobone/AutoBone.java +++ b/src/main/java/io/eiren/gui/autobone/AutoBone.java @@ -87,6 +87,10 @@ public void stopFrameRecording() { frameRecordingCursor = frames.length; } + public boolean isRecording() { + return frameRecordingCursor >= 0 && frameRecordingCursor < frames.length; + } + public void processFrames() { int cursorOffset = 1; float adjustRate = 0.5f; @@ -103,6 +107,10 @@ public void processFrames() { PoseFrame frame1 = frames[frameCursor1]; PoseFrame frame2 = frames[frameCursor2]; + if (frame1 == null || frame2 == null) { + continue; + } + setSkeletonLengths(skeleton1); setSkeletonLengths(skeleton2); From 90e3715426544339d02457df4f849e80e3a0823c Mon Sep 17 00:00:00 2001 From: ButterscotchVanilla Date: Tue, 10 Aug 2021 21:23:46 -0400 Subject: [PATCH 10/60] AutoBone: Add serialization/deserialization of recording --- .../java/io/eiren/gui/SkeletonConfig.java | 3 +- .../io/eiren/gui/autobone/PoseRecordIO.java | 94 +++++++++++++++++++ 2 files changed, 96 insertions(+), 1 deletion(-) create mode 100644 src/main/java/io/eiren/gui/autobone/PoseRecordIO.java diff --git a/src/main/java/io/eiren/gui/SkeletonConfig.java b/src/main/java/io/eiren/gui/SkeletonConfig.java index d66a3a6607..a4fe19754a 100644 --- a/src/main/java/io/eiren/gui/SkeletonConfig.java +++ b/src/main/java/io/eiren/gui/SkeletonConfig.java @@ -104,7 +104,7 @@ public void mouseClicked(MouseEvent e) { @Override public void run() { try { - setText("Wait"); + setText("Move"); LogManager.log.info("[AutoBone] Recording 250 samples at a 6 frame interval"); autoBone.startFrameRecording(250, 6); @@ -113,6 +113,7 @@ public void run() { } LogManager.log.info("[AutoBone] Done recording! Processing frames..."); + setText("Wait"); autoBone.processFrames(); LogManager.log.info("[AutoBone] Done processing!"); diff --git a/src/main/java/io/eiren/gui/autobone/PoseRecordIO.java b/src/main/java/io/eiren/gui/autobone/PoseRecordIO.java new file mode 100644 index 0000000000..3e72610b76 --- /dev/null +++ b/src/main/java/io/eiren/gui/autobone/PoseRecordIO.java @@ -0,0 +1,94 @@ +package io.eiren.gui.autobone; + +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.util.HashMap; +import java.util.Map.Entry; + +import com.jme3.math.Quaternion; +import com.jme3.math.Vector3f; + +import io.eiren.util.logging.LogManager; + +public class PoseRecordIO { + + protected final File file; + + public PoseRecordIO(String file) { + this.file = new File(file); + } + + public boolean writeToFile(PoseFrame[] frames) { + try (DataOutputStream outputStream = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(file)))) { + // Write every frame + outputStream.writeInt(frames.length); + for (PoseFrame frame : frames) { + // Write root position vector + outputStream.writeFloat(frame.rootPos.x); + outputStream.writeFloat(frame.rootPos.y); + outputStream.writeFloat(frame.rootPos.z); + + // Write rotations + outputStream.writeInt(frame.rotations.size()); + for (Entry entry : frame.rotations.entrySet()) { + // Write the label string + outputStream.writeUTF(entry.getKey()); + + // Write the rotation quaternion + Quaternion quat = entry.getValue(); + outputStream.writeFloat(quat.getX()); + outputStream.writeFloat(quat.getY()); + outputStream.writeFloat(quat.getZ()); + outputStream.writeFloat(quat.getW()); + } + } + } catch (Exception e) { + LogManager.log.severe("Error writing to file", e); + return false; + } + + return true; + } + + public PoseFrame[] readFromFile() { + try (DataInputStream inputStream = new DataInputStream(new BufferedInputStream(new FileInputStream(file)))) { + int frameCount = inputStream.readInt(); + + PoseFrame[] frames = new PoseFrame[frameCount]; + for (int i = 0; i < frameCount; i++) { + float vecX = inputStream.readFloat(); + float vecY = inputStream.readFloat(); + float vecZ = inputStream.readFloat(); + + Vector3f vector = new Vector3f(vecX, vecY, vecZ); + + int rotationCount = inputStream.readInt(); + HashMap rotations = new HashMap(rotationCount); + for (int j = 0; j < rotationCount; j++) { + String label = inputStream.readUTF(); + + float quatX = inputStream.readFloat(); + float quatY = inputStream.readFloat(); + float quatZ = inputStream.readFloat(); + float quatW = inputStream.readFloat(); + Quaternion quaternion = new Quaternion(quatX, quatY, quatZ, quatW); + + rotations.put(label, quaternion); + } + + frames[i] = new PoseFrame(vector, rotations); + } + + return frames; + } catch (Exception e) { + LogManager.log.severe("Error reading from file", e); + } + + return null; + } +} From 110554a180cacb879e5eb1743cdea990305cc957 Mon Sep 17 00:00:00 2001 From: ButterscotchVanilla Date: Tue, 10 Aug 2021 21:27:44 -0400 Subject: [PATCH 11/60] AutoBone: Add recording export to process --- src/main/java/io/eiren/gui/SkeletonConfig.java | 8 +++++++- src/main/java/io/eiren/gui/autobone/AutoBone.java | 4 ++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/main/java/io/eiren/gui/SkeletonConfig.java b/src/main/java/io/eiren/gui/SkeletonConfig.java index a4fe19754a..64b79d31aa 100644 --- a/src/main/java/io/eiren/gui/SkeletonConfig.java +++ b/src/main/java/io/eiren/gui/SkeletonConfig.java @@ -13,6 +13,7 @@ import javax.swing.event.MouseInputAdapter; import io.eiren.gui.autobone.AutoBone; +import io.eiren.gui.autobone.PoseRecordIO; import io.eiren.util.StringUtils; import io.eiren.util.ann.ThreadSafe; import io.eiren.vr.VRServer; @@ -112,8 +113,13 @@ public void run() { Thread.sleep(10); } - LogManager.log.info("[AutoBone] Done recording! Processing frames..."); setText("Wait"); + LogManager.log.info("[AutoBone] Done recording! Exporting frames to \"ABRecording.abf\"..."); + + PoseRecordIO exporter = new PoseRecordIO("ABRecording.abf"); + exporter.writeToFile(autoBone.getFrames()); + + LogManager.log.info("[AutoBone] Done exporting! Processing frames..."); autoBone.processFrames(); LogManager.log.info("[AutoBone] Done processing!"); diff --git a/src/main/java/io/eiren/gui/autobone/AutoBone.java b/src/main/java/io/eiren/gui/autobone/AutoBone.java index cb4ca080eb..30c6d0872f 100644 --- a/src/main/java/io/eiren/gui/autobone/AutoBone.java +++ b/src/main/java/io/eiren/gui/autobone/AutoBone.java @@ -91,6 +91,10 @@ public boolean isRecording() { return frameRecordingCursor >= 0 && frameRecordingCursor < frames.length; } + public PoseFrame[] getFrames() { + return frames; + } + public void processFrames() { int cursorOffset = 1; float adjustRate = 0.5f; From 202b15e8a85aa784e6d232b20911bf1413870908 Mon Sep 17 00:00:00 2001 From: ButterscotchVanilla Date: Tue, 10 Aug 2021 21:46:03 -0400 Subject: [PATCH 12/60] Specify Java 8 compatibility --- build.gradle | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/build.gradle b/build.gradle index 73afa976c7..d13d5ece9f 100644 --- a/build.gradle +++ b/build.gradle @@ -11,6 +11,9 @@ plugins { id 'java-library' } +sourceCompatibility = 1.8 +targetCompatibility = 1.8 + repositories { // Use jcenter for resolving dependencies. // You can declare any Maven/Ivy/file repository here. @@ -31,7 +34,7 @@ dependencies { // This dependency is used internally, and not exposed to consumers on their own compile classpath. implementation 'com.google.guava:guava:28.2-jre' - + // Use JUnit test framework testImplementation platform('org.junit:junit-bom:5.7.2') @@ -48,7 +51,7 @@ task serverJar (type: Jar, dependsOn: subprojects.tasks['build']) { manifest { attributes 'Main-Class': 'io.eiren.vr.Main' } - + // Pack all dependencies within the JAR from { configurations.compile.collect { it.isDirectory() ? it : zipTree(it) } From 1408a5c357ec9f41efcdbdbe95b8e4e9fd1167a6 Mon Sep 17 00:00:00 2001 From: ButterscotchVanilla Date: Tue, 10 Aug 2021 22:56:19 -0400 Subject: [PATCH 13/60] AutoBone: Use skeleton properly and update on tick & add a timer for sampling --- .../java/io/eiren/gui/SkeletonConfig.java | 3 +- .../java/io/eiren/gui/autobone/AutoBone.java | 37 +++++++++++++++---- 2 files changed, 30 insertions(+), 10 deletions(-) diff --git a/src/main/java/io/eiren/gui/SkeletonConfig.java b/src/main/java/io/eiren/gui/SkeletonConfig.java index 64b79d31aa..cf237166a3 100644 --- a/src/main/java/io/eiren/gui/SkeletonConfig.java +++ b/src/main/java/io/eiren/gui/SkeletonConfig.java @@ -106,8 +106,7 @@ public void mouseClicked(MouseEvent e) { public void run() { try { setText("Move"); - LogManager.log.info("[AutoBone] Recording 250 samples at a 6 frame interval"); - autoBone.startFrameRecording(250, 6); + autoBone.startFrameRecording(250, 60); while (autoBone.isRecording()) { Thread.sleep(10); diff --git a/src/main/java/io/eiren/gui/autobone/AutoBone.java b/src/main/java/io/eiren/gui/autobone/AutoBone.java index 30c6d0872f..da084fa630 100644 --- a/src/main/java/io/eiren/gui/autobone/AutoBone.java +++ b/src/main/java/io/eiren/gui/autobone/AutoBone.java @@ -4,6 +4,7 @@ import java.util.Map.Entry; import io.eiren.util.ann.ThreadSafe; +import io.eiren.util.ann.VRServerThread; import io.eiren.util.logging.LogManager; import io.eiren.util.StringUtils; import io.eiren.vr.VRServer; @@ -15,10 +16,13 @@ public class AutoBone { protected final VRServer server; + HumanSkeletonWithLegs skeleton = null; + protected PoseFrame[] frames = new PoseFrame[0]; protected int frameRecordingCursor = -1; - protected int frameRecordingInterval = 1; - protected int frameIntervalCounter = 0; + + protected long frameRecordingInterval = 60L; + protected long lastFrameTimeMs = 0L; protected final SimpleSkeleton skeleton1; protected final SimpleSkeleton skeleton2; @@ -43,6 +47,7 @@ public AutoBone(VRServer server) { this.skeleton2 = new SimpleSkeleton(server); server.addSkeletonUpdatedCallback(this::skeletonUpdated); + server.addOnTick(this::onTick); } public void reloadConfigValues() { @@ -67,19 +72,35 @@ public void setSkeletonLengths(SimpleSkeleton skeleton) { @ThreadSafe public void skeletonUpdated(HumanSkeleton newSkeleton) { - if (frameRecordingCursor >= 0 && frameRecordingCursor < frames.length && newSkeleton instanceof HumanSkeletonWithLegs && frameIntervalCounter++ % frameRecordingInterval == 0) { - frameIntervalCounter = 0; + java.awt.EventQueue.invokeLater(() -> { + if (newSkeleton instanceof HumanSkeletonWithLegs) { + skeleton = (HumanSkeletonWithLegs)newSkeleton; + LogManager.log.info("Received updated skeleton"); + } + }); + } + + @VRServerThread + public void onTick() { + if (frameRecordingCursor >= 0 && frameRecordingCursor < frames.length && skeleton != null && System.currentTimeMillis() - lastFrameTimeMs >= frameRecordingInterval) { + lastFrameTimeMs = System.currentTimeMillis(); - HumanSkeletonWithLegs newLegSkeleton = (HumanSkeletonWithLegs)newSkeleton; - PoseFrame frame = new PoseFrame(newLegSkeleton); + PoseFrame frame = new PoseFrame(skeleton); frames[frameRecordingCursor++] = frame; + + LogManager.log.info("Recorded frame " + frameRecordingCursor); } } - public void startFrameRecording(int numFrames, int interval) { + public void startFrameRecording(int numFrames, long interval) { frames = new PoseFrame[numFrames]; + frameRecordingInterval = interval; + lastFrameTimeMs = 0L; + frameRecordingCursor = 0; + + LogManager.log.info("[AutoBone] Recording " + numFrames + " samples at a " + interval + " ms frame interval"); } public void stopFrameRecording() { @@ -126,7 +147,7 @@ public void processFrames() { float error = getFootError(skeleton1, skeleton2); float adjustVal = error * adjustRate; - LogManager.log.info("[AutoBone] Current position error: " + StringUtils.prettyNumber(error)); + LogManager.log.info("[AutoBone] Current position error: " + error); for (Entry entry : configs.entrySet()) { float newLength = entry.getValue() + adjustVal; From 84f4a47df18693471d404901f4a0c4747b33d09d Mon Sep 17 00:00:00 2001 From: ButterscotchVanilla Date: Tue, 10 Aug 2021 23:12:46 -0400 Subject: [PATCH 14/60] AutoBone: Load recordings from "ABRecording_Load.abf" --- .../java/io/eiren/gui/SkeletonConfig.java | 38 ++++++++++++++----- .../java/io/eiren/gui/autobone/AutoBone.java | 4 ++ 2 files changed, 32 insertions(+), 10 deletions(-) diff --git a/src/main/java/io/eiren/gui/SkeletonConfig.java b/src/main/java/io/eiren/gui/SkeletonConfig.java index cf237166a3..f25b326983 100644 --- a/src/main/java/io/eiren/gui/SkeletonConfig.java +++ b/src/main/java/io/eiren/gui/SkeletonConfig.java @@ -3,6 +3,7 @@ import java.awt.event.ItemEvent; import java.awt.event.ItemListener; import java.awt.event.MouseEvent; +import java.io.File; import java.util.HashMap; import java.util.Map; import java.util.Map.Entry; @@ -13,6 +14,7 @@ import javax.swing.event.MouseInputAdapter; import io.eiren.gui.autobone.AutoBone; +import io.eiren.gui.autobone.PoseFrame; import io.eiren.gui.autobone.PoseRecordIO; import io.eiren.util.StringUtils; import io.eiren.util.ann.ThreadSafe; @@ -105,20 +107,36 @@ public void mouseClicked(MouseEvent e) { @Override public void run() { try { - setText("Move"); - autoBone.startFrameRecording(250, 60); + File recording = new File("ABRecording_Load.abf"); - while (autoBone.isRecording()) { - Thread.sleep(10); - } + if (recording.exists()) { + setText("Load"); + PoseRecordIO importer = new PoseRecordIO("ABRecording_Load.abf"); + PoseFrame[] frames = importer.readFromFile(); + + if (frames == null) { + throw new NullPointerException("Reading frames from \"ABRecording_Load.abf\" failed..."); + } + + autoBone.setFrames(frames); - setText("Wait"); - LogManager.log.info("[AutoBone] Done recording! Exporting frames to \"ABRecording.abf\"..."); + setText("Wait"); + } else { + setText("Move"); + autoBone.startFrameRecording(250, 60); - PoseRecordIO exporter = new PoseRecordIO("ABRecording.abf"); - exporter.writeToFile(autoBone.getFrames()); + while (autoBone.isRecording()) { + Thread.sleep(10); + } + + setText("Wait"); + LogManager.log.info("[AutoBone] Done recording! Exporting frames to \"ABRecording.abf\"..."); + + PoseRecordIO exporter = new PoseRecordIO("ABRecording.abf"); + exporter.writeToFile(autoBone.getFrames()); + LogManager.log.info("[AutoBone] Done exporting! Processing frames..."); + } - LogManager.log.info("[AutoBone] Done exporting! Processing frames..."); autoBone.processFrames(); LogManager.log.info("[AutoBone] Done processing!"); diff --git a/src/main/java/io/eiren/gui/autobone/AutoBone.java b/src/main/java/io/eiren/gui/autobone/AutoBone.java index da084fa630..effce6ac29 100644 --- a/src/main/java/io/eiren/gui/autobone/AutoBone.java +++ b/src/main/java/io/eiren/gui/autobone/AutoBone.java @@ -116,6 +116,10 @@ public PoseFrame[] getFrames() { return frames; } + public void setFrames(PoseFrame[] frames) { + this.frames = frames; + } + public void processFrames() { int cursorOffset = 1; float adjustRate = 0.5f; From d9bcc39ee66f873d54c803d30d571fe8bc5e6bdd Mon Sep 17 00:00:00 2001 From: ButterscotchVanilla Date: Tue, 10 Aug 2021 23:20:20 -0400 Subject: [PATCH 15/60] AutoBone: Disable feet --- src/main/java/io/eiren/gui/autobone/AutoBone.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/io/eiren/gui/autobone/AutoBone.java b/src/main/java/io/eiren/gui/autobone/AutoBone.java index effce6ac29..ee6d4350d3 100644 --- a/src/main/java/io/eiren/gui/autobone/AutoBone.java +++ b/src/main/java/io/eiren/gui/autobone/AutoBone.java @@ -35,7 +35,7 @@ public class AutoBone { put("Hips width", HumanSkeletonWithLegs.HIPS_WIDTH_DEFAULT); put("Knee height", 0.42f); put("Legs length", 0.84f); - put("Foot length", HumanSkeletonWithLegs.FOOT_LENGTH_DEFAULT); + //put("Foot length", HumanSkeletonWithLegs.FOOT_LENGTH_DEFAULT); // Feet aren't actually used }}; public AutoBone(VRServer server) { @@ -61,7 +61,7 @@ public void reloadConfigValues() { configs.put("Hips width", server.config.getFloat("body.hipsWidth", HumanSkeletonWithLegs.HIPS_WIDTH_DEFAULT)); configs.put("Knee height", server.config.getFloat("body.kneeHeight", 0.42f)); configs.put("Legs length", server.config.getFloat("body.legsLength", 0.84f)); - configs.put("Foot length", server.config.getFloat("body.footLength", HumanSkeletonWithLegs.FOOT_LENGTH_DEFAULT)); + //configs.put("Foot length", server.config.getFloat("body.footLength", HumanSkeletonWithLegs.FOOT_LENGTH_DEFAULT)); // Feet aren't actually used } public void setSkeletonLengths(SimpleSkeleton skeleton) { From ef88e2e4a95640818d3a03388423c14de7b57be2 Mon Sep 17 00:00:00 2001 From: ButterscotchVanilla Date: Wed, 11 Aug 2021 00:23:55 -0400 Subject: [PATCH 16/60] AutoBone: Modify error function, add average error logs, and add tuning variables --- .../java/io/eiren/gui/autobone/AutoBone.java | 85 +++++++++++++++---- 1 file changed, 70 insertions(+), 15 deletions(-) diff --git a/src/main/java/io/eiren/gui/autobone/AutoBone.java b/src/main/java/io/eiren/gui/autobone/AutoBone.java index ee6d4350d3..20092c3f98 100644 --- a/src/main/java/io/eiren/gui/autobone/AutoBone.java +++ b/src/main/java/io/eiren/gui/autobone/AutoBone.java @@ -6,7 +6,6 @@ import io.eiren.util.ann.ThreadSafe; import io.eiren.util.ann.VRServerThread; import io.eiren.util.logging.LogManager; -import io.eiren.util.StringUtils; import io.eiren.vr.VRServer; import io.eiren.vr.processor.HumanSkeleton; import io.eiren.vr.processor.HumanSkeletonWithLegs; @@ -14,6 +13,11 @@ public class AutoBone { + protected final static int NUM_EPOCHS = 10; + + protected final static float INITIAL_ADJUSTMENT_RATE = 1f; + protected final static float ADJUSTMENT_RATE_DECAY = 1.05f; + protected final VRServer server; HumanSkeletonWithLegs skeleton = null; @@ -121,13 +125,38 @@ public void setFrames(PoseFrame[] frames) { } public void processFrames() { + int epochs = NUM_EPOCHS; + int epochCounter = 0; + int cursorOffset = 1; - float adjustRate = 0.5f; + + float adjustRate = INITIAL_ADJUSTMENT_RATE; + + float sumError = 0f; + int errorCount = 0; for (;;) { // Detect end of iteration - if (cursorOffset >= frames.length) - break; + if (cursorOffset >= frames.length) { + epochCounter++; + + // Calculate average error over the epoch + float avgError = errorCount > 0 ? sumError / errorCount : -1f; + + // Reset error sum values + sumError = 0f; + errorCount = 0; + + LogManager.log.info("[AutoBone] Epoch " + epochCounter + " average error: " + avgError); + + if (epochCounter >= epochs) { + break; + } else { + // Reset cursor offset and decay the adjustment rate + cursorOffset = 1; + adjustRate /= ADJUSTMENT_RATE_DECAY; + } + } int frameCursor1 = 0; int frameCursor2 = cursorOffset++; @@ -136,6 +165,7 @@ public void processFrames() { PoseFrame frame1 = frames[frameCursor1]; PoseFrame frame2 = frames[frameCursor2]; + // If there's missing data, skip it if (frame1 == null || frame2 == null) { continue; } @@ -150,22 +180,46 @@ public void processFrames() { skeleton2.updatePose(); float error = getFootError(skeleton1, skeleton2); + + // In case of fire + if (Float.isNaN(error) || Float.isInfinite(error)) { + // Extinguish + LogManager.log.warning("[AutoBone] Error value is invalid, resetting variables to recover"); + reloadConfigValues(); + + // Reset error sum values + sumError = 0f; + errorCount = 0; + + // Continue on new data + continue; + } + + // Store the error count for logging purposes + sumError += error; + errorCount++; + float adjustVal = error * adjustRate; - LogManager.log.info("[AutoBone] Current position error: " + error); + + // if (frameCursor1 == 0 && frameCursor2 == 1) { + // LogManager.log.info("[AutoBone] Current position error: " + error); + // } for (Entry entry : configs.entrySet()) { - float newLength = entry.getValue() + adjustVal; + float originalLength = entry.getValue(); + + float newLength = originalLength + adjustVal; updateSekeletonBoneLength(entry.getKey(), newLength); float newError = getFootError(skeleton1, skeleton2); if (newError >= error) { - newLength = entry.getValue() - adjustVal; + newLength = originalLength - adjustVal; updateSekeletonBoneLength(entry.getKey(), newLength); newError = getFootError(skeleton1, skeleton2); if (newError >= error) { // Reset value and continue without getting new error values - updateSekeletonBoneLength(entry.getKey(), entry.getValue()); + updateSekeletonBoneLength(entry.getKey(), originalLength); continue; } else { configs.put(entry.getKey(), newLength); @@ -175,20 +229,21 @@ public void processFrames() { } // Update values with the new length - error = getFootError(skeleton1, skeleton2); - adjustVal = error * adjustRate; + // error = getFootError(skeleton1, skeleton2); + // adjustVal = error * adjustRate; + + // Reset the length to minimize bias in other variables, it's applied later + updateSekeletonBoneLength(entry.getKey(), originalLength); } } while (++frameCursor1 < frames.length && ++frameCursor2 < frames.length); } } protected static float getFootError(SimpleSkeleton skeleton1, SimpleSkeleton skeleton2) { - float distLeft = skeleton1.getLeftFootPos().distance(skeleton2.getLeftFootPos()); - float distRight = skeleton1.getRightFootPos().distance(skeleton2.getRightFootPos()); - - float avg = (distLeft + distRight) / 2f; + float distLeft = skeleton1.getLeftFootPos().distanceSquared(skeleton2.getLeftFootPos()); + float distRight = skeleton1.getRightFootPos().distanceSquared(skeleton2.getRightFootPos()); - return avg * avg; + return distLeft + distRight; } protected void updateSekeletonBoneLength(String joint, float newLength) { From c6cd13d9cd506428960c0f196b535ff37def4cc8 Mon Sep 17 00:00:00 2001 From: ButterscotchVanilla Date: Wed, 11 Aug 2021 01:13:24 -0400 Subject: [PATCH 17/60] AutoBone: Add data distance controls to control amount of context between poses --- src/main/java/io/eiren/gui/autobone/AutoBone.java | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/main/java/io/eiren/gui/autobone/AutoBone.java b/src/main/java/io/eiren/gui/autobone/AutoBone.java index 20092c3f98..3f90c1ae23 100644 --- a/src/main/java/io/eiren/gui/autobone/AutoBone.java +++ b/src/main/java/io/eiren/gui/autobone/AutoBone.java @@ -13,7 +13,10 @@ public class AutoBone { - protected final static int NUM_EPOCHS = 10; + protected final static int MIN_DATA_DISTANCE = 5; + protected final static int MAX_DATA_DISTANCE = 10; + + protected final static int NUM_EPOCHS = 100; protected final static float INITIAL_ADJUSTMENT_RATE = 1f; protected final static float ADJUSTMENT_RATE_DECAY = 1.05f; @@ -128,7 +131,7 @@ public void processFrames() { int epochs = NUM_EPOCHS; int epochCounter = 0; - int cursorOffset = 1; + int cursorOffset = MIN_DATA_DISTANCE; float adjustRate = INITIAL_ADJUSTMENT_RATE; @@ -137,7 +140,7 @@ public void processFrames() { for (;;) { // Detect end of iteration - if (cursorOffset >= frames.length) { + if (cursorOffset >= frames.length || cursorOffset > MAX_DATA_DISTANCE) { epochCounter++; // Calculate average error over the epoch @@ -153,7 +156,7 @@ public void processFrames() { break; } else { // Reset cursor offset and decay the adjustment rate - cursorOffset = 1; + cursorOffset = MIN_DATA_DISTANCE; adjustRate /= ADJUSTMENT_RATE_DECAY; } } From e0ac3bb8534641ec167068d3c8de07f2c775c653 Mon Sep 17 00:00:00 2001 From: ButterscotchVanilla Date: Wed, 11 Aug 2021 21:18:33 -0400 Subject: [PATCH 18/60] AutoBone: Make PoseRecordIO static and add height to algorithm error --- .../java/io/eiren/gui/SkeletonConfig.java | 17 +-- .../java/io/eiren/gui/autobone/AutoBone.java | 39 ++++-- .../io/eiren/gui/autobone/PoseRecordIO.java | 128 ++++++++++++------ 3 files changed, 124 insertions(+), 60 deletions(-) diff --git a/src/main/java/io/eiren/gui/SkeletonConfig.java b/src/main/java/io/eiren/gui/SkeletonConfig.java index f25b326983..abb61fa6cc 100644 --- a/src/main/java/io/eiren/gui/SkeletonConfig.java +++ b/src/main/java/io/eiren/gui/SkeletonConfig.java @@ -107,20 +107,22 @@ public void mouseClicked(MouseEvent e) { @Override public void run() { try { - File recording = new File("ABRecording_Load.abf"); + File saveRecording = new File("ABRecording.abf"); + File loadRecording = new File("ABRecording_Load.abf"); - if (recording.exists()) { + if (loadRecording.exists()) { setText("Load"); - PoseRecordIO importer = new PoseRecordIO("ABRecording_Load.abf"); - PoseFrame[] frames = importer.readFromFile(); + LogManager.log.info("[AutoBone] Detected recording at \"" + loadRecording.getPath() + "\", loading frames..."); + PoseFrame[] frames = PoseRecordIO.readFromFile(loadRecording); if (frames == null) { - throw new NullPointerException("Reading frames from \"ABRecording_Load.abf\" failed..."); + throw new NullPointerException("Reading frames from \"" + loadRecording.getPath() + "\" failed..."); } autoBone.setFrames(frames); setText("Wait"); + LogManager.log.info("[AutoBone] Done loading frames! Processing frames..."); } else { setText("Move"); autoBone.startFrameRecording(250, 60); @@ -130,10 +132,9 @@ public void run() { } setText("Wait"); - LogManager.log.info("[AutoBone] Done recording! Exporting frames to \"ABRecording.abf\"..."); + LogManager.log.info("[AutoBone] Done recording! Exporting frames to \"" + saveRecording.getPath() + "\"..."); + PoseRecordIO.writeToFile(saveRecording, autoBone.getFrames()); - PoseRecordIO exporter = new PoseRecordIO("ABRecording.abf"); - exporter.writeToFile(autoBone.getFrames()); LogManager.log.info("[AutoBone] Done exporting! Processing frames..."); } diff --git a/src/main/java/io/eiren/gui/autobone/AutoBone.java b/src/main/java/io/eiren/gui/autobone/AutoBone.java index 3f90c1ae23..3c2586e513 100644 --- a/src/main/java/io/eiren/gui/autobone/AutoBone.java +++ b/src/main/java/io/eiren/gui/autobone/AutoBone.java @@ -13,13 +13,13 @@ public class AutoBone { - protected final static int MIN_DATA_DISTANCE = 5; + protected final static int MIN_DATA_DISTANCE = 4; protected final static int MAX_DATA_DISTANCE = 10; - protected final static int NUM_EPOCHS = 100; + protected final static int NUM_EPOCHS = 200; - protected final static float INITIAL_ADJUSTMENT_RATE = 1f; - protected final static float ADJUSTMENT_RATE_DECAY = 1.05f; + protected final static float INITIAL_ADJUSTMENT_RATE = 0.9f; + protected final static float ADJUSTMENT_RATE_DECAY = 1.057f; protected final VRServer server; @@ -35,7 +35,7 @@ public class AutoBone { protected final SimpleSkeleton skeleton2; public final HashMap configs = new HashMap() {{ - put("Head", HumanSkeletonWithWaist.HEAD_SHIFT_DEFAULT); + //put("Head", HumanSkeletonWithWaist.HEAD_SHIFT_DEFAULT); put("Neck", HumanSkeletonWithWaist.NECK_LENGTH_DEFAULT); put("Waist", 0.85f); put("Chest", 0.42f); @@ -59,7 +59,7 @@ public AutoBone(VRServer server) { public void reloadConfigValues() { // Load waist configs - configs.put("Head", server.config.getFloat("body.headShift", HumanSkeletonWithWaist.HEAD_SHIFT_DEFAULT)); + //configs.put("Head", server.config.getFloat("body.headShift", HumanSkeletonWithWaist.HEAD_SHIFT_DEFAULT)); configs.put("Neck", server.config.getFloat("body.neckLength", HumanSkeletonWithWaist.NECK_LENGTH_DEFAULT)); configs.put("Waist", server.config.getFloat("body.waistDistance", 0.85f)); configs.put("Chest", server.config.getFloat("body.chestDistance", 0.42f)); @@ -127,6 +127,16 @@ public void setFrames(PoseFrame[] frames) { this.frames = frames; } + public float getHeight() { + float height = 0f; + + for (float length : configs.values()) { + height += length; + } + + return height; + } + public void processFrames() { int epochs = NUM_EPOCHS; int epochCounter = 0; @@ -138,6 +148,8 @@ public void processFrames() { float sumError = 0f; int errorCount = 0; + float originalHeight = getHeight(); + for (;;) { // Detect end of iteration if (cursorOffset >= frames.length || cursorOffset > MAX_DATA_DISTANCE) { @@ -211,12 +223,21 @@ public void processFrames() { for (Entry entry : configs.entrySet()) { float originalLength = entry.getValue(); - float newLength = originalLength + adjustVal; + float heightChange = originalHeight - (getHeight() + adjustVal); + // Use Math.abs() to retain the sign while being squared + float heightAdjustVal = (heightChange * Math.abs(heightChange)); + + float newLength = originalLength + adjustVal + heightAdjustVal; updateSekeletonBoneLength(entry.getKey(), newLength); float newError = getFootError(skeleton1, skeleton2); if (newError >= error) { - newLength = originalLength - adjustVal; + + heightChange = originalHeight - (getHeight() - adjustVal); + // Use Math.abs() to retain the sign while being squared + heightAdjustVal = (heightChange * Math.abs(heightChange)); + + newLength = (originalLength - adjustVal) + heightAdjustVal; updateSekeletonBoneLength(entry.getKey(), newLength); newError = getFootError(skeleton1, skeleton2); @@ -240,6 +261,8 @@ public void processFrames() { } } while (++frameCursor1 < frames.length && ++frameCursor2 < frames.length); } + + LogManager.log.info("[AutoBone] Original height: " + originalHeight + " New height: " + getHeight()); } protected static float getFootError(SimpleSkeleton skeleton1, SimpleSkeleton skeleton2) { diff --git a/src/main/java/io/eiren/gui/autobone/PoseRecordIO.java b/src/main/java/io/eiren/gui/autobone/PoseRecordIO.java index 3e72610b76..9ea3508f31 100644 --- a/src/main/java/io/eiren/gui/autobone/PoseRecordIO.java +++ b/src/main/java/io/eiren/gui/autobone/PoseRecordIO.java @@ -15,78 +15,118 @@ import io.eiren.util.logging.LogManager; -public class PoseRecordIO { +public final class PoseRecordIO { - protected final File file; - - public PoseRecordIO(String file) { - this.file = new File(file); + private PoseRecordIO() { + // Do not allow instantiating } - public boolean writeToFile(PoseFrame[] frames) { + public static boolean writeToFile(File file, PoseFrame[] frames) { try (DataOutputStream outputStream = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(file)))) { // Write every frame outputStream.writeInt(frames.length); for (PoseFrame frame : frames) { - // Write root position vector - outputStream.writeFloat(frame.rootPos.x); - outputStream.writeFloat(frame.rootPos.y); - outputStream.writeFloat(frame.rootPos.z); - - // Write rotations - outputStream.writeInt(frame.rotations.size()); - for (Entry entry : frame.rotations.entrySet()) { - // Write the label string - outputStream.writeUTF(entry.getKey()); - - // Write the rotation quaternion - Quaternion quat = entry.getValue(); - outputStream.writeFloat(quat.getX()); - outputStream.writeFloat(quat.getY()); - outputStream.writeFloat(quat.getZ()); - outputStream.writeFloat(quat.getW()); - } + writeFrame(outputStream, frame); + } + } catch (Exception e) { + LogManager.log.severe("Error writing frames to file", e); + return false; + } + + return true; + } + + public static boolean writeFrame(DataOutputStream outputStream, PoseFrame frame) { + try { + // Write root position vector + outputStream.writeFloat(frame.rootPos.x); + outputStream.writeFloat(frame.rootPos.y); + outputStream.writeFloat(frame.rootPos.z); + + // Write rotations + outputStream.writeInt(frame.rotations.size()); + for (Entry entry : frame.rotations.entrySet()) { + // Write the label string + outputStream.writeUTF(entry.getKey()); + + // Write the rotation quaternion + Quaternion quat = entry.getValue(); + outputStream.writeFloat(quat.getX()); + outputStream.writeFloat(quat.getY()); + outputStream.writeFloat(quat.getZ()); + outputStream.writeFloat(quat.getW()); } } catch (Exception e) { - LogManager.log.severe("Error writing to file", e); + LogManager.log.severe("Error writing frame to stream", e); + return false; + } + + return true; + } + + public static boolean writeFrame(File file, PoseFrame frame) { + try (DataOutputStream outputStream = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(file)))) { + writeFrame(outputStream, frame); + } catch (Exception e) { + LogManager.log.severe("Error writing frame to file", e); return false; } return true; } - public PoseFrame[] readFromFile() { + public static PoseFrame[] readFromFile(File file) { try (DataInputStream inputStream = new DataInputStream(new BufferedInputStream(new FileInputStream(file)))) { int frameCount = inputStream.readInt(); PoseFrame[] frames = new PoseFrame[frameCount]; for (int i = 0; i < frameCount; i++) { - float vecX = inputStream.readFloat(); - float vecY = inputStream.readFloat(); - float vecZ = inputStream.readFloat(); + frames[i] = readFrame(inputStream); + } + + return frames; + } catch (Exception e) { + LogManager.log.severe("Error reading frames from file", e); + } + + return null; + } - Vector3f vector = new Vector3f(vecX, vecY, vecZ); + public static PoseFrame readFrame(DataInputStream inputStream) { + try { + float vecX = inputStream.readFloat(); + float vecY = inputStream.readFloat(); + float vecZ = inputStream.readFloat(); - int rotationCount = inputStream.readInt(); - HashMap rotations = new HashMap(rotationCount); - for (int j = 0; j < rotationCount; j++) { - String label = inputStream.readUTF(); + Vector3f vector = new Vector3f(vecX, vecY, vecZ); - float quatX = inputStream.readFloat(); - float quatY = inputStream.readFloat(); - float quatZ = inputStream.readFloat(); - float quatW = inputStream.readFloat(); - Quaternion quaternion = new Quaternion(quatX, quatY, quatZ, quatW); + int rotationCount = inputStream.readInt(); + HashMap rotations = new HashMap(rotationCount); + for (int j = 0; j < rotationCount; j++) { + String label = inputStream.readUTF(); - rotations.put(label, quaternion); - } + float quatX = inputStream.readFloat(); + float quatY = inputStream.readFloat(); + float quatZ = inputStream.readFloat(); + float quatW = inputStream.readFloat(); + Quaternion quaternion = new Quaternion(quatX, quatY, quatZ, quatW); - frames[i] = new PoseFrame(vector, rotations); + rotations.put(label, quaternion); } - return frames; + return new PoseFrame(vector, rotations); + } catch (Exception e) { + LogManager.log.severe("Error reading frame from stream", e); + } + + return null; + } + + public static PoseFrame readFrame(File file) { + try { + return readFrame(new DataInputStream(new BufferedInputStream(new FileInputStream(file)))); } catch (Exception e) { - LogManager.log.severe("Error reading from file", e); + LogManager.log.severe("Error reading frame from file", e); } return null; From 1a078993f37c2d6a131523bffad2b5edd3e5fcd2 Mon Sep 17 00:00:00 2001 From: ButterscotchVanilla Date: Wed, 11 Aug 2021 22:28:06 -0400 Subject: [PATCH 19/60] AutoBone: Simplify length adjustment code --- .../java/io/eiren/gui/autobone/AutoBone.java | 40 ++++++------------- 1 file changed, 12 insertions(+), 28 deletions(-) diff --git a/src/main/java/io/eiren/gui/autobone/AutoBone.java b/src/main/java/io/eiren/gui/autobone/AutoBone.java index 3c2586e513..799278ce48 100644 --- a/src/main/java/io/eiren/gui/autobone/AutoBone.java +++ b/src/main/java/io/eiren/gui/autobone/AutoBone.java @@ -13,12 +13,12 @@ public class AutoBone { - protected final static int MIN_DATA_DISTANCE = 4; - protected final static int MAX_DATA_DISTANCE = 10; + protected final static int MIN_DATA_DISTANCE = 20; + protected final static int MAX_DATA_DISTANCE = 40; protected final static int NUM_EPOCHS = 200; - protected final static float INITIAL_ADJUSTMENT_RATE = 0.9f; + protected final static float INITIAL_ADJUSTMENT_RATE = 0.5f; protected final static float ADJUSTMENT_RATE_DECAY = 1.057f; protected final VRServer server; @@ -223,39 +223,23 @@ public void processFrames() { for (Entry entry : configs.entrySet()) { float originalLength = entry.getValue(); - float heightChange = originalHeight - (getHeight() + adjustVal); - // Use Math.abs() to retain the sign while being squared - float heightAdjustVal = (heightChange * Math.abs(heightChange)); + // Try positive and negative adjustments + for (int i = 0; i < 2; i++) { + float curAdjustVal = i == 0 ? adjustVal : -adjustVal; - float newLength = originalLength + adjustVal + heightAdjustVal; - updateSekeletonBoneLength(entry.getKey(), newLength); - float newError = getFootError(skeleton1, skeleton2); + float heightChange = originalHeight - (getHeight() + curAdjustVal); + float heightAdjustVal = heightChange * Math.abs(heightChange); // Use Math.abs to retain the sign - if (newError >= error) { - - heightChange = originalHeight - (getHeight() - adjustVal); - // Use Math.abs() to retain the sign while being squared - heightAdjustVal = (heightChange * Math.abs(heightChange)); - - newLength = (originalLength - adjustVal) + heightAdjustVal; + float newLength = originalLength + curAdjustVal + heightAdjustVal; updateSekeletonBoneLength(entry.getKey(), newLength); - newError = getFootError(skeleton1, skeleton2); + float newError = getFootError(skeleton1, skeleton2); - if (newError >= error) { - // Reset value and continue without getting new error values - updateSekeletonBoneLength(entry.getKey(), originalLength); - continue; - } else { + if (newError < error) { configs.put(entry.getKey(), newLength); + break; } - } else { - configs.put(entry.getKey(), newLength); } - // Update values with the new length - // error = getFootError(skeleton1, skeleton2); - // adjustVal = error * adjustRate; - // Reset the length to minimize bias in other variables, it's applied later updateSekeletonBoneLength(entry.getKey(), originalLength); } From faf0be6c53abc9c336e94f4e444c1ef5f934170e Mon Sep 17 00:00:00 2001 From: ButterscotchVanilla Date: Thu, 12 Aug 2021 01:01:20 -0400 Subject: [PATCH 20/60] AutoBone: Fine-tune algorithm and error function, apply results to skeleton --- .../java/io/eiren/gui/autobone/AutoBone.java | 110 +++++++++++++----- .../io/eiren/gui/autobone/SimpleSkeleton.java | 4 + 2 files changed, 86 insertions(+), 28 deletions(-) diff --git a/src/main/java/io/eiren/gui/autobone/AutoBone.java b/src/main/java/io/eiren/gui/autobone/AutoBone.java index 799278ce48..11533aabf1 100644 --- a/src/main/java/io/eiren/gui/autobone/AutoBone.java +++ b/src/main/java/io/eiren/gui/autobone/AutoBone.java @@ -13,13 +13,17 @@ public class AutoBone { - protected final static int MIN_DATA_DISTANCE = 20; - protected final static int MAX_DATA_DISTANCE = 40; + protected final static int MIN_DATA_DISTANCE = 15; + protected final static int MAX_DATA_DISTANCE = 50; - protected final static int NUM_EPOCHS = 200; + protected final static int NUM_EPOCHS = 50; - protected final static float INITIAL_ADJUSTMENT_RATE = 0.5f; - protected final static float ADJUSTMENT_RATE_DECAY = 1.057f; + protected final static float INITIAL_ADJUSTMENT_RATE = 1f; + protected final static float ADJUSTMENT_RATE_DECAY = 1.4f; + + protected final static float SLIDE_ERROR_FACTOR = 1f; + protected final static float OFFSET_ERROR_FACTOR = 0.75f; + protected final static float HEIGHT_ERROR_FACTOR = 1f; protected final VRServer server; @@ -34,16 +38,8 @@ public class AutoBone { protected final SimpleSkeleton skeleton1; protected final SimpleSkeleton skeleton2; - public final HashMap configs = new HashMap() {{ - //put("Head", HumanSkeletonWithWaist.HEAD_SHIFT_DEFAULT); - put("Neck", HumanSkeletonWithWaist.NECK_LENGTH_DEFAULT); - put("Waist", 0.85f); - put("Chest", 0.42f); - put("Hips width", HumanSkeletonWithLegs.HIPS_WIDTH_DEFAULT); - put("Knee height", 0.42f); - put("Legs length", 0.84f); - //put("Foot length", HumanSkeletonWithLegs.FOOT_LENGTH_DEFAULT); // Feet aren't actually used - }}; + // This is filled by reloadConfigValues() + public final HashMap configs = new HashMap(); public AutoBone(VRServer server) { this.server = server; @@ -59,7 +55,7 @@ public AutoBone(VRServer server) { public void reloadConfigValues() { // Load waist configs - //configs.put("Head", server.config.getFloat("body.headShift", HumanSkeletonWithWaist.HEAD_SHIFT_DEFAULT)); + configs.put("Head", server.config.getFloat("body.headShift", HumanSkeletonWithWaist.HEAD_SHIFT_DEFAULT)); configs.put("Neck", server.config.getFloat("body.neckLength", HumanSkeletonWithWaist.NECK_LENGTH_DEFAULT)); configs.put("Waist", server.config.getFloat("body.waistDistance", 0.85f)); configs.put("Chest", server.config.getFloat("body.chestDistance", 0.42f)); @@ -82,11 +78,24 @@ public void skeletonUpdated(HumanSkeleton newSkeleton) { java.awt.EventQueue.invokeLater(() -> { if (newSkeleton instanceof HumanSkeletonWithLegs) { skeleton = (HumanSkeletonWithLegs)newSkeleton; + applyConfigToSkeleton(newSkeleton); LogManager.log.info("Received updated skeleton"); } }); } + public boolean applyConfigToSkeleton(HumanSkeleton skeleton) { + if (skeleton == null) { + return false; + } + + for (Entry entry : configs.entrySet()) { + skeleton.setSkeletonConfig(entry.getKey(), entry.getValue()); + } + + return true; + } + @VRServerThread public void onTick() { if (frameRecordingCursor >= 0 && frameRecordingCursor < frames.length && skeleton != null && System.currentTimeMillis() - lastFrameTimeMs >= frameRecordingInterval) { @@ -130,8 +139,14 @@ public void setFrames(PoseFrame[] frames) { public float getHeight() { float height = 0f; - for (float length : configs.values()) { - height += length; + Float waistLength = configs.get("Waist"); + if (waistLength != null) { + height += waistLength; + } + + Float legsLength = configs.get("Legs length"); + if (legsLength != null) { + height += legsLength; } return height; @@ -139,7 +154,7 @@ public float getHeight() { public void processFrames() { int epochs = NUM_EPOCHS; - int epochCounter = 0; + int epochCounter = -1; int cursorOffset = MIN_DATA_DISTANCE; @@ -194,7 +209,7 @@ public void processFrames() { skeleton1.updatePose(); skeleton2.updatePose(); - float error = getFootError(skeleton1, skeleton2); + float error = getError(skeleton1, skeleton2, getHeight() - originalHeight); // In case of fire if (Float.isNaN(error) || Float.isInfinite(error)) { @@ -221,18 +236,44 @@ public void processFrames() { // } for (Entry entry : configs.entrySet()) { + // Skip adjustment if the epoch is before starting (for logging only) + if (epochCounter < 0) { + break; + } + float originalLength = entry.getValue(); // Try positive and negative adjustments for (int i = 0; i < 2; i++) { float curAdjustVal = i == 0 ? adjustVal : -adjustVal; + float newLength = originalLength + curAdjustVal; + + // No small or negative numbers!!! Bad algorithm! + if (newLength < 0.01f) { + continue; + } - float heightChange = originalHeight - (getHeight() + curAdjustVal); - float heightAdjustVal = heightChange * Math.abs(heightChange); // Use Math.abs to retain the sign + Float val; + switch (entry.getKey()) { + case "Chest": + // If the chest length is an invalid value, skip it + val = configs.get("Waist"); + if (val == null || newLength >= 0.75f * val) { + continue; + } + break; + + case "Knee height": + val = configs.get("Legs length"); + // If the knee height is an invalid value, skip it + if (val == null || newLength >= 0.75f * val) { + continue; + } + break; + } - float newLength = originalLength + curAdjustVal + heightAdjustVal; updateSekeletonBoneLength(entry.getKey(), newLength); - float newError = getFootError(skeleton1, skeleton2); + float newError = getError(skeleton1, skeleton2, (getHeight() + curAdjustVal) - originalHeight); if (newError < error) { configs.put(entry.getKey(), newLength); @@ -247,13 +288,26 @@ public void processFrames() { } LogManager.log.info("[AutoBone] Original height: " + originalHeight + " New height: " + getHeight()); + + LogManager.log.info("[AutoBone] Done! Applying to skeleton..."); + applyConfigToSkeleton(skeleton); } - protected static float getFootError(SimpleSkeleton skeleton1, SimpleSkeleton skeleton2) { - float distLeft = skeleton1.getLeftFootPos().distanceSquared(skeleton2.getLeftFootPos()); - float distRight = skeleton1.getRightFootPos().distanceSquared(skeleton2.getRightFootPos()); + protected static float getError(SimpleSkeleton skeleton1, SimpleSkeleton skeleton2, float heightChange) { + float slideLeft = skeleton1.getLeftFootPos().distance(skeleton2.getLeftFootPos()); + float slideRight = skeleton1.getRightFootPos().distance(skeleton2.getRightFootPos()); + + // Averaged error + float slideError = (slideLeft + slideRight) / 2f; + + float dist1 = skeleton1.getLeftFootPos().y - skeleton1.getRightFootPos().y; + float dist2 = skeleton2.getLeftFootPos().y - skeleton2.getRightFootPos().y; + + // Averaged error + float distError = (dist1 + dist2) / 2f; - return distLeft + distRight; + // Minimize sliding, minimize foot height offset, minimize change in total height + return (SLIDE_ERROR_FACTOR * (slideError * slideError)) + (OFFSET_ERROR_FACTOR * (distError * distError)) + (HEIGHT_ERROR_FACTOR * (heightChange * heightChange)); } protected void updateSekeletonBoneLength(String joint, float newLength) { diff --git a/src/main/java/io/eiren/gui/autobone/SimpleSkeleton.java b/src/main/java/io/eiren/gui/autobone/SimpleSkeleton.java index 71e44334f3..969ed95c10 100644 --- a/src/main/java/io/eiren/gui/autobone/SimpleSkeleton.java +++ b/src/main/java/io/eiren/gui/autobone/SimpleSkeleton.java @@ -201,6 +201,10 @@ public void updatePose() { hmdNode.update(); } + public Vector3f getHMDPos() { + return hmdNode.worldTransform.getTranslation(); + } + public Vector3f getLeftFootPos() { return leftAnkleNode.worldTransform.getTranslation(); } From efbe409399c85a692420193db98f55a50a099992 Mon Sep 17 00:00:00 2001 From: ButterscotchVanilla Date: Thu, 12 Aug 2021 01:09:55 -0400 Subject: [PATCH 21/60] AutoBone: Only allow one AutoBone thread --- src/main/java/io/eiren/gui/SkeletonConfig.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/main/java/io/eiren/gui/SkeletonConfig.java b/src/main/java/io/eiren/gui/SkeletonConfig.java index abb61fa6cc..4387dacd46 100644 --- a/src/main/java/io/eiren/gui/SkeletonConfig.java +++ b/src/main/java/io/eiren/gui/SkeletonConfig.java @@ -28,6 +28,7 @@ public class SkeletonConfig extends EJBag { private final VRServer server; private final VRServerGUI gui; private final AutoBone autoBone; + private Thread autoBoneThread = null; private Map labels = new HashMap<>(); public SkeletonConfig(VRServer server, VRServerGUI gui) { @@ -103,6 +104,11 @@ public void itemStateChanged(ItemEvent e) { addMouseListener(new MouseInputAdapter() { @Override public void mouseClicked(MouseEvent e) { + // Prevent running multiple times + if (autoBoneThread != null) { + return; + } + Thread thread = new Thread() { @Override public void run() { @@ -161,10 +167,12 @@ public void run() { LogManager.log.severe("[AutoBone] Failed adjustment!", e1); } finally { setText("Auto"); + autoBoneThread = null; } } }; + autoBoneThread = thread; thread.start(); } }); From 707e4c6ddeb452430fa27a52a90f63d8478aebe6 Mon Sep 17 00:00:00 2001 From: ButterscotchVanilla Date: Thu, 12 Aug 2021 03:06:48 -0400 Subject: [PATCH 22/60] AutoBone: Auto-detect height, add more restrains, fine-tuning adjustment values --- .../java/io/eiren/gui/autobone/AutoBone.java | 64 ++++++++++++++----- 1 file changed, 49 insertions(+), 15 deletions(-) diff --git a/src/main/java/io/eiren/gui/autobone/AutoBone.java b/src/main/java/io/eiren/gui/autobone/AutoBone.java index 11533aabf1..40f90a8e10 100644 --- a/src/main/java/io/eiren/gui/autobone/AutoBone.java +++ b/src/main/java/io/eiren/gui/autobone/AutoBone.java @@ -13,17 +13,19 @@ public class AutoBone { - protected final static int MIN_DATA_DISTANCE = 15; - protected final static int MAX_DATA_DISTANCE = 50; + protected final static int MIN_DATA_DISTANCE = 1; + protected final static int MAX_DATA_DISTANCE = 1000; - protected final static int NUM_EPOCHS = 50; + protected final static int NUM_EPOCHS = 20; protected final static float INITIAL_ADJUSTMENT_RATE = 1f; - protected final static float ADJUSTMENT_RATE_DECAY = 1.4f; + protected final static float ADJUSTMENT_RATE_DECAY = 1.1f; protected final static float SLIDE_ERROR_FACTOR = 1f; protected final static float OFFSET_ERROR_FACTOR = 0.75f; - protected final static float HEIGHT_ERROR_FACTOR = 1f; + protected final static float HEIGHT_ERROR_FACTOR = 0.75f; + + protected final static float HEADSET_HEIGHT_RATIO = 0.91f; protected final VRServer server; @@ -33,7 +35,7 @@ public class AutoBone { protected int frameRecordingCursor = -1; protected long frameRecordingInterval = 60L; - protected long lastFrameTimeMs = 0L; + protected long lastFrameTimeMs = -1L; protected final SimpleSkeleton skeleton1; protected final SimpleSkeleton skeleton2; @@ -79,7 +81,7 @@ public void skeletonUpdated(HumanSkeleton newSkeleton) { if (newSkeleton instanceof HumanSkeletonWithLegs) { skeleton = (HumanSkeletonWithLegs)newSkeleton; applyConfigToSkeleton(newSkeleton); - LogManager.log.info("Received updated skeleton"); + LogManager.log.info("[AutoBone] Received updated skeleton"); } }); } @@ -93,6 +95,7 @@ public boolean applyConfigToSkeleton(HumanSkeleton skeleton) { skeleton.setSkeletonConfig(entry.getKey(), entry.getValue()); } + LogManager.log.info("[AutoBone] Configured skeleton bone lengths"); return true; } @@ -152,6 +155,16 @@ public float getHeight() { return height; } + public float getMaxHmdHeight(PoseFrame[] frames) { + float maxHeight = 0f; + for (PoseFrame frame : frames) { + if (frame.rootPos.y > maxHeight) { + maxHeight = frame.rootPos.y; + } + } + return maxHeight; + } + public void processFrames() { int epochs = NUM_EPOCHS; int epochCounter = -1; @@ -163,7 +176,15 @@ public void processFrames() { float sumError = 0f; int errorCount = 0; - float originalHeight = getHeight(); + float hmdHeight = getMaxHmdHeight(frames); + if (hmdHeight <= 0.50f) { + LogManager.log.warning("[AutoBone] Max headset height detected (Value seems too low, did you stand straight while measuring?): " + hmdHeight); + } else { + LogManager.log.info("[AutoBone] Max headset height detected: " + hmdHeight); + } + + // Estimate target height from HMD height + float targetHeight = hmdHeight * HEADSET_HEIGHT_RATIO; for (;;) { // Detect end of iteration @@ -209,7 +230,7 @@ public void processFrames() { skeleton1.updatePose(); skeleton2.updatePose(); - float error = getError(skeleton1, skeleton2, getHeight() - originalHeight); + float error = getError(skeleton1, skeleton2, getHeight() - targetHeight); // In case of fire if (Float.isNaN(error) || Float.isInfinite(error)) { @@ -253,27 +274,40 @@ public void processFrames() { continue; } + // Detect and skip invalid values Float val; switch (entry.getKey()) { case "Chest": - // If the chest length is an invalid value, skip it val = configs.get("Waist"); - if (val == null || newLength >= 0.75f * val) { + if (val == null || newLength <= 0.2f * val || newLength >= 0.6f * val) { + continue; + } + break; + + case "Hips width": + val = configs.get("Waist"); + if (val == null || newLength < 0.08f || newLength >= 0.4f * val) { continue; } break; case "Knee height": val = configs.get("Legs length"); - // If the knee height is an invalid value, skip it - if (val == null || newLength >= 0.75f * val) { + if (val == null || newLength <= 0.375f * val || newLength >= 0.625f * val) { + continue; + } + break; + + case "Legs length": + val = configs.get("Waist"); + if (val == null || newLength <= 0.667f * val || newLength >= 1.333f * val) { continue; } break; } updateSekeletonBoneLength(entry.getKey(), newLength); - float newError = getError(skeleton1, skeleton2, (getHeight() + curAdjustVal) - originalHeight); + float newError = getError(skeleton1, skeleton2, (getHeight() + curAdjustVal) - targetHeight); if (newError < error) { configs.put(entry.getKey(), newLength); @@ -287,7 +321,7 @@ public void processFrames() { } while (++frameCursor1 < frames.length && ++frameCursor2 < frames.length); } - LogManager.log.info("[AutoBone] Original height: " + originalHeight + " New height: " + getHeight()); + LogManager.log.info("[AutoBone] Target height: " + targetHeight + " New height: " + getHeight()); LogManager.log.info("[AutoBone] Done! Applying to skeleton..."); applyConfigToSkeleton(skeleton); From 629984c7926c436758e68f1b08fe0a2010e31bb3 Mon Sep 17 00:00:00 2001 From: ButterscotchVanilla Date: Thu, 12 Aug 2021 03:58:06 -0400 Subject: [PATCH 23/60] AutoBone: Remove feet from skeleton, read from AutoBone configs, and make skeletons local --- .../java/io/eiren/gui/autobone/AutoBone.java | 20 +++--- .../io/eiren/gui/autobone/SimpleSkeleton.java | 64 +++++++------------ 2 files changed, 34 insertions(+), 50 deletions(-) diff --git a/src/main/java/io/eiren/gui/autobone/AutoBone.java b/src/main/java/io/eiren/gui/autobone/AutoBone.java index 40f90a8e10..460dd246ad 100644 --- a/src/main/java/io/eiren/gui/autobone/AutoBone.java +++ b/src/main/java/io/eiren/gui/autobone/AutoBone.java @@ -1,6 +1,7 @@ package io.eiren.gui.autobone; import java.util.HashMap; +import java.util.Set; import java.util.Map.Entry; import io.eiren.util.ann.ThreadSafe; @@ -37,9 +38,6 @@ public class AutoBone { protected long frameRecordingInterval = 60L; protected long lastFrameTimeMs = -1L; - protected final SimpleSkeleton skeleton1; - protected final SimpleSkeleton skeleton2; - // This is filled by reloadConfigValues() public final HashMap configs = new HashMap(); @@ -48,9 +46,6 @@ public AutoBone(VRServer server) { reloadConfigValues(); - this.skeleton1 = new SimpleSkeleton(server); - this.skeleton2 = new SimpleSkeleton(server); - server.addSkeletonUpdatedCallback(this::skeletonUpdated); server.addOnTick(this::onTick); } @@ -166,6 +161,11 @@ public float getMaxHmdHeight(PoseFrame[] frames) { } public void processFrames() { + Set> configSet = configs.entrySet(); + + SimpleSkeleton skeleton1 = new SimpleSkeleton(configSet); + SimpleSkeleton skeleton2 = new SimpleSkeleton(configSet); + int epochs = NUM_EPOCHS; int epochCounter = -1; @@ -306,17 +306,17 @@ public void processFrames() { break; } - updateSekeletonBoneLength(entry.getKey(), newLength); + updateSekeletonBoneLength(skeleton1, skeleton2, entry.getKey(), newLength); float newError = getError(skeleton1, skeleton2, (getHeight() + curAdjustVal) - targetHeight); if (newError < error) { - configs.put(entry.getKey(), newLength); + entry.setValue(newLength); break; } } // Reset the length to minimize bias in other variables, it's applied later - updateSekeletonBoneLength(entry.getKey(), originalLength); + updateSekeletonBoneLength(skeleton1, skeleton2, entry.getKey(), originalLength); } } while (++frameCursor1 < frames.length && ++frameCursor2 < frames.length); } @@ -344,7 +344,7 @@ protected static float getError(SimpleSkeleton skeleton1, SimpleSkeleton skeleto return (SLIDE_ERROR_FACTOR * (slideError * slideError)) + (OFFSET_ERROR_FACTOR * (distError * distError)) + (HEIGHT_ERROR_FACTOR * (heightChange * heightChange)); } - protected void updateSekeletonBoneLength(String joint, float newLength) { + protected void updateSekeletonBoneLength(SimpleSkeleton skeleton1, SimpleSkeleton skeleton2, String joint, float newLength) { skeleton1.setSkeletonConfig(joint, newLength); skeleton2.setSkeletonConfig(joint, newLength); diff --git a/src/main/java/io/eiren/gui/autobone/SimpleSkeleton.java b/src/main/java/io/eiren/gui/autobone/SimpleSkeleton.java index 969ed95c10..db196eaa1a 100644 --- a/src/main/java/io/eiren/gui/autobone/SimpleSkeleton.java +++ b/src/main/java/io/eiren/gui/autobone/SimpleSkeleton.java @@ -10,11 +10,10 @@ import io.eiren.vr.processor.HumanSkeletonWithLegs; import io.eiren.vr.processor.HumanSkeletonWithWaist; import io.eiren.vr.processor.TransformNode; +import io.eiren.yaml.YamlFile; public class SimpleSkeleton { - protected final VRServer server; - // Waist protected final TransformNode hmdNode = new TransformNode("HMD", false); protected final TransformNode headNode = new TransformNode("Head", false); @@ -40,11 +39,9 @@ public class SimpleSkeleton { protected final TransformNode leftHipNode = new TransformNode("Left-Hip", false); protected final TransformNode leftKneeNode = new TransformNode("Left-Knee", false); protected final TransformNode leftAnkleNode = new TransformNode("Left-Ankle", false); - protected final TransformNode leftFootNode = new TransformNode("Left-Foot", false); protected final TransformNode rightHipNode = new TransformNode("Right-Hip", false); protected final TransformNode rightKneeNode = new TransformNode("Right-Knee", false); protected final TransformNode rightAnkleNode = new TransformNode("Right-Ankle", false); - protected final TransformNode rightFootNode = new TransformNode("Right-Foot", false); /** * Distance between centers of both hips @@ -58,25 +55,10 @@ public class SimpleSkeleton { * Distance from waist to ankle */ protected float legsLength = 0.84f; - protected float footLength = HumanSkeletonWithLegs.FOOT_LENGTH_DEFAULT; protected final HashMap nodes = new HashMap(); - public SimpleSkeleton(VRServer server) { - this.server = server; - - // Load waist configs - headShift = server.config.getFloat("body.headShift", headShift); - neckLength = server.config.getFloat("body.neckLength", neckLength); - chestDistance = server.config.getFloat("body.chestDistance", chestDistance); - waistDistance = server.config.getFloat("body.waistDistance", waistDistance); - - // Load leg configs - hipsWidth = server.config.getFloat("body.hipsWidth", hipsWidth); - kneeHeight = server.config.getFloat("body.kneeHeight", kneeHeight); - legsLength = server.config.getFloat("body.legsLength", legsLength); - footLength = server.config.getFloat("body.footLength", footLength); - + public SimpleSkeleton() { // Assemble skeleton to waist hmdNode.attachChild(headNode); headNode.localTransform.setTranslation(0, 0, headShift); @@ -109,18 +91,20 @@ public SimpleSkeleton(VRServer server) { rightKneeNode.attachChild(rightAnkleNode); rightAnkleNode.localTransform.setTranslation(0, -kneeHeight, 0); - leftAnkleNode.attachChild(leftFootNode); - leftFootNode.localTransform.setTranslation(0, 0, -footLength); - - rightAnkleNode.attachChild(rightFootNode); - rightFootNode.localTransform.setTranslation(0, 0, -footLength); - // Set up a HashMap to get nodes by name easily hmdNode.depthFirstTraversal(visitor -> { nodes.put(visitor.getName(), visitor); }); } + public SimpleSkeleton(Iterable> configs) { + // Initialize + this(); + + // Set configs + setSkeletonConfigs(configs); + } + public void setPoseFromSkeleton(HumanSkeletonWithLegs humanSkeleton) { TransformNode rootNode = humanSkeleton.getRootNode(); @@ -153,6 +137,12 @@ public void setPoseFromFrame(PoseFrame frame) { } } + public void setSkeletonConfigs(Iterable> configs) { + for (Entry config : configs) { + setSkeletonConfig(config.getKey(), config.getValue()); + } + } + public void setSkeletonConfig(String joint, float newLength) { switch(joint) { case "Head": @@ -189,11 +179,6 @@ public void setSkeletonConfig(String joint, float newLength) { leftKneeNode.localTransform.setTranslation(0, -(legsLength - kneeHeight), 0); rightKneeNode.localTransform.setTranslation(0, -(legsLength - kneeHeight), 0); break; - case "Foot length": - footLength = newLength; - leftFootNode.localTransform.setTranslation(0, 0, -footLength); - rightFootNode.localTransform.setTranslation(0, 0, -footLength); - break; } } @@ -213,17 +198,16 @@ public Vector3f getRightFootPos() { return rightAnkleNode.worldTransform.getTranslation(); } - public void saveConfigs() { + public void saveConfigs(YamlFile config) { // Save waist configs - server.config.setProperty("body.headShift", headShift); - server.config.setProperty("body.neckLength", neckLength); - server.config.setProperty("body.waistDistance", waistDistance); - server.config.setProperty("body.chestDistance", chestDistance); + config.setProperty("body.headShift", headShift); + config.setProperty("body.neckLength", neckLength); + config.setProperty("body.waistDistance", waistDistance); + config.setProperty("body.chestDistance", chestDistance); // Save leg configs - server.config.setProperty("body.hipsWidth", hipsWidth); - server.config.setProperty("body.kneeHeight", kneeHeight); - server.config.setProperty("body.legsLength", legsLength); - server.config.setProperty("body.footLength", footLength); + config.setProperty("body.hipsWidth", hipsWidth); + config.setProperty("body.kneeHeight", kneeHeight); + config.setProperty("body.legsLength", legsLength); } } From 0dab8f0c946cbbdf60b1079eae52703e2697e9a5 Mon Sep 17 00:00:00 2001 From: ButterscotchVanilla Date: Thu, 12 Aug 2021 05:52:57 -0400 Subject: [PATCH 24/60] AutoBone: Fix grammar to be clearer --- src/main/java/io/eiren/gui/autobone/AutoBone.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/io/eiren/gui/autobone/AutoBone.java b/src/main/java/io/eiren/gui/autobone/AutoBone.java index 460dd246ad..46f7324c83 100644 --- a/src/main/java/io/eiren/gui/autobone/AutoBone.java +++ b/src/main/java/io/eiren/gui/autobone/AutoBone.java @@ -178,7 +178,7 @@ public void processFrames() { float hmdHeight = getMaxHmdHeight(frames); if (hmdHeight <= 0.50f) { - LogManager.log.warning("[AutoBone] Max headset height detected (Value seems too low, did you stand straight while measuring?): " + hmdHeight); + LogManager.log.warning("[AutoBone] Max headset height detected (Value seems too low, did you not stand up straight while measuring?): " + hmdHeight); } else { LogManager.log.info("[AutoBone] Max headset height detected: " + hmdHeight); } From a52384de2e4f5d94426673c875d895376d8da503 Mon Sep 17 00:00:00 2001 From: ButterscotchVanilla Date: Thu, 12 Aug 2021 06:22:32 -0400 Subject: [PATCH 25/60] AutoBone: This decreases error magically? Fine-tune leg to body ratio range --- src/main/java/io/eiren/gui/autobone/AutoBone.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/io/eiren/gui/autobone/AutoBone.java b/src/main/java/io/eiren/gui/autobone/AutoBone.java index 46f7324c83..3eeada352d 100644 --- a/src/main/java/io/eiren/gui/autobone/AutoBone.java +++ b/src/main/java/io/eiren/gui/autobone/AutoBone.java @@ -300,7 +300,7 @@ public void processFrames() { case "Legs length": val = configs.get("Waist"); - if (val == null || newLength <= 0.667f * val || newLength >= 1.333f * val) { + if (val == null || newLength <= 0.5235f * val || newLength >= 1.7235f * val) { continue; } break; From 760dbfa5b9f304243be87faf1a0daceafe8918ee Mon Sep 17 00:00:00 2001 From: ButterscotchVanilla Date: Thu, 12 Aug 2021 19:30:36 -0400 Subject: [PATCH 26/60] AutoBone: Load multiple recordings, fine-tune values and extract ratios, fix restricted values from getting stuck --- .../java/io/eiren/gui/SkeletonConfig.java | 20 ++++ .../java/io/eiren/gui/autobone/AutoBone.java | 97 ++++++++++++++----- 2 files changed, 93 insertions(+), 24 deletions(-) diff --git a/src/main/java/io/eiren/gui/SkeletonConfig.java b/src/main/java/io/eiren/gui/SkeletonConfig.java index 4387dacd46..2d7db7c1a4 100644 --- a/src/main/java/io/eiren/gui/SkeletonConfig.java +++ b/src/main/java/io/eiren/gui/SkeletonConfig.java @@ -18,6 +18,7 @@ import io.eiren.gui.autobone.PoseRecordIO; import io.eiren.util.StringUtils; import io.eiren.util.ann.ThreadSafe; +import io.eiren.util.collections.FastList; import io.eiren.vr.VRServer; import io.eiren.vr.processor.HumanSkeletonWithLegs; import io.eiren.vr.processor.HumanSkeleton; @@ -125,6 +126,25 @@ public void run() { throw new NullPointerException("Reading frames from \"" + loadRecording.getPath() + "\" failed..."); } + FastList newFrames = new FastList(frames); + int recordNumber = 1; + for (;;) { + File loadRecordingI = new File("ABRecording_Load" + recordNumber++ + ".abf"); + + if (loadRecordingI.exists()) { + PoseFrame[] framesI = PoseRecordIO.readFromFile(loadRecordingI); + + if (framesI == null) { + throw new NullPointerException("Reading frames from \"" + loadRecordingI.getPath() + "\" failed..."); + } + + newFrames.addAll(framesI); + } else { + frames = newFrames.toArray(frames); + break; + } + } + autoBone.setFrames(frames); setText("Wait"); diff --git a/src/main/java/io/eiren/gui/autobone/AutoBone.java b/src/main/java/io/eiren/gui/autobone/AutoBone.java index 3eeada352d..7664d27361 100644 --- a/src/main/java/io/eiren/gui/autobone/AutoBone.java +++ b/src/main/java/io/eiren/gui/autobone/AutoBone.java @@ -15,7 +15,7 @@ public class AutoBone { protected final static int MIN_DATA_DISTANCE = 1; - protected final static int MAX_DATA_DISTANCE = 1000; + protected final static int MAX_DATA_DISTANCE = 200; protected final static int NUM_EPOCHS = 20; @@ -28,6 +28,18 @@ public class AutoBone { protected final static float HEADSET_HEIGHT_RATIO = 0.91f; + protected final static float CHEST_WAIST_RATIO_MIN = 0.2f; + protected final static float CHEST_WAIST_RATIO_MAX = 0.6f; + + protected final static float HIP_MIN = 0.08f; + protected final static float HIP_WAIST_RATIO_MAX = 0.4f; + + protected final static float LEG_WAIST_RATIO_MIN = 0.5235f; + protected final static float LEG_WAIST_RATIO_MAX = 1.7235f; + + protected final static float KNEE_LEG_RATIO_MIN = 0.42f; + protected final static float KNEE_LEG_RATIO_MAX = 0.58f; + protected final VRServer server; HumanSkeletonWithLegs skeleton = null; @@ -161,17 +173,26 @@ public float getMaxHmdHeight(PoseFrame[] frames) { } public void processFrames() { + processFrames(NUM_EPOCHS, true, INITIAL_ADJUSTMENT_RATE, ADJUSTMENT_RATE_DECAY, MIN_DATA_DISTANCE, MAX_DATA_DISTANCE); + } + + public void processFrames(int epochs, boolean calcInitError) { + processFrames(epochs, calcInitError, INITIAL_ADJUSTMENT_RATE, ADJUSTMENT_RATE_DECAY, MIN_DATA_DISTANCE, MAX_DATA_DISTANCE); + } + + public void processFrames(int epochs, boolean calcInitError, float adjustRate, float adjustRateDecay) { + processFrames(epochs, calcInitError, adjustRate, adjustRateDecay, MIN_DATA_DISTANCE, MAX_DATA_DISTANCE); + } + + public void processFrames(int epochs, boolean calcInitError, float adjustRate, float adjustRateDecay, int minDataDist, int maxDataDist) { Set> configSet = configs.entrySet(); SimpleSkeleton skeleton1 = new SimpleSkeleton(configSet); SimpleSkeleton skeleton2 = new SimpleSkeleton(configSet); - int epochs = NUM_EPOCHS; - int epochCounter = -1; + int epochCounter = calcInitError ? -1 : 0; - int cursorOffset = MIN_DATA_DISTANCE; - - float adjustRate = INITIAL_ADJUSTMENT_RATE; + int cursorOffset = minDataDist; float sumError = 0f; int errorCount = 0; @@ -188,7 +209,7 @@ public void processFrames() { for (;;) { // Detect end of iteration - if (cursorOffset >= frames.length || cursorOffset > MAX_DATA_DISTANCE) { + if (cursorOffset >= frames.length || cursorOffset > maxDataDist) { epochCounter++; // Calculate average error over the epoch @@ -204,8 +225,8 @@ public void processFrames() { break; } else { // Reset cursor offset and decay the adjustment rate - cursorOffset = MIN_DATA_DISTANCE; - adjustRate /= ADJUSTMENT_RATE_DECAY; + cursorOffset = minDataDist; + adjustRate /= adjustRateDecay; } } @@ -256,7 +277,7 @@ public void processFrames() { // LogManager.log.info("[AutoBone] Current position error: " + error); // } - for (Entry entry : configs.entrySet()) { + entryLoop: for (Entry entry : configs.entrySet()) { // Skip adjustment if the epoch is before starting (for logging only) if (epochCounter < 0) { break; @@ -265,7 +286,7 @@ public void processFrames() { float originalLength = entry.getValue(); // Try positive and negative adjustments - for (int i = 0; i < 2; i++) { + posNegAdj: for (int i = 0; i < 2; i++) { float curAdjustVal = i == 0 ? adjustVal : -adjustVal; float newLength = originalLength + curAdjustVal; @@ -274,34 +295,62 @@ public void processFrames() { continue; } - // Detect and skip invalid values + // Detect and fix invalid values, skipping adjustment Float val; switch (entry.getKey()) { case "Chest": val = configs.get("Waist"); - if (val == null || newLength <= 0.2f * val || newLength >= 0.6f * val) { - continue; + if (val == null || newLength <= CHEST_WAIST_RATIO_MIN * val || newLength >= CHEST_WAIST_RATIO_MAX * val) { + entry.setValue(Math.min(Math.max(newLength, CHEST_WAIST_RATIO_MIN * val), CHEST_WAIST_RATIO_MAX * val)); + + // If bone length hasn't been changed on skeleton, skip right to next entry + if (i > 0) { + break posNegAdj; + } else { + continue entryLoop; + } } break; case "Hips width": val = configs.get("Waist"); - if (val == null || newLength < 0.08f || newLength >= 0.4f * val) { - continue; + if (val == null || newLength < HIP_MIN || newLength >= HIP_WAIST_RATIO_MAX * val) { + entry.setValue(Math.min(Math.max(newLength, HIP_MIN), HIP_WAIST_RATIO_MAX * val)); + + // If bone length hasn't been changed on skeleton, skip right to next entry + if (i > 0) { + break posNegAdj; + } else { + continue entryLoop; + } } break; - case "Knee height": - val = configs.get("Legs length"); - if (val == null || newLength <= 0.375f * val || newLength >= 0.625f * val) { - continue; + case "Legs length": + val = configs.get("Waist"); + if (val == null || newLength <= LEG_WAIST_RATIO_MIN * val || newLength >= LEG_WAIST_RATIO_MAX * val) { + entry.setValue(Math.min(Math.max(newLength, LEG_WAIST_RATIO_MIN * val), LEG_WAIST_RATIO_MAX * val)); + + // If bone length hasn't been changed on skeleton, skip right to next entry + if (i > 0) { + break posNegAdj; + } else { + continue entryLoop; + } } break; - case "Legs length": - val = configs.get("Waist"); - if (val == null || newLength <= 0.5235f * val || newLength >= 1.7235f * val) { - continue; + case "Knee height": + val = configs.get("Legs length"); + if (val == null || newLength <= KNEE_LEG_RATIO_MIN * val || newLength >= KNEE_LEG_RATIO_MAX * val) { + entry.setValue(Math.min(Math.max(newLength, KNEE_LEG_RATIO_MIN * val), KNEE_LEG_RATIO_MAX * val)); + + // If bone length hasn't been changed on skeleton, skip right to next entry + if (i > 0) { + break posNegAdj; + } else { + continue entryLoop; + } } break; } From eee7d67591b897469a5ba0f76debf3e7ef77a505 Mon Sep 17 00:00:00 2001 From: ButterscotchVanilla Date: Thu, 12 Aug 2021 19:36:50 -0400 Subject: [PATCH 27/60] AutoBone: Allow manual target height value --- .../java/io/eiren/gui/autobone/AutoBone.java | 39 ++++++++++++++----- 1 file changed, 29 insertions(+), 10 deletions(-) diff --git a/src/main/java/io/eiren/gui/autobone/AutoBone.java b/src/main/java/io/eiren/gui/autobone/AutoBone.java index 7664d27361..528be16197 100644 --- a/src/main/java/io/eiren/gui/autobone/AutoBone.java +++ b/src/main/java/io/eiren/gui/autobone/AutoBone.java @@ -173,18 +173,34 @@ public float getMaxHmdHeight(PoseFrame[] frames) { } public void processFrames() { - processFrames(NUM_EPOCHS, true, INITIAL_ADJUSTMENT_RATE, ADJUSTMENT_RATE_DECAY, MIN_DATA_DISTANCE, MAX_DATA_DISTANCE); + processFrames(NUM_EPOCHS, true); + } + + public void processFrames(float targetHeight) { + processFrames(NUM_EPOCHS, true, targetHeight); } public void processFrames(int epochs, boolean calcInitError) { - processFrames(epochs, calcInitError, INITIAL_ADJUSTMENT_RATE, ADJUSTMENT_RATE_DECAY, MIN_DATA_DISTANCE, MAX_DATA_DISTANCE); + processFrames(epochs, calcInitError, INITIAL_ADJUSTMENT_RATE, ADJUSTMENT_RATE_DECAY); + } + + public void processFrames(int epochs, boolean calcInitError, float targetHeight) { + processFrames(epochs, calcInitError, INITIAL_ADJUSTMENT_RATE, ADJUSTMENT_RATE_DECAY, targetHeight); } public void processFrames(int epochs, boolean calcInitError, float adjustRate, float adjustRateDecay) { processFrames(epochs, calcInitError, adjustRate, adjustRateDecay, MIN_DATA_DISTANCE, MAX_DATA_DISTANCE); } + public void processFrames(int epochs, boolean calcInitError, float adjustRate, float adjustRateDecay, float targetHeight) { + processFrames(epochs, calcInitError, adjustRate, adjustRateDecay, MIN_DATA_DISTANCE, MAX_DATA_DISTANCE, targetHeight); + } + public void processFrames(int epochs, boolean calcInitError, float adjustRate, float adjustRateDecay, int minDataDist, int maxDataDist) { + processFrames(epochs, calcInitError, adjustRate, adjustRateDecay, minDataDist, maxDataDist, -1f); + } + + public void processFrames(int epochs, boolean calcInitError, float adjustRate, float adjustRateDecay, int minDataDist, int maxDataDist, float targetHeight) { Set> configSet = configs.entrySet(); SimpleSkeleton skeleton1 = new SimpleSkeleton(configSet); @@ -197,15 +213,18 @@ public void processFrames(int epochs, boolean calcInitError, float adjustRate, f float sumError = 0f; int errorCount = 0; - float hmdHeight = getMaxHmdHeight(frames); - if (hmdHeight <= 0.50f) { - LogManager.log.warning("[AutoBone] Max headset height detected (Value seems too low, did you not stand up straight while measuring?): " + hmdHeight); - } else { - LogManager.log.info("[AutoBone] Max headset height detected: " + hmdHeight); - } + // If target height isn't specified, auto-detect + if (targetHeight < 0f) { + float hmdHeight = getMaxHmdHeight(frames); + if (hmdHeight <= 0.50f) { + LogManager.log.warning("[AutoBone] Max headset height detected (Value seems too low, did you not stand up straight while measuring?): " + hmdHeight); + } else { + LogManager.log.info("[AutoBone] Max headset height detected: " + hmdHeight); + } - // Estimate target height from HMD height - float targetHeight = hmdHeight * HEADSET_HEIGHT_RATIO; + // Estimate target height from HMD height + targetHeight = hmdHeight * HEADSET_HEIGHT_RATIO; + } for (;;) { // Detect end of iteration From 0ba2450152423328ad36e6088ceb8affdd4ca536 Mon Sep 17 00:00:00 2001 From: ButterscotchVanilla Date: Thu, 12 Aug 2021 20:04:27 -0400 Subject: [PATCH 28/60] AutoBone: Fine-tune chest-waist and leg-waist ratios --- src/main/java/io/eiren/gui/autobone/AutoBone.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/main/java/io/eiren/gui/autobone/AutoBone.java b/src/main/java/io/eiren/gui/autobone/AutoBone.java index 528be16197..6f115a9872 100644 --- a/src/main/java/io/eiren/gui/autobone/AutoBone.java +++ b/src/main/java/io/eiren/gui/autobone/AutoBone.java @@ -28,14 +28,15 @@ public class AutoBone { protected final static float HEADSET_HEIGHT_RATIO = 0.91f; - protected final static float CHEST_WAIST_RATIO_MIN = 0.2f; - protected final static float CHEST_WAIST_RATIO_MAX = 0.6f; + protected final static float CHEST_WAIST_RATIO_MIN = 0.333f; + protected final static float CHEST_WAIST_RATIO_MAX = 0.667f; protected final static float HIP_MIN = 0.08f; protected final static float HIP_WAIST_RATIO_MAX = 0.4f; - protected final static float LEG_WAIST_RATIO_MIN = 0.5235f; - protected final static float LEG_WAIST_RATIO_MAX = 1.7235f; + // Human average is 1.1235 (SD 0.07) + protected final static float LEG_WAIST_RATIO_MIN = 1.1235f - 0.27f; + protected final static float LEG_WAIST_RATIO_MAX = 1.1235f + 0.27f; protected final static float KNEE_LEG_RATIO_MIN = 0.42f; protected final static float KNEE_LEG_RATIO_MAX = 0.58f; From cd7d4d102b4c17ebc46183208e993ce675fa79f9 Mon Sep 17 00:00:00 2001 From: ButterscotchVanilla Date: Thu, 12 Aug 2021 22:59:01 -0400 Subject: [PATCH 29/60] AutoBone: Add config input for recording and adjustment values --- src/main/java/io/eiren/gui/SkeletonConfig.java | 13 +++++++++++-- src/main/java/io/eiren/gui/autobone/AutoBone.java | 1 - 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/main/java/io/eiren/gui/SkeletonConfig.java b/src/main/java/io/eiren/gui/SkeletonConfig.java index 2d7db7c1a4..508b1590d5 100644 --- a/src/main/java/io/eiren/gui/SkeletonConfig.java +++ b/src/main/java/io/eiren/gui/SkeletonConfig.java @@ -151,7 +151,8 @@ public void run() { LogManager.log.info("[AutoBone] Done loading frames! Processing frames..."); } else { setText("Move"); - autoBone.startFrameRecording(250, 60); + // 400 samples at 50 ms per sample is 20 seconds + autoBone.startFrameRecording(server.config.getInt("autobone.sampleCount", 400), server.config.getInt("autobone.sampleRateMs", 50)); while (autoBone.isRecording()) { Thread.sleep(10); @@ -164,7 +165,15 @@ public void run() { LogManager.log.info("[AutoBone] Done exporting! Processing frames..."); } - autoBone.processFrames(); + int epochs = server.config.getInt("autobone.epochCount", 20); + boolean calcInitError = server.config.getBoolean("autobone.calculateInitialError", true); + float adjustRate = server.config.getFloat("autobone.adjustRate", 1f); + float adjustRateDecay = server.config.getFloat("autobone.adjustRateDecay", 1.1f); + int minDataDist = server.config.getInt("autobone.minimumDataDistance", 1); + int maxDataDist = server.config.getInt("autobone.maximumDataDistance", 200); + float targetHeight = server.config.getFloat("autobone.manualTargetHeight", -1f); + autoBone.processFrames(epochs, calcInitError, adjustRate, adjustRateDecay, minDataDist, maxDataDist, targetHeight); + LogManager.log.info("[AutoBone] Done processing!"); boolean first = true; diff --git a/src/main/java/io/eiren/gui/autobone/AutoBone.java b/src/main/java/io/eiren/gui/autobone/AutoBone.java index 6f115a9872..bd4b6a3f82 100644 --- a/src/main/java/io/eiren/gui/autobone/AutoBone.java +++ b/src/main/java/io/eiren/gui/autobone/AutoBone.java @@ -74,7 +74,6 @@ public void reloadConfigValues() { configs.put("Hips width", server.config.getFloat("body.hipsWidth", HumanSkeletonWithLegs.HIPS_WIDTH_DEFAULT)); configs.put("Knee height", server.config.getFloat("body.kneeHeight", 0.42f)); configs.put("Legs length", server.config.getFloat("body.legsLength", 0.84f)); - //configs.put("Foot length", server.config.getFloat("body.footLength", HumanSkeletonWithLegs.FOOT_LENGTH_DEFAULT)); // Feet aren't actually used } public void setSkeletonLengths(SimpleSkeleton skeleton) { From 1dc05ba1961f34958f7e0a5b42e3db2c9494f8f2 Mon Sep 17 00:00:00 2001 From: ButterscotchVanilla Date: Fri, 13 Aug 2021 15:12:15 -0400 Subject: [PATCH 30/60] AutoBone: Save configs without needing to have a skeleton --- .../java/io/eiren/gui/autobone/AutoBone.java | 49 ++++++++++++++++++- 1 file changed, 47 insertions(+), 2 deletions(-) diff --git a/src/main/java/io/eiren/gui/autobone/AutoBone.java b/src/main/java/io/eiren/gui/autobone/AutoBone.java index bd4b6a3f82..379abf9d2a 100644 --- a/src/main/java/io/eiren/gui/autobone/AutoBone.java +++ b/src/main/java/io/eiren/gui/autobone/AutoBone.java @@ -72,8 +72,8 @@ public void reloadConfigValues() { // Load leg configs configs.put("Hips width", server.config.getFloat("body.hipsWidth", HumanSkeletonWithLegs.HIPS_WIDTH_DEFAULT)); - configs.put("Knee height", server.config.getFloat("body.kneeHeight", 0.42f)); configs.put("Legs length", server.config.getFloat("body.legsLength", 0.84f)); + configs.put("Knee height", server.config.getFloat("body.kneeHeight", 0.42f)); } public void setSkeletonLengths(SimpleSkeleton skeleton) { @@ -102,10 +102,52 @@ public boolean applyConfigToSkeleton(HumanSkeleton skeleton) { skeleton.setSkeletonConfig(entry.getKey(), entry.getValue()); } + server.saveConfig(); + LogManager.log.info("[AutoBone] Configured skeleton bone lengths"); return true; } + // This doesn't require a skeleton, therefore can be used if skeleton is null + public void saveConfigs() { + Float headOffset = configs.get("Head"); + if (headOffset != null) { + server.config.setProperty("body.headShift", headOffset); + } + + Float neckLength = configs.get("Neck"); + if (neckLength != null) { + server.config.setProperty("body.neckLength", neckLength); + } + + Float waistLength = configs.get("Waist"); + if (waistLength != null) { + server.config.setProperty("body.waistDistance", waistLength); + } + + Float chestDistance = configs.get("Chest"); + if (chestDistance != null) { + server.config.setProperty("body.chestDistance", chestDistance); + } + + Float hipsWidth = configs.get("Hips width"); + if (hipsWidth != null) { + server.config.setProperty("body.hipsWidth", hipsWidth); + } + + Float legsLength = configs.get("Legs length"); + if (legsLength != null) { + server.config.setProperty("body.legsLength", legsLength); + } + + Float kneeHeight = configs.get("Knee height"); + if (kneeHeight != null) { + server.config.setProperty("body.kneeHeight", kneeHeight); + } + + server.saveConfig(); + } + @VRServerThread public void onTick() { if (frameRecordingCursor >= 0 && frameRecordingCursor < frames.length && skeleton != null && System.currentTimeMillis() - lastFrameTimeMs >= frameRecordingInterval) { @@ -392,7 +434,10 @@ public void processFrames(int epochs, boolean calcInitError, float adjustRate, f LogManager.log.info("[AutoBone] Target height: " + targetHeight + " New height: " + getHeight()); LogManager.log.info("[AutoBone] Done! Applying to skeleton..."); - applyConfigToSkeleton(skeleton); + if (!applyConfigToSkeleton(skeleton)) { + LogManager.log.info("[AutoBone] Applying to skeleton failed, only saving configs..."); + saveConfigs(); + } } protected static float getError(SimpleSkeleton skeleton1, SimpleSkeleton skeleton2, float heightChange) { From d77724a911ae546eafae64282dd3f51d87d3720e Mon Sep 17 00:00:00 2001 From: Butterscotch! Date: Fri, 13 Aug 2021 15:25:05 -0400 Subject: [PATCH 31/60] Change CI to build on any branch --- .github/workflows/gradle.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index 83f5da4368..2418c459bb 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -3,11 +3,7 @@ name: SlimeVR Server -on: - push: - branches: [ main ] - pull_request: - branches: [ main ] +on: [push, pull_request] jobs: test: From a2f54f67a3b338cce08dd8c9af74c6df48bc2d5e Mon Sep 17 00:00:00 2001 From: ButterscotchVanilla Date: Fri, 13 Aug 2021 15:33:16 -0400 Subject: [PATCH 32/60] AutoBone: Update GUI values after adjustment --- src/main/java/io/eiren/gui/SkeletonConfig.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/java/io/eiren/gui/SkeletonConfig.java b/src/main/java/io/eiren/gui/SkeletonConfig.java index 508b1590d5..1572e00171 100644 --- a/src/main/java/io/eiren/gui/SkeletonConfig.java +++ b/src/main/java/io/eiren/gui/SkeletonConfig.java @@ -192,6 +192,9 @@ public void run() { configInfo.append(']'); LogManager.log.info("[AutoBone] Length values: " + configInfo.toString()); + + // Update GUI values after adjustment + refreshAll(); } catch (Exception e1) { LogManager.log.severe("[AutoBone] Failed adjustment!", e1); } finally { From 7fd3297fededfd4cb614da0ac2d7f9b715a22b9e Mon Sep 17 00:00:00 2001 From: ButterscotchVanilla Date: Sat, 14 Aug 2021 00:35:24 -0400 Subject: [PATCH 33/60] AutoBone: Fix error function, add error derivative, consider positive and negative equally, etc --- .../java/io/eiren/gui/SkeletonConfig.java | 14 +- .../java/io/eiren/gui/autobone/AutoBone.java | 208 ++++++++++-------- .../io/eiren/gui/autobone/SimpleSkeleton.java | 1 - 3 files changed, 128 insertions(+), 95 deletions(-) diff --git a/src/main/java/io/eiren/gui/SkeletonConfig.java b/src/main/java/io/eiren/gui/SkeletonConfig.java index 1572e00171..fe16ec3b8d 100644 --- a/src/main/java/io/eiren/gui/SkeletonConfig.java +++ b/src/main/java/io/eiren/gui/SkeletonConfig.java @@ -151,8 +151,8 @@ public void run() { LogManager.log.info("[AutoBone] Done loading frames! Processing frames..."); } else { setText("Move"); - // 400 samples at 50 ms per sample is 20 seconds - autoBone.startFrameRecording(server.config.getInt("autobone.sampleCount", 400), server.config.getInt("autobone.sampleRateMs", 50)); + // 1000 samples at 20 ms per sample is 20 seconds + autoBone.startFrameRecording(server.config.getInt("autobone.sampleCount", 1000), server.config.getInt("autobone.sampleRateMs", 20)); while (autoBone.isRecording()) { Thread.sleep(10); @@ -165,12 +165,12 @@ public void run() { LogManager.log.info("[AutoBone] Done exporting! Processing frames..."); } - int epochs = server.config.getInt("autobone.epochCount", 20); + int epochs = server.config.getInt("autobone.epochCount", AutoBone.NUM_EPOCHS); boolean calcInitError = server.config.getBoolean("autobone.calculateInitialError", true); - float adjustRate = server.config.getFloat("autobone.adjustRate", 1f); - float adjustRateDecay = server.config.getFloat("autobone.adjustRateDecay", 1.1f); - int minDataDist = server.config.getInt("autobone.minimumDataDistance", 1); - int maxDataDist = server.config.getInt("autobone.maximumDataDistance", 200); + float adjustRate = server.config.getFloat("autobone.adjustRate", AutoBone.INITIAL_ADJUSTMENT_RATE); + float adjustRateDecay = server.config.getFloat("autobone.adjustRateDecay", AutoBone.ADJUSTMENT_RATE_DECAY); + int minDataDist = server.config.getInt("autobone.minimumDataDistance", AutoBone.MIN_DATA_DISTANCE); + int maxDataDist = server.config.getInt("autobone.maximumDataDistance", AutoBone.MAX_DATA_DISTANCE); float targetHeight = server.config.getFloat("autobone.manualTargetHeight", -1f); autoBone.processFrames(epochs, calcInitError, adjustRate, adjustRateDecay, minDataDist, maxDataDist, targetHeight); diff --git a/src/main/java/io/eiren/gui/autobone/AutoBone.java b/src/main/java/io/eiren/gui/autobone/AutoBone.java index 379abf9d2a..461a2b7f46 100644 --- a/src/main/java/io/eiren/gui/autobone/AutoBone.java +++ b/src/main/java/io/eiren/gui/autobone/AutoBone.java @@ -1,12 +1,15 @@ package io.eiren.gui.autobone; import java.util.HashMap; +import java.util.Map; import java.util.Set; import java.util.Map.Entry; import io.eiren.util.ann.ThreadSafe; import io.eiren.util.ann.VRServerThread; import io.eiren.util.logging.LogManager; +import io.eiren.util.StringUtils; +import io.eiren.util.collections.FastList; import io.eiren.vr.VRServer; import io.eiren.vr.processor.HumanSkeleton; import io.eiren.vr.processor.HumanSkeletonWithLegs; @@ -14,29 +17,32 @@ public class AutoBone { - protected final static int MIN_DATA_DISTANCE = 1; - protected final static int MAX_DATA_DISTANCE = 200; + public final static int MIN_DATA_DISTANCE = 1; + public final static int MAX_DATA_DISTANCE = 2; - protected final static int NUM_EPOCHS = 20; + public final static int NUM_EPOCHS = 50; - protected final static float INITIAL_ADJUSTMENT_RATE = 1f; - protected final static float ADJUSTMENT_RATE_DECAY = 1.1f; + public final static float INITIAL_ADJUSTMENT_RATE = 1f; + public final static float ADJUSTMENT_RATE_DECAY = 1.01f; - protected final static float SLIDE_ERROR_FACTOR = 1f; - protected final static float OFFSET_ERROR_FACTOR = 0.75f; - protected final static float HEIGHT_ERROR_FACTOR = 0.75f; + protected final static float SLIDE_ERROR_FACTOR = 1.0f; + protected final static float OFFSET_ERROR_FACTOR = 0.0f; + protected final static float HEIGHT_ERROR_FACTOR = 0.0f; - protected final static float HEADSET_HEIGHT_RATIO = 0.91f; + protected final static float HEADSET_HEIGHT_RATIO = 1.0f; - protected final static float CHEST_WAIST_RATIO_MIN = 0.333f; - protected final static float CHEST_WAIST_RATIO_MAX = 0.667f; + protected final static float NECK_WAIST_RATIO_MIN = 0.2f; + protected final static float NECK_WAIST_RATIO_MAX = 0.3f; + + protected final static float CHEST_WAIST_RATIO_MIN = 0.35f; + protected final static float CHEST_WAIST_RATIO_MAX = 0.6f; protected final static float HIP_MIN = 0.08f; protected final static float HIP_WAIST_RATIO_MAX = 0.4f; // Human average is 1.1235 (SD 0.07) - protected final static float LEG_WAIST_RATIO_MIN = 1.1235f - 0.27f; - protected final static float LEG_WAIST_RATIO_MAX = 1.1235f + 0.27f; + protected final static float LEG_WAIST_RATIO_MIN = 1.1235f - ((0.07f * 3f) + 0.05f); + protected final static float LEG_WAIST_RATIO_MAX = 1.1235f + ((0.07f * 3f) + 0.05f); protected final static float KNEE_LEG_RATIO_MIN = 0.42f; protected final static float KNEE_LEG_RATIO_MAX = 0.58f; @@ -49,11 +55,17 @@ public class AutoBone { protected int frameRecordingCursor = -1; protected long frameRecordingInterval = 60L; - protected long lastFrameTimeMs = -1L; + protected long nextFrameTimeMs = -1L; // This is filled by reloadConfigValues() public final HashMap configs = new HashMap(); + public final FastList heightConfigs = new FastList(new String[] { + "Neck", + "Waist", + "Legs length" + }); + public AutoBone(VRServer server) { this.server = server; @@ -150,8 +162,8 @@ public void saveConfigs() { @VRServerThread public void onTick() { - if (frameRecordingCursor >= 0 && frameRecordingCursor < frames.length && skeleton != null && System.currentTimeMillis() - lastFrameTimeMs >= frameRecordingInterval) { - lastFrameTimeMs = System.currentTimeMillis(); + if (frameRecordingCursor >= 0 && frameRecordingCursor < frames.length && skeleton != null && System.currentTimeMillis() >= nextFrameTimeMs) { + nextFrameTimeMs = System.currentTimeMillis() + frameRecordingInterval; PoseFrame frame = new PoseFrame(skeleton); frames[frameRecordingCursor++] = frame; @@ -164,7 +176,7 @@ public void startFrameRecording(int numFrames, long interval) { frames = new PoseFrame[numFrames]; frameRecordingInterval = interval; - lastFrameTimeMs = 0L; + nextFrameTimeMs = -1L; frameRecordingCursor = 0; @@ -188,17 +200,14 @@ public void setFrames(PoseFrame[] frames) { this.frames = frames; } - public float getHeight() { + public float getHeight(Map configs) { float height = 0f; - Float waistLength = configs.get("Waist"); - if (waistLength != null) { - height += waistLength; - } - - Float legsLength = configs.get("Legs length"); - if (legsLength != null) { - height += legsLength; + for (String heightConfig : heightConfigs) { + Float length = configs.get(heightConfig); + if (length != null) { + height += length; + } } return height; @@ -257,15 +266,20 @@ public void processFrames(int epochs, boolean calcInitError, float adjustRate, f // If target height isn't specified, auto-detect if (targetHeight < 0f) { - float hmdHeight = getMaxHmdHeight(frames); - if (hmdHeight <= 0.50f) { - LogManager.log.warning("[AutoBone] Max headset height detected (Value seems too low, did you not stand up straight while measuring?): " + hmdHeight); + if (skeleton != null) { + targetHeight = getHeight(skeleton.getSkeletonConfig()); + LogManager.log.warning("[AutoBone] Target height loaded from skeleton (Make sure you reset before running!): " + targetHeight); } else { - LogManager.log.info("[AutoBone] Max headset height detected: " + hmdHeight); - } + float hmdHeight = getMaxHmdHeight(frames); + if (hmdHeight <= 0.50f) { + LogManager.log.warning("[AutoBone] Max headset height detected (Value seems too low, did you not stand up straight while measuring?): " + hmdHeight); + } else { + LogManager.log.info("[AutoBone] Max headset height detected: " + hmdHeight); + } - // Estimate target height from HMD height - targetHeight = hmdHeight * HEADSET_HEIGHT_RATIO; + // Estimate target height from HMD height + targetHeight = hmdHeight * HEADSET_HEIGHT_RATIO; + } } for (;;) { @@ -312,7 +326,9 @@ public void processFrames(int epochs, boolean calcInitError, float adjustRate, f skeleton1.updatePose(); skeleton2.updatePose(); - float error = getError(skeleton1, skeleton2, getHeight() - targetHeight); + float curHeight = getHeight(configs); + float errorDeriv = getErrorDeriv(skeleton1, skeleton2, curHeight - targetHeight); + float error = errorFunc(errorDeriv); // In case of fire if (Float.isNaN(error) || Float.isInfinite(error)) { @@ -329,16 +345,12 @@ public void processFrames(int epochs, boolean calcInitError, float adjustRate, f } // Store the error count for logging purposes - sumError += error; + sumError += errorDeriv; errorCount++; float adjustVal = error * adjustRate; - // if (frameCursor1 == 0 && frameCursor2 == 1) { - // LogManager.log.info("[AutoBone] Current position error: " + error); - // } - - entryLoop: for (Entry entry : configs.entrySet()) { + for (Entry entry : configs.entrySet()) { // Skip adjustment if the epoch is before starting (for logging only) if (epochCounter < 0) { break; @@ -347,7 +359,9 @@ public void processFrames(int epochs, boolean calcInitError, float adjustRate, f float originalLength = entry.getValue(); // Try positive and negative adjustments - posNegAdj: for (int i = 0; i < 2; i++) { + float minError = error; + float finalNewLength = -1f; + for (int i = 0; i < 2; i++) { float curAdjustVal = i == 0 ? adjustVal : -adjustVal; float newLength = originalLength + curAdjustVal; @@ -356,73 +370,75 @@ public void processFrames(int epochs, boolean calcInitError, float adjustRate, f continue; } - // Detect and fix invalid values, skipping adjustment + updateSekeletonBoneLength(skeleton1, skeleton2, entry.getKey(), newLength); + + float newHeight = heightConfigs.contains(entry.getKey()) ? curHeight + curAdjustVal : curHeight; + float newError = errorFunc(getErrorDeriv(skeleton1, skeleton2, newHeight - targetHeight)); + + if (newError < minError) { + minError = newError; + finalNewLength = newLength; + } + } + + if (finalNewLength > 0f) { + // Keep values within a reasonable range + /* Float val; switch (entry.getKey()) { + case "Neck": + val = configs.get("Waist"); + if (val == null || finalNewLength <= NECK_WAIST_RATIO_MIN * val || finalNewLength >= NECK_WAIST_RATIO_MAX * val) { + entry.setValue(Math.min(Math.max(finalNewLength, NECK_WAIST_RATIO_MIN * val), NECK_WAIST_RATIO_MAX * val)); + } else { + entry.setValue(finalNewLength); + } + break; + case "Chest": val = configs.get("Waist"); - if (val == null || newLength <= CHEST_WAIST_RATIO_MIN * val || newLength >= CHEST_WAIST_RATIO_MAX * val) { - entry.setValue(Math.min(Math.max(newLength, CHEST_WAIST_RATIO_MIN * val), CHEST_WAIST_RATIO_MAX * val)); - - // If bone length hasn't been changed on skeleton, skip right to next entry - if (i > 0) { - break posNegAdj; - } else { - continue entryLoop; - } + if (val == null || finalNewLength <= CHEST_WAIST_RATIO_MIN * val || finalNewLength >= CHEST_WAIST_RATIO_MAX * val) { + entry.setValue(Math.min(Math.max(finalNewLength, CHEST_WAIST_RATIO_MIN * val), CHEST_WAIST_RATIO_MAX * val)); + } else { + entry.setValue(finalNewLength); } break; case "Hips width": val = configs.get("Waist"); - if (val == null || newLength < HIP_MIN || newLength >= HIP_WAIST_RATIO_MAX * val) { - entry.setValue(Math.min(Math.max(newLength, HIP_MIN), HIP_WAIST_RATIO_MAX * val)); - - // If bone length hasn't been changed on skeleton, skip right to next entry - if (i > 0) { - break posNegAdj; - } else { - continue entryLoop; - } + if (val == null || finalNewLength < HIP_MIN || finalNewLength >= HIP_WAIST_RATIO_MAX * val) { + entry.setValue(Math.min(Math.max(finalNewLength, HIP_MIN), HIP_WAIST_RATIO_MAX * val)); + } else { + entry.setValue(finalNewLength); } break; case "Legs length": val = configs.get("Waist"); - if (val == null || newLength <= LEG_WAIST_RATIO_MIN * val || newLength >= LEG_WAIST_RATIO_MAX * val) { - entry.setValue(Math.min(Math.max(newLength, LEG_WAIST_RATIO_MIN * val), LEG_WAIST_RATIO_MAX * val)); - - // If bone length hasn't been changed on skeleton, skip right to next entry - if (i > 0) { - break posNegAdj; - } else { - continue entryLoop; - } + if (val == null || finalNewLength <= LEG_WAIST_RATIO_MIN * val || finalNewLength >= LEG_WAIST_RATIO_MAX * val) { + entry.setValue(Math.min(Math.max(finalNewLength, LEG_WAIST_RATIO_MIN * val), LEG_WAIST_RATIO_MAX * val)); + } else { + entry.setValue(finalNewLength); } break; case "Knee height": val = configs.get("Legs length"); - if (val == null || newLength <= KNEE_LEG_RATIO_MIN * val || newLength >= KNEE_LEG_RATIO_MAX * val) { - entry.setValue(Math.min(Math.max(newLength, KNEE_LEG_RATIO_MIN * val), KNEE_LEG_RATIO_MAX * val)); - - // If bone length hasn't been changed on skeleton, skip right to next entry - if (i > 0) { - break posNegAdj; - } else { - continue entryLoop; - } + if (val == null || finalNewLength <= KNEE_LEG_RATIO_MIN * val || finalNewLength >= KNEE_LEG_RATIO_MAX * val) { + entry.setValue(Math.min(Math.max(finalNewLength, KNEE_LEG_RATIO_MIN * val), KNEE_LEG_RATIO_MAX * val)); + } else { + entry.setValue(finalNewLength); } break; - } - - updateSekeletonBoneLength(skeleton1, skeleton2, entry.getKey(), newLength); - float newError = getError(skeleton1, skeleton2, (getHeight() + curAdjustVal) - targetHeight); - if (newError < error) { - entry.setValue(newLength); + default: + entry.setValue(finalNewLength); break; } + */ + + // Temp while above is commented + entry.setValue(finalNewLength); } // Reset the length to minimize bias in other variables, it's applied later @@ -431,7 +447,17 @@ public void processFrames(int epochs, boolean calcInitError, float adjustRate, f } while (++frameCursor1 < frames.length && ++frameCursor2 < frames.length); } - LogManager.log.info("[AutoBone] Target height: " + targetHeight + " New height: " + getHeight()); + LogManager.log.info("[AutoBone] Target height: " + targetHeight + " New height: " + getHeight(configs)); + + try { + LogManager.log.info("[AutoBone] Ratios: [{Neck-Waist: " + StringUtils.prettyNumber(configs.get("Neck") / configs.get("Waist")) + + "}, {Chest-Waist: " + StringUtils.prettyNumber(configs.get("Chest") / configs.get("Waist")) + + "}, {Hip-Waist: " + StringUtils.prettyNumber(configs.get("Hips width") / configs.get("Waist")) + + "}, {Leg-Waist: " + StringUtils.prettyNumber(configs.get("Legs length") / configs.get("Waist")) + + "}, {Knee-Leg: " + StringUtils.prettyNumber(configs.get("Knee height") / configs.get("Legs length")) + "}]"); + } catch (Exception e) { + // I literally couldn't care less, this is only for debugging + } LogManager.log.info("[AutoBone] Done! Applying to skeleton..."); if (!applyConfigToSkeleton(skeleton)) { @@ -440,7 +466,7 @@ public void processFrames(int epochs, boolean calcInitError, float adjustRate, f } } - protected static float getError(SimpleSkeleton skeleton1, SimpleSkeleton skeleton2, float heightChange) { + protected static float getErrorDeriv(SimpleSkeleton skeleton1, SimpleSkeleton skeleton2, float heightChange) { float slideLeft = skeleton1.getLeftFootPos().distance(skeleton2.getLeftFootPos()); float slideRight = skeleton1.getRightFootPos().distance(skeleton2.getRightFootPos()); @@ -454,7 +480,15 @@ protected static float getError(SimpleSkeleton skeleton1, SimpleSkeleton skeleto float distError = (dist1 + dist2) / 2f; // Minimize sliding, minimize foot height offset, minimize change in total height - return (SLIDE_ERROR_FACTOR * (slideError * slideError)) + (OFFSET_ERROR_FACTOR * (distError * distError)) + (HEIGHT_ERROR_FACTOR * (heightChange * heightChange)); + return ((SLIDE_ERROR_FACTOR * Math.abs(slideError)) + + (OFFSET_ERROR_FACTOR * Math.abs(distError)) + + (HEIGHT_ERROR_FACTOR * Math.abs(heightChange))) / + (SLIDE_ERROR_FACTOR + OFFSET_ERROR_FACTOR + HEIGHT_ERROR_FACTOR); + } + + // Mean square error function + protected static float errorFunc(float errorDeriv) { + return 0.5f * (errorDeriv * errorDeriv); } protected void updateSekeletonBoneLength(SimpleSkeleton skeleton1, SimpleSkeleton skeleton2, String joint, float newLength) { diff --git a/src/main/java/io/eiren/gui/autobone/SimpleSkeleton.java b/src/main/java/io/eiren/gui/autobone/SimpleSkeleton.java index db196eaa1a..7029eeb552 100644 --- a/src/main/java/io/eiren/gui/autobone/SimpleSkeleton.java +++ b/src/main/java/io/eiren/gui/autobone/SimpleSkeleton.java @@ -6,7 +6,6 @@ import com.jme3.math.Quaternion; import com.jme3.math.Vector3f; -import io.eiren.vr.VRServer; import io.eiren.vr.processor.HumanSkeletonWithLegs; import io.eiren.vr.processor.HumanSkeletonWithWaist; import io.eiren.vr.processor.TransformNode; From e3b125f2443fd0f8a324aadeb10be500820032df Mon Sep 17 00:00:00 2001 From: ButterscotchVanilla Date: Sat, 14 Aug 2021 03:41:33 -0400 Subject: [PATCH 34/60] AutoBone: Add bulk recording loading, add height diff stat --- .../java/io/eiren/gui/SkeletonConfig.java | 103 ++++++++++-------- .../java/io/eiren/gui/autobone/AutoBone.java | 33 ++++-- 2 files changed, 82 insertions(+), 54 deletions(-) diff --git a/src/main/java/io/eiren/gui/SkeletonConfig.java b/src/main/java/io/eiren/gui/SkeletonConfig.java index fe16ec3b8d..4dce271914 100644 --- a/src/main/java/io/eiren/gui/SkeletonConfig.java +++ b/src/main/java/io/eiren/gui/SkeletonConfig.java @@ -115,38 +115,30 @@ public void mouseClicked(MouseEvent e) { public void run() { try { File saveRecording = new File("ABRecording.abf"); - File loadRecording = new File("ABRecording_Load.abf"); + File recordFolder = new File("LoadRecordings"); - if (loadRecording.exists()) { - setText("Load"); - LogManager.log.info("[AutoBone] Detected recording at \"" + loadRecording.getPath() + "\", loading frames..."); - PoseFrame[] frames = PoseRecordIO.readFromFile(loadRecording); - - if (frames == null) { - throw new NullPointerException("Reading frames from \"" + loadRecording.getPath() + "\" failed..."); - } + FastList frameRecordings = new FastList(); - FastList newFrames = new FastList(frames); - int recordNumber = 1; - for (;;) { - File loadRecordingI = new File("ABRecording_Load" + recordNumber++ + ".abf"); + if (recordFolder.isDirectory()) { + setText("Load"); - if (loadRecordingI.exists()) { - PoseFrame[] framesI = PoseRecordIO.readFromFile(loadRecordingI); + for (File file : recordFolder.listFiles()) { + if (file.isFile() && org.apache.commons.lang3.StringUtils.endsWithIgnoreCase(file.getName(), ".abf")) { + LogManager.log.info("[AutoBone] Detected recording at \"" + file.getPath() + "\", loading frames..."); + PoseFrame[] frames = PoseRecordIO.readFromFile(file); - if (framesI == null) { - throw new NullPointerException("Reading frames from \"" + loadRecordingI.getPath() + "\" failed..."); + if (frames == null) { + LogManager.log.severe("Reading frames from \"" + file.getPath() + "\" failed..."); + } else { + frameRecordings.add(frames); } - - newFrames.addAll(framesI); } else { - frames = newFrames.toArray(frames); break; } } + } - autoBone.setFrames(frames); - + if (frameRecordings.size() > 0) { setText("Wait"); LogManager.log.info("[AutoBone] Done loading frames! Processing frames..."); } else { @@ -161,37 +153,60 @@ public void run() { setText("Wait"); LogManager.log.info("[AutoBone] Done recording! Exporting frames to \"" + saveRecording.getPath() + "\"..."); PoseRecordIO.writeToFile(saveRecording, autoBone.getFrames()); + frameRecordings.add(autoBone.getFrames()); LogManager.log.info("[AutoBone] Done exporting! Processing frames..."); } - int epochs = server.config.getInt("autobone.epochCount", AutoBone.NUM_EPOCHS); - boolean calcInitError = server.config.getBoolean("autobone.calculateInitialError", true); - float adjustRate = server.config.getFloat("autobone.adjustRate", AutoBone.INITIAL_ADJUSTMENT_RATE); - float adjustRateDecay = server.config.getFloat("autobone.adjustRateDecay", AutoBone.ADJUSTMENT_RATE_DECAY); - int minDataDist = server.config.getInt("autobone.minimumDataDistance", AutoBone.MIN_DATA_DISTANCE); - int maxDataDist = server.config.getInt("autobone.maximumDataDistance", AutoBone.MAX_DATA_DISTANCE); - float targetHeight = server.config.getFloat("autobone.manualTargetHeight", -1f); - autoBone.processFrames(epochs, calcInitError, adjustRate, adjustRateDecay, minDataDist, maxDataDist, targetHeight); - - LogManager.log.info("[AutoBone] Done processing!"); - - boolean first = true; - StringBuilder configInfo = new StringBuilder("["); - - for (Entry entry : autoBone.configs.entrySet()) { - if (!first) { - configInfo.append(", "); - } else { - first = false; + FastList heightPercentError = new FastList(frameRecordings.size()); + for (PoseFrame[] recording : frameRecordings) { + autoBone.setFrames(recording); + + int epochs = server.config.getInt("autobone.epochCount", AutoBone.NUM_EPOCHS); + boolean calcInitError = server.config.getBoolean("autobone.calculateInitialError", true); + float adjustRate = server.config.getFloat("autobone.adjustRate", AutoBone.INITIAL_ADJUSTMENT_RATE); + float adjustRateDecay = server.config.getFloat("autobone.adjustRateDecay", AutoBone.ADJUSTMENT_RATE_DECAY); + int minDataDist = server.config.getInt("autobone.minimumDataDistance", AutoBone.MIN_DATA_DISTANCE); + int maxDataDist = server.config.getInt("autobone.maximumDataDistance", AutoBone.MAX_DATA_DISTANCE); + float targetHeight = server.config.getFloat("autobone.manualTargetHeight", -1f); + heightPercentError.add(autoBone.processFrames(epochs, calcInitError, adjustRate, adjustRateDecay, minDataDist, maxDataDist, targetHeight)); + + LogManager.log.info("[AutoBone] Done processing!"); + + boolean first = true; + StringBuilder configInfo = new StringBuilder("["); + + for (Entry entry : autoBone.configs.entrySet()) { + if (!first) { + configInfo.append(", "); + } else { + first = false; + } + + configInfo.append("{" + entry.getKey() + ": " + StringUtils.prettyNumber(entry.getValue()) + "}"); } - configInfo.append("{" + entry.getKey() + ": " + StringUtils.prettyNumber(entry.getValue()) + "}"); + configInfo.append(']'); + + LogManager.log.info("[AutoBone] Length values: " + configInfo.toString()); } - configInfo.append(']'); + if (heightPercentError.size() > 0) { + float mean = 0f; + for (float val : heightPercentError) { + mean += val; + } + mean /= heightPercentError.size(); + + float std = 0f; + for (float val : heightPercentError) { + float stdVal = val - mean; + std += stdVal * stdVal; + } + std = (float)Math.sqrt(std / heightPercentError.size()); - LogManager.log.info("[AutoBone] Length values: " + configInfo.toString()); + LogManager.log.info("[AutoBone] Average height error: " + StringUtils.prettyNumber(mean, 6) + " (SD " + StringUtils.prettyNumber(std, 6) + ")"); + } // Update GUI values after adjustment refreshAll(); diff --git a/src/main/java/io/eiren/gui/autobone/AutoBone.java b/src/main/java/io/eiren/gui/autobone/AutoBone.java index 461a2b7f46..5279369b26 100644 --- a/src/main/java/io/eiren/gui/autobone/AutoBone.java +++ b/src/main/java/io/eiren/gui/autobone/AutoBone.java @@ -18,12 +18,12 @@ public class AutoBone { public final static int MIN_DATA_DISTANCE = 1; - public final static int MAX_DATA_DISTANCE = 2; + public final static int MAX_DATA_DISTANCE = 1; public final static int NUM_EPOCHS = 50; public final static float INITIAL_ADJUSTMENT_RATE = 1f; - public final static float ADJUSTMENT_RATE_DECAY = 1.01f; + public final static float ADJUSTMENT_RATE_DECAY = 1.1f; protected final static float SLIDE_ERROR_FACTOR = 1.0f; protected final static float OFFSET_ERROR_FACTOR = 0.0f; @@ -251,7 +251,7 @@ public void processFrames(int epochs, boolean calcInitError, float adjustRate, f processFrames(epochs, calcInitError, adjustRate, adjustRateDecay, minDataDist, maxDataDist, -1f); } - public void processFrames(int epochs, boolean calcInitError, float adjustRate, float adjustRateDecay, int minDataDist, int maxDataDist, float targetHeight) { + public float processFrames(int epochs, boolean calcInitError, float adjustRate, float adjustRateDecay, int minDataDist, int maxDataDist, float targetHeight) { Set> configSet = configs.entrySet(); SimpleSkeleton skeleton1 = new SimpleSkeleton(configSet); @@ -383,12 +383,14 @@ public void processFrames(int epochs, boolean calcInitError, float adjustRate, f if (finalNewLength > 0f) { // Keep values within a reasonable range - /* Float val; switch (entry.getKey()) { + /* case "Neck": val = configs.get("Waist"); - if (val == null || finalNewLength <= NECK_WAIST_RATIO_MIN * val || finalNewLength >= NECK_WAIST_RATIO_MAX * val) { + if (val == null) { + break; + } else if (finalNewLength <= NECK_WAIST_RATIO_MIN * val || finalNewLength >= NECK_WAIST_RATIO_MAX * val) { entry.setValue(Math.min(Math.max(finalNewLength, NECK_WAIST_RATIO_MIN * val), NECK_WAIST_RATIO_MAX * val)); } else { entry.setValue(finalNewLength); @@ -397,7 +399,9 @@ public void processFrames(int epochs, boolean calcInitError, float adjustRate, f case "Chest": val = configs.get("Waist"); - if (val == null || finalNewLength <= CHEST_WAIST_RATIO_MIN * val || finalNewLength >= CHEST_WAIST_RATIO_MAX * val) { + if (val == null) { + break; + } else if (finalNewLength <= CHEST_WAIST_RATIO_MIN * val || finalNewLength >= CHEST_WAIST_RATIO_MAX * val) { entry.setValue(Math.min(Math.max(finalNewLength, CHEST_WAIST_RATIO_MIN * val), CHEST_WAIST_RATIO_MAX * val)); } else { entry.setValue(finalNewLength); @@ -406,7 +410,9 @@ public void processFrames(int epochs, boolean calcInitError, float adjustRate, f case "Hips width": val = configs.get("Waist"); - if (val == null || finalNewLength < HIP_MIN || finalNewLength >= HIP_WAIST_RATIO_MAX * val) { + if (val == null) { + break; + } else if (finalNewLength < HIP_MIN || finalNewLength >= HIP_WAIST_RATIO_MAX * val) { entry.setValue(Math.min(Math.max(finalNewLength, HIP_MIN), HIP_WAIST_RATIO_MAX * val)); } else { entry.setValue(finalNewLength); @@ -415,7 +421,9 @@ public void processFrames(int epochs, boolean calcInitError, float adjustRate, f case "Legs length": val = configs.get("Waist"); - if (val == null || finalNewLength <= LEG_WAIST_RATIO_MIN * val || finalNewLength >= LEG_WAIST_RATIO_MAX * val) { + if (val == null) { + break; + } else if (finalNewLength <= LEG_WAIST_RATIO_MIN * val || finalNewLength >= LEG_WAIST_RATIO_MAX * val) { entry.setValue(Math.min(Math.max(finalNewLength, LEG_WAIST_RATIO_MIN * val), LEG_WAIST_RATIO_MAX * val)); } else { entry.setValue(finalNewLength); @@ -424,7 +432,9 @@ public void processFrames(int epochs, boolean calcInitError, float adjustRate, f case "Knee height": val = configs.get("Legs length"); - if (val == null || finalNewLength <= KNEE_LEG_RATIO_MIN * val || finalNewLength >= KNEE_LEG_RATIO_MAX * val) { + if (val == null) { + break; + } else if (finalNewLength <= KNEE_LEG_RATIO_MIN * val || finalNewLength >= KNEE_LEG_RATIO_MAX * val) { entry.setValue(Math.min(Math.max(finalNewLength, KNEE_LEG_RATIO_MIN * val), KNEE_LEG_RATIO_MAX * val)); } else { entry.setValue(finalNewLength); @@ -447,7 +457,8 @@ public void processFrames(int epochs, boolean calcInitError, float adjustRate, f } while (++frameCursor1 < frames.length && ++frameCursor2 < frames.length); } - LogManager.log.info("[AutoBone] Target height: " + targetHeight + " New height: " + getHeight(configs)); + float finalHeight = getHeight(configs); + LogManager.log.info("[AutoBone] Target height: " + targetHeight + " New height: " + finalHeight); try { LogManager.log.info("[AutoBone] Ratios: [{Neck-Waist: " + StringUtils.prettyNumber(configs.get("Neck") / configs.get("Waist")) + @@ -464,6 +475,8 @@ public void processFrames(int epochs, boolean calcInitError, float adjustRate, f LogManager.log.info("[AutoBone] Applying to skeleton failed, only saving configs..."); saveConfigs(); } + + return Math.abs(finalHeight - targetHeight); } protected static float getErrorDeriv(SimpleSkeleton skeleton1, SimpleSkeleton skeleton2, float heightChange) { From 294141e22376c4a2ff215d2178b7827df4481987 Mon Sep 17 00:00:00 2001 From: ButterscotchVanilla Date: Sat, 14 Aug 2021 03:55:06 -0400 Subject: [PATCH 35/60] AutoBone: Oops --- src/main/java/io/eiren/gui/autobone/AutoBone.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/io/eiren/gui/autobone/AutoBone.java b/src/main/java/io/eiren/gui/autobone/AutoBone.java index 5279369b26..246a39ff8d 100644 --- a/src/main/java/io/eiren/gui/autobone/AutoBone.java +++ b/src/main/java/io/eiren/gui/autobone/AutoBone.java @@ -383,9 +383,9 @@ public float processFrames(int epochs, boolean calcInitError, float adjustRate, if (finalNewLength > 0f) { // Keep values within a reasonable range + /* Float val; switch (entry.getKey()) { - /* case "Neck": val = configs.get("Waist"); if (val == null) { From aaee64ce02c7f0ba163940957c9169b78d765fa5 Mon Sep 17 00:00:00 2001 From: ButterscotchVanilla Date: Sat, 14 Aug 2021 05:49:16 -0400 Subject: [PATCH 36/60] AutoBone: Add stabilization, more fine-tuning as usual --- .../java/io/eiren/gui/autobone/AutoBone.java | 22 ++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/src/main/java/io/eiren/gui/autobone/AutoBone.java b/src/main/java/io/eiren/gui/autobone/AutoBone.java index 246a39ff8d..2b3aa47b0b 100644 --- a/src/main/java/io/eiren/gui/autobone/AutoBone.java +++ b/src/main/java/io/eiren/gui/autobone/AutoBone.java @@ -18,12 +18,12 @@ public class AutoBone { public final static int MIN_DATA_DISTANCE = 1; - public final static int MAX_DATA_DISTANCE = 1; + public final static int MAX_DATA_DISTANCE = 2; - public final static int NUM_EPOCHS = 50; + public final static int NUM_EPOCHS = 75; - public final static float INITIAL_ADJUSTMENT_RATE = 1f; - public final static float ADJUSTMENT_RATE_DECAY = 1.1f; + public final static float INITIAL_ADJUSTMENT_RATE = 2.5f; + public final static float ADJUSTMENT_RATE_DECAY = 1.045f; protected final static float SLIDE_ERROR_FACTOR = 1.0f; protected final static float OFFSET_ERROR_FACTOR = 0.0f; @@ -213,6 +213,16 @@ public float getHeight(Map configs) { return height; } + public float getLengthSum(Map configs) { + float length = 0f; + + for (float boneLength : configs.values()) { + length += boneLength; + } + + return length; + } + public float getMaxHmdHeight(PoseFrame[] frames) { float maxHeight = 0f; for (PoseFrame frame : frames) { @@ -326,6 +336,7 @@ public float processFrames(int epochs, boolean calcInitError, float adjustRate, skeleton1.updatePose(); skeleton2.updatePose(); + float totalLength = getLengthSum(configs); float curHeight = getHeight(configs); float errorDeriv = getErrorDeriv(skeleton1, skeleton2, curHeight - targetHeight); float error = errorFunc(errorDeriv); @@ -362,7 +373,8 @@ public float processFrames(int epochs, boolean calcInitError, float adjustRate, float minError = error; float finalNewLength = -1f; for (int i = 0; i < 2; i++) { - float curAdjustVal = i == 0 ? adjustVal : -adjustVal; + // Scale by the ratio for smooth adjustment and more stable results + float curAdjustVal = (i == 0 ? adjustVal : -adjustVal) * (originalLength / totalLength); float newLength = originalLength + curAdjustVal; // No small or negative numbers!!! Bad algorithm! From 807ccc69cea365fd256c2f0bfb9fb08fa2534c09 Mon Sep 17 00:00:00 2001 From: ButterscotchVanilla Date: Sat, 14 Aug 2021 06:11:47 -0400 Subject: [PATCH 37/60] AutoBone: Print file name before processing frames --- src/main/java/io/eiren/gui/SkeletonConfig.java | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/main/java/io/eiren/gui/SkeletonConfig.java b/src/main/java/io/eiren/gui/SkeletonConfig.java index 4dce271914..6cc48c63f7 100644 --- a/src/main/java/io/eiren/gui/SkeletonConfig.java +++ b/src/main/java/io/eiren/gui/SkeletonConfig.java @@ -13,6 +13,8 @@ import javax.swing.JLabel; import javax.swing.event.MouseInputAdapter; +import org.apache.commons.lang3.tuple.Pair; + import io.eiren.gui.autobone.AutoBone; import io.eiren.gui.autobone.PoseFrame; import io.eiren.gui.autobone.PoseRecordIO; @@ -117,7 +119,7 @@ public void run() { File saveRecording = new File("ABRecording.abf"); File recordFolder = new File("LoadRecordings"); - FastList frameRecordings = new FastList(); + FastList> frameRecordings = new FastList>(); if (recordFolder.isDirectory()) { setText("Load"); @@ -130,7 +132,7 @@ public void run() { if (frames == null) { LogManager.log.severe("Reading frames from \"" + file.getPath() + "\" failed..."); } else { - frameRecordings.add(frames); + frameRecordings.add(Pair.of(file.getName(), frames)); } } else { break; @@ -153,14 +155,15 @@ public void run() { setText("Wait"); LogManager.log.info("[AutoBone] Done recording! Exporting frames to \"" + saveRecording.getPath() + "\"..."); PoseRecordIO.writeToFile(saveRecording, autoBone.getFrames()); - frameRecordings.add(autoBone.getFrames()); + frameRecordings.add(Pair.of("", autoBone.getFrames())); LogManager.log.info("[AutoBone] Done exporting! Processing frames..."); } FastList heightPercentError = new FastList(frameRecordings.size()); - for (PoseFrame[] recording : frameRecordings) { - autoBone.setFrames(recording); + for (Pair recording : frameRecordings) { + LogManager.log.info("[AutoBone] Processing frames from \"" + recording.getKey() + "\"..."); + autoBone.setFrames(recording.getValue()); int epochs = server.config.getInt("autobone.epochCount", AutoBone.NUM_EPOCHS); boolean calcInitError = server.config.getBoolean("autobone.calculateInitialError", true); From 4775dcd57a0b47189e724deeea69874ea7173559 Mon Sep 17 00:00:00 2001 From: ButterscotchVanilla Date: Sat, 14 Aug 2021 08:11:24 -0400 Subject: [PATCH 38/60] AutoBone: Add more configs, fix recording reading --- .../java/io/eiren/gui/SkeletonConfig.java | 18 ++-- .../java/io/eiren/gui/autobone/AutoBone.java | 95 ++++++++----------- 2 files changed, 49 insertions(+), 64 deletions(-) diff --git a/src/main/java/io/eiren/gui/SkeletonConfig.java b/src/main/java/io/eiren/gui/SkeletonConfig.java index 6cc48c63f7..16f8cf21af 100644 --- a/src/main/java/io/eiren/gui/SkeletonConfig.java +++ b/src/main/java/io/eiren/gui/SkeletonConfig.java @@ -134,8 +134,6 @@ public void run() { } else { frameRecordings.add(Pair.of(file.getName(), frames)); } - } else { - break; } } } @@ -165,14 +163,18 @@ public void run() { LogManager.log.info("[AutoBone] Processing frames from \"" + recording.getKey() + "\"..."); autoBone.setFrames(recording.getValue()); - int epochs = server.config.getInt("autobone.epochCount", AutoBone.NUM_EPOCHS); + autoBone.minDataDistance = server.config.getInt("autobone.minimumDataDistance", autoBone.minDataDistance); + autoBone.maxDataDistance = server.config.getInt("autobone.maximumDataDistance", autoBone.maxDataDistance); + autoBone.numEpochs = server.config.getInt("autobone.epochCount", autoBone.numEpochs); + autoBone.initialAdjustRate = server.config.getFloat("autobone.adjustRate", autoBone.initialAdjustRate); + autoBone.adjustRateDecay = server.config.getFloat("autobone.adjustRateDecay", autoBone.adjustRateDecay); + autoBone.slideErrorFactor = server.config.getFloat("autobone.slideErrorFactor", autoBone.slideErrorFactor); + autoBone.offsetErrorFactor = server.config.getFloat("autobone.offsetErrorFactor", autoBone.offsetErrorFactor); + autoBone.heightErrorFactor = server.config.getFloat("autobone.heightErrorFactor", autoBone.heightErrorFactor); + boolean calcInitError = server.config.getBoolean("autobone.calculateInitialError", true); - float adjustRate = server.config.getFloat("autobone.adjustRate", AutoBone.INITIAL_ADJUSTMENT_RATE); - float adjustRateDecay = server.config.getFloat("autobone.adjustRateDecay", AutoBone.ADJUSTMENT_RATE_DECAY); - int minDataDist = server.config.getInt("autobone.minimumDataDistance", AutoBone.MIN_DATA_DISTANCE); - int maxDataDist = server.config.getInt("autobone.maximumDataDistance", AutoBone.MAX_DATA_DISTANCE); float targetHeight = server.config.getFloat("autobone.manualTargetHeight", -1f); - heightPercentError.add(autoBone.processFrames(epochs, calcInitError, adjustRate, adjustRateDecay, minDataDist, maxDataDist, targetHeight)); + heightPercentError.add(autoBone.processFrames(calcInitError, targetHeight)); LogManager.log.info("[AutoBone] Done processing!"); diff --git a/src/main/java/io/eiren/gui/autobone/AutoBone.java b/src/main/java/io/eiren/gui/autobone/AutoBone.java index 2b3aa47b0b..0ddf0f3afa 100644 --- a/src/main/java/io/eiren/gui/autobone/AutoBone.java +++ b/src/main/java/io/eiren/gui/autobone/AutoBone.java @@ -17,35 +17,35 @@ public class AutoBone { - public final static int MIN_DATA_DISTANCE = 1; - public final static int MAX_DATA_DISTANCE = 2; + public int minDataDistance = 1; + public int maxDataDistance = 2; - public final static int NUM_EPOCHS = 75; + public int numEpochs = 100; - public final static float INITIAL_ADJUSTMENT_RATE = 2.5f; - public final static float ADJUSTMENT_RATE_DECAY = 1.045f; + public float initialAdjustRate = 2.5f; + public float adjustRateDecay = 1.03f; - protected final static float SLIDE_ERROR_FACTOR = 1.0f; - protected final static float OFFSET_ERROR_FACTOR = 0.0f; - protected final static float HEIGHT_ERROR_FACTOR = 0.0f; + public float slideErrorFactor = 1.0f; + public float offsetErrorFactor = 0.0f; + public float heightErrorFactor = 0.004f; - protected final static float HEADSET_HEIGHT_RATIO = 1.0f; + /* + public float NECK_WAIST_RATIO_MIN = 0.2f; + public float NECK_WAIST_RATIO_MAX = 0.3f; - protected final static float NECK_WAIST_RATIO_MIN = 0.2f; - protected final static float NECK_WAIST_RATIO_MAX = 0.3f; + public float CHEST_WAIST_RATIO_MIN = 0.35f; + public float CHEST_WAIST_RATIO_MAX = 0.6f; - protected final static float CHEST_WAIST_RATIO_MIN = 0.35f; - protected final static float CHEST_WAIST_RATIO_MAX = 0.6f; - - protected final static float HIP_MIN = 0.08f; - protected final static float HIP_WAIST_RATIO_MAX = 0.4f; + public float HIP_MIN = 0.08f; + public float HIP_WAIST_RATIO_MAX = 0.4f; // Human average is 1.1235 (SD 0.07) - protected final static float LEG_WAIST_RATIO_MIN = 1.1235f - ((0.07f * 3f) + 0.05f); - protected final static float LEG_WAIST_RATIO_MAX = 1.1235f + ((0.07f * 3f) + 0.05f); + public float LEG_WAIST_RATIO_MIN = 1.1235f - ((0.07f * 3f) + 0.05f); + public float LEG_WAIST_RATIO_MAX = 1.1235f + ((0.07f * 3f) + 0.05f); - protected final static float KNEE_LEG_RATIO_MIN = 0.42f; - protected final static float KNEE_LEG_RATIO_MAX = 0.58f; + public float KNEE_LEG_RATIO_MIN = 0.42f; + public float KNEE_LEG_RATIO_MAX = 0.58f; + */ protected final VRServer server; @@ -234,34 +234,14 @@ public float getMaxHmdHeight(PoseFrame[] frames) { } public void processFrames() { - processFrames(NUM_EPOCHS, true); + processFrames(-1f); } public void processFrames(float targetHeight) { - processFrames(NUM_EPOCHS, true, targetHeight); - } - - public void processFrames(int epochs, boolean calcInitError) { - processFrames(epochs, calcInitError, INITIAL_ADJUSTMENT_RATE, ADJUSTMENT_RATE_DECAY); - } - - public void processFrames(int epochs, boolean calcInitError, float targetHeight) { - processFrames(epochs, calcInitError, INITIAL_ADJUSTMENT_RATE, ADJUSTMENT_RATE_DECAY, targetHeight); - } - - public void processFrames(int epochs, boolean calcInitError, float adjustRate, float adjustRateDecay) { - processFrames(epochs, calcInitError, adjustRate, adjustRateDecay, MIN_DATA_DISTANCE, MAX_DATA_DISTANCE); - } - - public void processFrames(int epochs, boolean calcInitError, float adjustRate, float adjustRateDecay, float targetHeight) { - processFrames(epochs, calcInitError, adjustRate, adjustRateDecay, MIN_DATA_DISTANCE, MAX_DATA_DISTANCE, targetHeight); + processFrames(true, targetHeight); } - public void processFrames(int epochs, boolean calcInitError, float adjustRate, float adjustRateDecay, int minDataDist, int maxDataDist) { - processFrames(epochs, calcInitError, adjustRate, adjustRateDecay, minDataDist, maxDataDist, -1f); - } - - public float processFrames(int epochs, boolean calcInitError, float adjustRate, float adjustRateDecay, int minDataDist, int maxDataDist, float targetHeight) { + public float processFrames(boolean calcInitError, float targetHeight) { Set> configSet = configs.entrySet(); SimpleSkeleton skeleton1 = new SimpleSkeleton(configSet); @@ -269,7 +249,9 @@ public float processFrames(int epochs, boolean calcInitError, float adjustRate, int epochCounter = calcInitError ? -1 : 0; - int cursorOffset = minDataDist; + int cursorOffset = minDataDistance; + + float adjustRate = initialAdjustRate; float sumError = 0f; int errorCount = 0; @@ -288,13 +270,13 @@ public float processFrames(int epochs, boolean calcInitError, float adjustRate, } // Estimate target height from HMD height - targetHeight = hmdHeight * HEADSET_HEIGHT_RATIO; + targetHeight = hmdHeight; } } for (;;) { // Detect end of iteration - if (cursorOffset >= frames.length || cursorOffset > maxDataDist) { + if (cursorOffset >= frames.length || cursorOffset > maxDataDistance) { epochCounter++; // Calculate average error over the epoch @@ -306,11 +288,11 @@ public float processFrames(int epochs, boolean calcInitError, float adjustRate, LogManager.log.info("[AutoBone] Epoch " + epochCounter + " average error: " + avgError); - if (epochCounter >= epochs) { + if (epochCounter >= numEpochs) { break; } else { // Reset cursor offset and decay the adjustment rate - cursorOffset = minDataDist; + cursorOffset = minDataDistance; adjustRate /= adjustRateDecay; } } @@ -338,7 +320,7 @@ public float processFrames(int epochs, boolean calcInitError, float adjustRate, float totalLength = getLengthSum(configs); float curHeight = getHeight(configs); - float errorDeriv = getErrorDeriv(skeleton1, skeleton2, curHeight - targetHeight); + float errorDeriv = getErrorDeriv(skeleton1, skeleton2, targetHeight - curHeight); float error = errorFunc(errorDeriv); // In case of fire @@ -370,6 +352,7 @@ public float processFrames(int epochs, boolean calcInitError, float adjustRate, float originalLength = entry.getValue(); // Try positive and negative adjustments + boolean isHeightVar = heightConfigs.contains(entry.getKey()); float minError = error; float finalNewLength = -1f; for (int i = 0; i < 2; i++) { @@ -384,8 +367,8 @@ public float processFrames(int epochs, boolean calcInitError, float adjustRate, updateSekeletonBoneLength(skeleton1, skeleton2, entry.getKey(), newLength); - float newHeight = heightConfigs.contains(entry.getKey()) ? curHeight + curAdjustVal : curHeight; - float newError = errorFunc(getErrorDeriv(skeleton1, skeleton2, newHeight - targetHeight)); + float newHeight = isHeightVar ? curHeight + curAdjustVal : curHeight; + float newError = errorFunc(getErrorDeriv(skeleton1, skeleton2, targetHeight - newHeight)); if (newError < minError) { minError = newError; @@ -491,7 +474,7 @@ public float processFrames(int epochs, boolean calcInitError, float adjustRate, return Math.abs(finalHeight - targetHeight); } - protected static float getErrorDeriv(SimpleSkeleton skeleton1, SimpleSkeleton skeleton2, float heightChange) { + protected float getErrorDeriv(SimpleSkeleton skeleton1, SimpleSkeleton skeleton2, float heightChange) { float slideLeft = skeleton1.getLeftFootPos().distance(skeleton2.getLeftFootPos()); float slideRight = skeleton1.getRightFootPos().distance(skeleton2.getRightFootPos()); @@ -505,10 +488,10 @@ protected static float getErrorDeriv(SimpleSkeleton skeleton1, SimpleSkeleton sk float distError = (dist1 + dist2) / 2f; // Minimize sliding, minimize foot height offset, minimize change in total height - return ((SLIDE_ERROR_FACTOR * Math.abs(slideError)) + - (OFFSET_ERROR_FACTOR * Math.abs(distError)) + - (HEIGHT_ERROR_FACTOR * Math.abs(heightChange))) / - (SLIDE_ERROR_FACTOR + OFFSET_ERROR_FACTOR + HEIGHT_ERROR_FACTOR); + return ((slideErrorFactor * Math.abs(slideError)) + + (offsetErrorFactor * Math.abs(distError)) + + (heightErrorFactor * Math.abs(heightChange))) / + (slideErrorFactor + offsetErrorFactor + heightErrorFactor); } // Mean square error function From 380ae27762eadbcc1b6830c97f84f4695f0de9fc Mon Sep 17 00:00:00 2001 From: ButterscotchVanilla Date: Sat, 14 Aug 2021 08:30:44 -0400 Subject: [PATCH 39/60] AutoBone: Support no chest tracker --- src/main/java/io/eiren/gui/autobone/AutoBone.java | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/main/java/io/eiren/gui/autobone/AutoBone.java b/src/main/java/io/eiren/gui/autobone/AutoBone.java index 0ddf0f3afa..388b5b488d 100644 --- a/src/main/java/io/eiren/gui/autobone/AutoBone.java +++ b/src/main/java/io/eiren/gui/autobone/AutoBone.java @@ -14,6 +14,8 @@ import io.eiren.vr.processor.HumanSkeleton; import io.eiren.vr.processor.HumanSkeletonWithLegs; import io.eiren.vr.processor.HumanSkeletonWithWaist; +import io.eiren.vr.processor.TrackerBodyPosition; +import io.eiren.vr.trackers.TrackerUtils; public class AutoBone { @@ -80,7 +82,15 @@ public void reloadConfigValues() { configs.put("Head", server.config.getFloat("body.headShift", HumanSkeletonWithWaist.HEAD_SHIFT_DEFAULT)); configs.put("Neck", server.config.getFloat("body.neckLength", HumanSkeletonWithWaist.NECK_LENGTH_DEFAULT)); configs.put("Waist", server.config.getFloat("body.waistDistance", 0.85f)); - configs.put("Chest", server.config.getFloat("body.chestDistance", 0.42f)); + + if (server.config.getBoolean("autobone.forceChestTracker", false) || + TrackerUtils.findTrackerForBodyPosition(server.getAllTrackers(), TrackerBodyPosition.CHEST) != null) { + // If force enabled or has a chest tracker + configs.put("Chest", server.config.getFloat("body.chestDistance", 0.42f)); + } else { + // Otherwise make sure it's not used + configs.remove("Chest"); + } // Load leg configs configs.put("Hips width", server.config.getFloat("body.hipsWidth", HumanSkeletonWithLegs.HIPS_WIDTH_DEFAULT)); From e1d17f61c4d40262e7f0bbca334d5de86409f442 Mon Sep 17 00:00:00 2001 From: ButterscotchVanilla Date: Sat, 14 Aug 2021 08:42:35 -0400 Subject: [PATCH 40/60] AutoBone: Properly handle ratio output --- .../java/io/eiren/gui/SkeletonConfig.java | 21 +++++++++++++++++++ .../java/io/eiren/gui/autobone/AutoBone.java | 10 --------- 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/src/main/java/io/eiren/gui/SkeletonConfig.java b/src/main/java/io/eiren/gui/SkeletonConfig.java index 16f8cf21af..17c377009e 100644 --- a/src/main/java/io/eiren/gui/SkeletonConfig.java +++ b/src/main/java/io/eiren/gui/SkeletonConfig.java @@ -178,6 +178,26 @@ public void run() { LogManager.log.info("[AutoBone] Done processing!"); + //#region Stats/Values + Float neckLength = autoBone.configs.get("Neck"); + Float chestLength = autoBone.configs.get("Chest"); + Float waistLength = autoBone.configs.get("Waist"); + Float hipWidth = autoBone.configs.get("Hips width"); + Float legsLength = autoBone.configs.get("Legs length"); + Float kneeHeight = autoBone.configs.get("Knee height"); + + float neckWaist = neckLength != null && waistLength != null ? neckLength / waistLength : 0f; + float chestWaist = chestLength != null && waistLength != null ? chestLength / waistLength : 0f; + float hipWaist = hipWidth != null && waistLength != null ? hipWidth / waistLength : 0f; + float legWaist = legsLength != null && waistLength != null ? legsLength / waistLength : 0f; + float kneeLeg = kneeHeight != null && legsLength != null ? kneeHeight / legsLength : 0f; + + LogManager.log.info("[AutoBone] Ratios: [{Neck-Waist: " + StringUtils.prettyNumber(neckWaist) + + "}, {Chest-Waist: " + StringUtils.prettyNumber(chestWaist) + + "}, {Hip-Waist: " + StringUtils.prettyNumber(hipWaist) + + "}, {Leg-Waist: " + StringUtils.prettyNumber(legWaist) + + "}, {Knee-Leg: " + StringUtils.prettyNumber(kneeLeg) + "}]"); + boolean first = true; StringBuilder configInfo = new StringBuilder("["); @@ -212,6 +232,7 @@ public void run() { LogManager.log.info("[AutoBone] Average height error: " + StringUtils.prettyNumber(mean, 6) + " (SD " + StringUtils.prettyNumber(std, 6) + ")"); } + //#endregion // Update GUI values after adjustment refreshAll(); diff --git a/src/main/java/io/eiren/gui/autobone/AutoBone.java b/src/main/java/io/eiren/gui/autobone/AutoBone.java index 388b5b488d..326c9bce58 100644 --- a/src/main/java/io/eiren/gui/autobone/AutoBone.java +++ b/src/main/java/io/eiren/gui/autobone/AutoBone.java @@ -465,16 +465,6 @@ public float processFrames(boolean calcInitError, float targetHeight) { float finalHeight = getHeight(configs); LogManager.log.info("[AutoBone] Target height: " + targetHeight + " New height: " + finalHeight); - try { - LogManager.log.info("[AutoBone] Ratios: [{Neck-Waist: " + StringUtils.prettyNumber(configs.get("Neck") / configs.get("Waist")) + - "}, {Chest-Waist: " + StringUtils.prettyNumber(configs.get("Chest") / configs.get("Waist")) + - "}, {Hip-Waist: " + StringUtils.prettyNumber(configs.get("Hips width") / configs.get("Waist")) + - "}, {Leg-Waist: " + StringUtils.prettyNumber(configs.get("Legs length") / configs.get("Waist")) + - "}, {Knee-Leg: " + StringUtils.prettyNumber(configs.get("Knee height") / configs.get("Legs length")) + "}]"); - } catch (Exception e) { - // I literally couldn't care less, this is only for debugging - } - LogManager.log.info("[AutoBone] Done! Applying to skeleton..."); if (!applyConfigToSkeleton(skeleton)) { LogManager.log.info("[AutoBone] Applying to skeleton failed, only saving configs..."); From 855d15cec5da30ae089d329555b71768526982ac Mon Sep 17 00:00:00 2001 From: ButterscotchVanilla Date: Sun, 15 Aug 2021 21:55:48 -0400 Subject: [PATCH 41/60] AutoBone: Fix configs not updating when AutoBone is run --- src/main/java/io/eiren/gui/SkeletonConfig.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/io/eiren/gui/SkeletonConfig.java b/src/main/java/io/eiren/gui/SkeletonConfig.java index 17c377009e..3aed300b03 100644 --- a/src/main/java/io/eiren/gui/SkeletonConfig.java +++ b/src/main/java/io/eiren/gui/SkeletonConfig.java @@ -161,6 +161,7 @@ public void run() { FastList heightPercentError = new FastList(frameRecordings.size()); for (Pair recording : frameRecordings) { LogManager.log.info("[AutoBone] Processing frames from \"" + recording.getKey() + "\"..."); + autoBone.reloadConfigValues(); autoBone.setFrames(recording.getValue()); autoBone.minDataDistance = server.config.getInt("autobone.minimumDataDistance", autoBone.minDataDistance); From 4f8165c8e1cb99d63f9bc1d95d24944ac79fabc4 Mon Sep 17 00:00:00 2001 From: ButterscotchVanilla Date: Mon, 16 Aug 2021 01:45:59 -0400 Subject: [PATCH 42/60] Set gradle compiler encoding to UTF-8 --- build.gradle | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/build.gradle b/build.gradle index d13d5ece9f..3f8c75b1bd 100644 --- a/build.gradle +++ b/build.gradle @@ -14,6 +14,21 @@ plugins { sourceCompatibility = 1.8 targetCompatibility = 1.8 +// Set compiler to use UTF-8 +compileJava.options.encoding = 'UTF-8' +compileTestJava.options.encoding = 'UTF-8' +javadoc.options.encoding = 'UTF-8' + +tasks.withType(JavaCompile) { + options.encoding = 'UTF-8' +} +tasks.withType(Test) { + systemProperty('file.encoding', 'UTF-8') +} +tasks.withType(Javadoc){ + options.encoding = 'UTF-8' +} + repositories { // Use jcenter for resolving dependencies. // You can declare any Maven/Ivy/file repository here. From 4a2878b92e0d6d5d9ecbf539f5410ffe9d85a318 Mon Sep 17 00:00:00 2001 From: ButterscotchVanilla Date: Mon, 16 Aug 2021 01:56:06 -0400 Subject: [PATCH 43/60] AutoBone: Separate pose recorder from AutoBone & save multiple recordings --- .../java/io/eiren/gui/SkeletonConfig.java | 72 +++++++++------ .../java/io/eiren/gui/autobone/AutoBone.java | 79 +++------------- .../io/eiren/gui/autobone/PoseRecorder.java | 92 +++++++++++++++++++ 3 files changed, 152 insertions(+), 91 deletions(-) create mode 100644 src/main/java/io/eiren/gui/autobone/PoseRecorder.java diff --git a/src/main/java/io/eiren/gui/SkeletonConfig.java b/src/main/java/io/eiren/gui/SkeletonConfig.java index 3aed300b03..9d15617f31 100644 --- a/src/main/java/io/eiren/gui/SkeletonConfig.java +++ b/src/main/java/io/eiren/gui/SkeletonConfig.java @@ -7,12 +7,14 @@ import java.util.HashMap; import java.util.Map; import java.util.Map.Entry; +import java.util.concurrent.Future; import javax.swing.JButton; import javax.swing.JCheckBox; import javax.swing.JLabel; import javax.swing.event.MouseInputAdapter; +import io.eiren.gui.autobone.PoseRecorder; import org.apache.commons.lang3.tuple.Pair; import io.eiren.gui.autobone.AutoBone; @@ -30,6 +32,7 @@ public class SkeletonConfig extends EJBag { private final VRServer server; private final VRServerGUI gui; + private final PoseRecorder poseRecorder; private final AutoBone autoBone; private Thread autoBoneThread = null; private Map labels = new HashMap<>(); @@ -38,6 +41,7 @@ public SkeletonConfig(VRServer server, VRServerGUI gui) { super(); this.server = server; this.gui = gui; + this.poseRecorder = new PoseRecorder(server); this.autoBone = new AutoBone(server); setAlignmentY(TOP_ALIGNMENT); @@ -116,23 +120,24 @@ public void mouseClicked(MouseEvent e) { @Override public void run() { try { - File saveRecording = new File("ABRecording.abf"); - File recordFolder = new File("LoadRecordings"); - FastList> frameRecordings = new FastList>(); - if (recordFolder.isDirectory()) { + File loadFolder = new File("LoadRecordings"); + if (loadFolder.isDirectory()) { setText("Load"); - for (File file : recordFolder.listFiles()) { - if (file.isFile() && org.apache.commons.lang3.StringUtils.endsWithIgnoreCase(file.getName(), ".abf")) { - LogManager.log.info("[AutoBone] Detected recording at \"" + file.getPath() + "\", loading frames..."); - PoseFrame[] frames = PoseRecordIO.readFromFile(file); - - if (frames == null) { - LogManager.log.severe("Reading frames from \"" + file.getPath() + "\" failed..."); - } else { - frameRecordings.add(Pair.of(file.getName(), frames)); + File[] files = loadFolder.listFiles(); + if (files != null) { + for (File file : files) { + if (file.isFile() && org.apache.commons.lang3.StringUtils.endsWithIgnoreCase(file.getName(), ".abf")) { + LogManager.log.info("[AutoBone] Detected recording at \"" + file.getPath() + "\", loading frames..."); + PoseFrame[] frames = PoseRecordIO.readFromFile(file); + + if (frames == null) { + LogManager.log.severe("Reading frames from \"" + file.getPath() + "\" failed..."); + } else { + frameRecordings.add(Pair.of(file.getName(), frames)); + } } } } @@ -140,29 +145,44 @@ public void run() { if (frameRecordings.size() > 0) { setText("Wait"); - LogManager.log.info("[AutoBone] Done loading frames! Processing frames..."); + LogManager.log.info("[AutoBone] Done loading frames!"); } else { setText("Move"); // 1000 samples at 20 ms per sample is 20 seconds - autoBone.startFrameRecording(server.config.getInt("autobone.sampleCount", 1000), server.config.getInt("autobone.sampleRateMs", 20)); - - while (autoBone.isRecording()) { - Thread.sleep(10); - } + int sampleCount = server.config.getInt("autobone.sampleCount", 1000); + long sampleRate = server.config.getLong("autobone.sampleRateMs", 20L); + Future framesFuture = poseRecorder.startFrameRecording(sampleCount, sampleRate); + PoseFrame[] frames = framesFuture.get(); + LogManager.log.info("[AutoBone] Done recording!"); setText("Wait"); - LogManager.log.info("[AutoBone] Done recording! Exporting frames to \"" + saveRecording.getPath() + "\"..."); - PoseRecordIO.writeToFile(saveRecording, autoBone.getFrames()); - frameRecordings.add(Pair.of("", autoBone.getFrames())); - - LogManager.log.info("[AutoBone] Done exporting! Processing frames..."); + if (server.config.getBoolean("autobone.saveRecordings", true)) { + File saveFolder = new File("Recordings"); + if (saveFolder.isDirectory() || saveFolder.mkdirs()) { + File saveRecording; + int recordingIndex = 1; + do { + saveRecording = new File(saveFolder, "ABRecording" + recordingIndex++ + ".abf"); + } while (saveRecording.exists()); + + LogManager.log.info("[AutoBone] Exporting frames to \"" + saveRecording.getPath() + "\"..."); + if (PoseRecordIO.writeToFile(saveRecording, frames)) { + LogManager.log.info("[AutoBone] Done exporting! Recording can be found at \"" + saveRecording.getPath() + "\"."); + } else { + LogManager.log.severe("[AutoBone] Failed to export the recording to \"" + saveRecording.getPath() + "\"."); + } + } else { + LogManager.log.severe("[AutoBone] Failed to create the recording directory \"" + saveFolder.getPath() + "\"."); + } + } + frameRecordings.add(Pair.of("", frames)); } + LogManager.log.info("[AutoBone] Processing frames..."); FastList heightPercentError = new FastList(frameRecordings.size()); for (Pair recording : frameRecordings) { LogManager.log.info("[AutoBone] Processing frames from \"" + recording.getKey() + "\"..."); autoBone.reloadConfigValues(); - autoBone.setFrames(recording.getValue()); autoBone.minDataDistance = server.config.getInt("autobone.minimumDataDistance", autoBone.minDataDistance); autoBone.maxDataDistance = server.config.getInt("autobone.maximumDataDistance", autoBone.maxDataDistance); @@ -175,7 +195,7 @@ public void run() { boolean calcInitError = server.config.getBoolean("autobone.calculateInitialError", true); float targetHeight = server.config.getFloat("autobone.manualTargetHeight", -1f); - heightPercentError.add(autoBone.processFrames(calcInitError, targetHeight)); + heightPercentError.add(autoBone.processFrames(recording.getValue(), calcInitError, targetHeight)); LogManager.log.info("[AutoBone] Done processing!"); diff --git a/src/main/java/io/eiren/gui/autobone/AutoBone.java b/src/main/java/io/eiren/gui/autobone/AutoBone.java index 326c9bce58..c540a4d750 100644 --- a/src/main/java/io/eiren/gui/autobone/AutoBone.java +++ b/src/main/java/io/eiren/gui/autobone/AutoBone.java @@ -6,9 +6,7 @@ import java.util.Map.Entry; import io.eiren.util.ann.ThreadSafe; -import io.eiren.util.ann.VRServerThread; import io.eiren.util.logging.LogManager; -import io.eiren.util.StringUtils; import io.eiren.util.collections.FastList; import io.eiren.vr.VRServer; import io.eiren.vr.processor.HumanSkeleton; @@ -53,12 +51,6 @@ public class AutoBone { HumanSkeletonWithLegs skeleton = null; - protected PoseFrame[] frames = new PoseFrame[0]; - protected int frameRecordingCursor = -1; - - protected long frameRecordingInterval = 60L; - protected long nextFrameTimeMs = -1L; - // This is filled by reloadConfigValues() public final HashMap configs = new HashMap(); @@ -74,7 +66,6 @@ public AutoBone(VRServer server) { reloadConfigValues(); server.addSkeletonUpdatedCallback(this::skeletonUpdated); - server.addOnTick(this::onTick); } public void reloadConfigValues() { @@ -88,7 +79,7 @@ public void reloadConfigValues() { // If force enabled or has a chest tracker configs.put("Chest", server.config.getFloat("body.chestDistance", 0.42f)); } else { - // Otherwise make sure it's not used + // Otherwise, make sure it's not used configs.remove("Chest"); } @@ -106,13 +97,11 @@ public void setSkeletonLengths(SimpleSkeleton skeleton) { @ThreadSafe public void skeletonUpdated(HumanSkeleton newSkeleton) { - java.awt.EventQueue.invokeLater(() -> { - if (newSkeleton instanceof HumanSkeletonWithLegs) { - skeleton = (HumanSkeletonWithLegs)newSkeleton; - applyConfigToSkeleton(newSkeleton); - LogManager.log.info("[AutoBone] Received updated skeleton"); - } - }); + if (newSkeleton instanceof HumanSkeletonWithLegs) { + skeleton = (HumanSkeletonWithLegs)newSkeleton; + applyConfigToSkeleton(newSkeleton); + LogManager.log.info("[AutoBone] Received updated skeleton"); + } } public boolean applyConfigToSkeleton(HumanSkeleton skeleton) { @@ -170,46 +159,6 @@ public void saveConfigs() { server.saveConfig(); } - @VRServerThread - public void onTick() { - if (frameRecordingCursor >= 0 && frameRecordingCursor < frames.length && skeleton != null && System.currentTimeMillis() >= nextFrameTimeMs) { - nextFrameTimeMs = System.currentTimeMillis() + frameRecordingInterval; - - PoseFrame frame = new PoseFrame(skeleton); - frames[frameRecordingCursor++] = frame; - - LogManager.log.info("Recorded frame " + frameRecordingCursor); - } - } - - public void startFrameRecording(int numFrames, long interval) { - frames = new PoseFrame[numFrames]; - - frameRecordingInterval = interval; - nextFrameTimeMs = -1L; - - frameRecordingCursor = 0; - - LogManager.log.info("[AutoBone] Recording " + numFrames + " samples at a " + interval + " ms frame interval"); - } - - public void stopFrameRecording() { - // Set to end of the frame array to prevent race condition with `frameRecordingCursor++` - frameRecordingCursor = frames.length; - } - - public boolean isRecording() { - return frameRecordingCursor >= 0 && frameRecordingCursor < frames.length; - } - - public PoseFrame[] getFrames() { - return frames; - } - - public void setFrames(PoseFrame[] frames) { - this.frames = frames; - } - public float getHeight(Map configs) { float height = 0f; @@ -243,15 +192,15 @@ public float getMaxHmdHeight(PoseFrame[] frames) { return maxHeight; } - public void processFrames() { - processFrames(-1f); + public void processFrames(PoseFrame[] frames) { + processFrames(frames, -1f); } - public void processFrames(float targetHeight) { - processFrames(true, targetHeight); + public void processFrames(PoseFrame[] frames, float targetHeight) { + processFrames(frames, true, targetHeight); } - public float processFrames(boolean calcInitError, float targetHeight) { + public float processFrames(PoseFrame[] frames, boolean calcInitError, float targetHeight) { Set> configSet = configs.entrySet(); SimpleSkeleton skeleton1 = new SimpleSkeleton(configSet); @@ -375,7 +324,7 @@ public float processFrames(boolean calcInitError, float targetHeight) { continue; } - updateSekeletonBoneLength(skeleton1, skeleton2, entry.getKey(), newLength); + updateSkeletonBoneLength(skeleton1, skeleton2, entry.getKey(), newLength); float newHeight = isHeightVar ? curHeight + curAdjustVal : curHeight; float newError = errorFunc(getErrorDeriv(skeleton1, skeleton2, targetHeight - newHeight)); @@ -457,7 +406,7 @@ public float processFrames(boolean calcInitError, float targetHeight) { } // Reset the length to minimize bias in other variables, it's applied later - updateSekeletonBoneLength(skeleton1, skeleton2, entry.getKey(), originalLength); + updateSkeletonBoneLength(skeleton1, skeleton2, entry.getKey(), originalLength); } } while (++frameCursor1 < frames.length && ++frameCursor2 < frames.length); } @@ -499,7 +448,7 @@ protected static float errorFunc(float errorDeriv) { return 0.5f * (errorDeriv * errorDeriv); } - protected void updateSekeletonBoneLength(SimpleSkeleton skeleton1, SimpleSkeleton skeleton2, String joint, float newLength) { + protected void updateSkeletonBoneLength(SimpleSkeleton skeleton1, SimpleSkeleton skeleton2, String joint, float newLength) { skeleton1.setSkeletonConfig(joint, newLength); skeleton2.setSkeletonConfig(joint, newLength); diff --git a/src/main/java/io/eiren/gui/autobone/PoseRecorder.java b/src/main/java/io/eiren/gui/autobone/PoseRecorder.java new file mode 100644 index 0000000000..a6499ea891 --- /dev/null +++ b/src/main/java/io/eiren/gui/autobone/PoseRecorder.java @@ -0,0 +1,92 @@ +package io.eiren.gui.autobone; + +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; + +import io.eiren.util.ann.ThreadSafe; +import io.eiren.util.ann.VRServerThread; +import io.eiren.util.collections.FastList; +import io.eiren.util.logging.LogManager; +import io.eiren.vr.VRServer; +import io.eiren.vr.processor.HumanSkeleton; +import io.eiren.vr.processor.HumanSkeletonWithLegs; + +public class PoseRecorder { + + protected final FastList frames = new FastList(); + + protected int numFrames = -1; + protected long frameRecordingInterval = 60L; + protected long nextFrameTimeMs = -1L; + + protected CompletableFuture currentRecording; + + protected final VRServer server; + HumanSkeletonWithLegs skeleton = null; + + public PoseRecorder(VRServer server) { + this.server = server; + server.addOnTick(this::onTick); + server.addSkeletonUpdatedCallback(this::skeletonUpdated); + } + + @VRServerThread + public void onTick() { + HumanSkeletonWithLegs skeleton = this.skeleton; + if (skeleton != null && frames.size() < numFrames && System.currentTimeMillis() >= nextFrameTimeMs) { + nextFrameTimeMs = System.currentTimeMillis() + frameRecordingInterval; + + frames.add(new PoseFrame(skeleton)); + + // If done recording + CompletableFuture currentRecording = this.currentRecording; + if (currentRecording != null && frames.size() >= numFrames) { + currentRecording.complete(frames.toArray(new PoseFrame[0])); + } + } + } + + @ThreadSafe + public void skeletonUpdated(HumanSkeleton newSkeleton) { + if (newSkeleton instanceof HumanSkeletonWithLegs) { + skeleton = (HumanSkeletonWithLegs) newSkeleton; + } + } + + public synchronized Future startFrameRecording(int numFrames, long interval) { + stopFrameRecording(); + + // Clear old frames and ensure new size can be held + frames.clear(); + frames.ensureCapacity(numFrames); + + this.numFrames = numFrames; + + frameRecordingInterval = interval; + nextFrameTimeMs = -1L; + + LogManager.log.info("[PoseRecorder] Recording " + numFrames + " samples at a " + interval + " ms frame interval"); + + currentRecording = new CompletableFuture(); + return currentRecording; + } + + public synchronized void stopFrameRecording() { + numFrames = -1; + + // Synchronized, this value can be expected to stay the same + if (currentRecording != null) { + currentRecording.complete(frames.toArray(new PoseFrame[0])); + } + } + + public boolean isRecording() { + return numFrames > frames.size(); + } + + public PoseFrame[] getFrames() throws ExecutionException, InterruptedException { + CompletableFuture currentRecording = this.currentRecording; + return currentRecording != null ? currentRecording.get() : null; + } +} From 6b68a983a5688bf101e04e0974093b457f5a5487 Mon Sep 17 00:00:00 2001 From: ButterscotchVanilla Date: Mon, 16 Aug 2021 02:26:58 -0400 Subject: [PATCH 44/60] AutoBone: Restructure processFrames and remove unused code --- .../java/io/eiren/gui/autobone/AutoBone.java | 225 ++++++------------ 1 file changed, 69 insertions(+), 156 deletions(-) diff --git a/src/main/java/io/eiren/gui/autobone/AutoBone.java b/src/main/java/io/eiren/gui/autobone/AutoBone.java index c540a4d750..eac995fa4e 100644 --- a/src/main/java/io/eiren/gui/autobone/AutoBone.java +++ b/src/main/java/io/eiren/gui/autobone/AutoBone.java @@ -17,6 +17,8 @@ public class AutoBone { + public int cursorIncrement = 1; + public int minDataDistance = 1; public int maxDataDistance = 2; @@ -206,15 +208,6 @@ public float processFrames(PoseFrame[] frames, boolean calcInitError, float targ SimpleSkeleton skeleton1 = new SimpleSkeleton(configSet); SimpleSkeleton skeleton2 = new SimpleSkeleton(configSet); - int epochCounter = calcInitError ? -1 : 0; - - int cursorOffset = minDataDistance; - - float adjustRate = initialAdjustRate; - - float sumError = 0f; - int errorCount = 0; - // If target height isn't specified, auto-detect if (targetHeight < 0f) { if (skeleton != null) { @@ -233,182 +226,102 @@ public float processFrames(PoseFrame[] frames, boolean calcInitError, float targ } } - for (;;) { - // Detect end of iteration - if (cursorOffset >= frames.length || cursorOffset > maxDataDistance) { - epochCounter++; - - // Calculate average error over the epoch - float avgError = errorCount > 0 ? sumError / errorCount : -1f; - - // Reset error sum values - sumError = 0f; - errorCount = 0; - - LogManager.log.info("[AutoBone] Epoch " + epochCounter + " average error: " + avgError); - - if (epochCounter >= numEpochs) { - break; - } else { - // Reset cursor offset and decay the adjustment rate - cursorOffset = minDataDistance; - adjustRate /= adjustRateDecay; - } - } + for (int epoch = calcInitError ? -1 : 0; epoch < numEpochs; epoch++) { + float sumError = 0f; + int errorCount = 0; - int frameCursor1 = 0; - int frameCursor2 = cursorOffset++; + float adjustRate = epoch >= 0 ? (float)(initialAdjustRate / Math.pow(adjustRateDecay, epoch)) : 0f; - do { - PoseFrame frame1 = frames[frameCursor1]; - PoseFrame frame2 = frames[frameCursor2]; - - // If there's missing data, skip it - if (frame1 == null || frame2 == null) { - continue; - } + for (int cursorOffset = minDataDistance; cursorOffset <= maxDataDistance && cursorOffset < frames.length ; cursorOffset++) { + for (int frameCursor = 0; frameCursor < frames.length - cursorOffset; frameCursor += cursorIncrement) { + PoseFrame frame1 = frames[frameCursor]; + PoseFrame frame2 = frames[frameCursor + cursorOffset]; - setSkeletonLengths(skeleton1); - setSkeletonLengths(skeleton2); - - skeleton1.setPoseFromFrame(frame1); - skeleton2.setPoseFromFrame(frame2); - - skeleton1.updatePose(); - skeleton2.updatePose(); + // If there's missing data, skip it + if (frame1 == null || frame2 == null) { + continue; + } - float totalLength = getLengthSum(configs); - float curHeight = getHeight(configs); - float errorDeriv = getErrorDeriv(skeleton1, skeleton2, targetHeight - curHeight); - float error = errorFunc(errorDeriv); + setSkeletonLengths(skeleton1); + setSkeletonLengths(skeleton2); - // In case of fire - if (Float.isNaN(error) || Float.isInfinite(error)) { - // Extinguish - LogManager.log.warning("[AutoBone] Error value is invalid, resetting variables to recover"); - reloadConfigValues(); + skeleton1.setPoseFromFrame(frame1); + skeleton2.setPoseFromFrame(frame2); - // Reset error sum values - sumError = 0f; - errorCount = 0; + skeleton1.updatePose(); + skeleton2.updatePose(); - // Continue on new data - continue; - } + float totalLength = getLengthSum(configs); + float curHeight = getHeight(configs); + float errorDeriv = getErrorDeriv(skeleton1, skeleton2, targetHeight - curHeight); + float error = errorFunc(errorDeriv); - // Store the error count for logging purposes - sumError += errorDeriv; - errorCount++; + // In case of fire + if (Float.isNaN(error) || Float.isInfinite(error)) { + // Extinguish + LogManager.log.warning("[AutoBone] Error value is invalid, resetting variables to recover"); + reloadConfigValues(); - float adjustVal = error * adjustRate; + // Reset error sum values + sumError = 0f; + errorCount = 0; - for (Entry entry : configs.entrySet()) { - // Skip adjustment if the epoch is before starting (for logging only) - if (epochCounter < 0) { - break; + // Continue on new data + continue; } - float originalLength = entry.getValue(); + // Store the error count for logging purposes + sumError += errorDeriv; + errorCount++; - // Try positive and negative adjustments - boolean isHeightVar = heightConfigs.contains(entry.getKey()); - float minError = error; - float finalNewLength = -1f; - for (int i = 0; i < 2; i++) { - // Scale by the ratio for smooth adjustment and more stable results - float curAdjustVal = (i == 0 ? adjustVal : -adjustVal) * (originalLength / totalLength); - float newLength = originalLength + curAdjustVal; + float adjustVal = error * adjustRate; - // No small or negative numbers!!! Bad algorithm! - if (newLength < 0.01f) { - continue; + for (Entry entry : configs.entrySet()) { + // Skip adjustment if the epoch is before starting (for logging only) + if (epoch < 0) { + break; } - updateSkeletonBoneLength(skeleton1, skeleton2, entry.getKey(), newLength); + float originalLength = entry.getValue(); - float newHeight = isHeightVar ? curHeight + curAdjustVal : curHeight; - float newError = errorFunc(getErrorDeriv(skeleton1, skeleton2, targetHeight - newHeight)); + // Try positive and negative adjustments + boolean isHeightVar = heightConfigs.contains(entry.getKey()); + float minError = error; + float finalNewLength = -1f; + for (int i = 0; i < 2; i++) { + // Scale by the ratio for smooth adjustment and more stable results + float curAdjustVal = (i == 0 ? adjustVal : -adjustVal) * (originalLength / totalLength); + float newLength = originalLength + curAdjustVal; - if (newError < minError) { - minError = newError; - finalNewLength = newLength; - } - } - - if (finalNewLength > 0f) { - // Keep values within a reasonable range - /* - Float val; - switch (entry.getKey()) { - case "Neck": - val = configs.get("Waist"); - if (val == null) { - break; - } else if (finalNewLength <= NECK_WAIST_RATIO_MIN * val || finalNewLength >= NECK_WAIST_RATIO_MAX * val) { - entry.setValue(Math.min(Math.max(finalNewLength, NECK_WAIST_RATIO_MIN * val), NECK_WAIST_RATIO_MAX * val)); - } else { - entry.setValue(finalNewLength); + // No small or negative numbers!!! Bad algorithm! + if (newLength < 0.01f) { + continue; } - break; - case "Chest": - val = configs.get("Waist"); - if (val == null) { - break; - } else if (finalNewLength <= CHEST_WAIST_RATIO_MIN * val || finalNewLength >= CHEST_WAIST_RATIO_MAX * val) { - entry.setValue(Math.min(Math.max(finalNewLength, CHEST_WAIST_RATIO_MIN * val), CHEST_WAIST_RATIO_MAX * val)); - } else { - entry.setValue(finalNewLength); - } - break; - - case "Hips width": - val = configs.get("Waist"); - if (val == null) { - break; - } else if (finalNewLength < HIP_MIN || finalNewLength >= HIP_WAIST_RATIO_MAX * val) { - entry.setValue(Math.min(Math.max(finalNewLength, HIP_MIN), HIP_WAIST_RATIO_MAX * val)); - } else { - entry.setValue(finalNewLength); - } - break; + updateSkeletonBoneLength(skeleton1, skeleton2, entry.getKey(), newLength); - case "Legs length": - val = configs.get("Waist"); - if (val == null) { - break; - } else if (finalNewLength <= LEG_WAIST_RATIO_MIN * val || finalNewLength >= LEG_WAIST_RATIO_MAX * val) { - entry.setValue(Math.min(Math.max(finalNewLength, LEG_WAIST_RATIO_MIN * val), LEG_WAIST_RATIO_MAX * val)); - } else { - entry.setValue(finalNewLength); - } - break; + float newHeight = isHeightVar ? curHeight + curAdjustVal : curHeight; + float newError = errorFunc(getErrorDeriv(skeleton1, skeleton2, targetHeight - newHeight)); - case "Knee height": - val = configs.get("Legs length"); - if (val == null) { - break; - } else if (finalNewLength <= KNEE_LEG_RATIO_MIN * val || finalNewLength >= KNEE_LEG_RATIO_MAX * val) { - entry.setValue(Math.min(Math.max(finalNewLength, KNEE_LEG_RATIO_MIN * val), KNEE_LEG_RATIO_MAX * val)); - } else { - entry.setValue(finalNewLength); + if (newError < minError) { + minError = newError; + finalNewLength = newLength; } - break; + } - default: + if (finalNewLength > 0f) { entry.setValue(finalNewLength); - break; } - */ - // Temp while above is commented - entry.setValue(finalNewLength); + // Reset the length to minimize bias in other variables, it's applied later + updateSkeletonBoneLength(skeleton1, skeleton2, entry.getKey(), originalLength); } - - // Reset the length to minimize bias in other variables, it's applied later - updateSkeletonBoneLength(skeleton1, skeleton2, entry.getKey(), originalLength); } - } while (++frameCursor1 < frames.length && ++frameCursor2 < frames.length); + } + + // Calculate average error over the epoch + float avgError = errorCount > 0 ? sumError / errorCount : -1f; + LogManager.log.info("[AutoBone] Epoch " + (epoch + 1) + " average error: " + avgError); } float finalHeight = getHeight(configs); From 89e2ea610a33e2f0350307f527e8bf0ffc2b4251 Mon Sep 17 00:00:00 2001 From: ButterscotchVanilla Date: Mon, 16 Aug 2021 03:44:18 -0400 Subject: [PATCH 45/60] AutoBone: Automatically update node positions --- .../java/io/eiren/gui/autobone/AutoBone.java | 23 ++++---------- .../io/eiren/gui/autobone/SimpleSkeleton.java | 30 +++++++++++++++++++ 2 files changed, 36 insertions(+), 17 deletions(-) diff --git a/src/main/java/io/eiren/gui/autobone/AutoBone.java b/src/main/java/io/eiren/gui/autobone/AutoBone.java index eac995fa4e..8d3dafc423 100644 --- a/src/main/java/io/eiren/gui/autobone/AutoBone.java +++ b/src/main/java/io/eiren/gui/autobone/AutoBone.java @@ -91,12 +91,6 @@ public void reloadConfigValues() { configs.put("Knee height", server.config.getFloat("body.kneeHeight", 0.42f)); } - public void setSkeletonLengths(SimpleSkeleton skeleton) { - for (Entry entry : configs.entrySet()) { - skeleton.setSkeletonConfig(entry.getKey(), entry.getValue()); - } - } - @ThreadSafe public void skeletonUpdated(HumanSkeleton newSkeleton) { if (newSkeleton instanceof HumanSkeletonWithLegs) { @@ -232,7 +226,7 @@ public float processFrames(PoseFrame[] frames, boolean calcInitError, float targ float adjustRate = epoch >= 0 ? (float)(initialAdjustRate / Math.pow(adjustRateDecay, epoch)) : 0f; - for (int cursorOffset = minDataDistance; cursorOffset <= maxDataDistance && cursorOffset < frames.length ; cursorOffset++) { + for (int cursorOffset = minDataDistance; cursorOffset <= maxDataDistance && cursorOffset < frames.length; cursorOffset++) { for (int frameCursor = 0; frameCursor < frames.length - cursorOffset; frameCursor += cursorIncrement) { PoseFrame frame1 = frames[frameCursor]; PoseFrame frame2 = frames[frameCursor + cursorOffset]; @@ -242,15 +236,13 @@ public float processFrames(PoseFrame[] frames, boolean calcInitError, float targ continue; } - setSkeletonLengths(skeleton1); - setSkeletonLengths(skeleton2); + configSet = configs.entrySet(); + skeleton1.setSkeletonConfigs(configSet); + skeleton2.setSkeletonConfigs(configSet); skeleton1.setPoseFromFrame(frame1); skeleton2.setPoseFromFrame(frame2); - skeleton1.updatePose(); - skeleton2.updatePose(); - float totalLength = getLengthSum(configs); float curHeight = getHeight(configs); float errorDeriv = getErrorDeriv(skeleton1, skeleton2, targetHeight - curHeight); @@ -362,10 +354,7 @@ protected static float errorFunc(float errorDeriv) { } protected void updateSkeletonBoneLength(SimpleSkeleton skeleton1, SimpleSkeleton skeleton2, String joint, float newLength) { - skeleton1.setSkeletonConfig(joint, newLength); - skeleton2.setSkeletonConfig(joint, newLength); - - skeleton1.updatePose(); - skeleton2.updatePose(); + skeleton1.setSkeletonConfig(joint, newLength, true); + skeleton2.setSkeletonConfig(joint, newLength, true); } } diff --git a/src/main/java/io/eiren/gui/autobone/SimpleSkeleton.java b/src/main/java/io/eiren/gui/autobone/SimpleSkeleton.java index 7029eeb552..710fbf464b 100644 --- a/src/main/java/io/eiren/gui/autobone/SimpleSkeleton.java +++ b/src/main/java/io/eiren/gui/autobone/SimpleSkeleton.java @@ -134,6 +134,8 @@ public void setPoseFromFrame(PoseFrame frame) { targetNode.localTransform.setRotation(rotation.getValue()); } } + + updatePose(); } public void setSkeletonConfigs(Iterable> configs) { @@ -143,28 +145,48 @@ public void setSkeletonConfigs(Iterable> configs) { } public void setSkeletonConfig(String joint, float newLength) { + setSkeletonConfig(joint, newLength, false); + } + + public void setSkeletonConfig(String joint, float newLength, boolean updatePose) { switch(joint) { case "Head": headShift = newLength; headNode.localTransform.setTranslation(0, 0, headShift); + if (updatePose) { + headNode.update(); + } break; case "Neck": neckLength = newLength; neckNode.localTransform.setTranslation(0, -neckLength, 0); + if (updatePose) { + neckNode.update(); + } break; case "Waist": waistDistance = newLength; waistNode.localTransform.setTranslation(0, -(waistDistance - chestDistance), 0); + if (updatePose) { + waistNode.update(); + } break; case "Chest": chestDistance = newLength; chestNode.localTransform.setTranslation(0, -chestDistance, 0); waistNode.localTransform.setTranslation(0, -(waistDistance - chestDistance), 0); + if (updatePose) { + chestNode.update(); + } break; case "Hips width": hipsWidth = newLength; leftHipNode.localTransform.setTranslation(-hipsWidth / 2, 0, 0); rightHipNode.localTransform.setTranslation(hipsWidth / 2, 0, 0); + if (updatePose) { + leftHipNode.update(); + rightHipNode.update(); + } break; case "Knee height": kneeHeight = newLength; @@ -172,11 +194,19 @@ public void setSkeletonConfig(String joint, float newLength) { rightAnkleNode.localTransform.setTranslation(0, -kneeHeight, 0); leftKneeNode.localTransform.setTranslation(0, -(legsLength - kneeHeight), 0); rightKneeNode.localTransform.setTranslation(0, -(legsLength - kneeHeight), 0); + if (updatePose) { + leftKneeNode.update(); + rightKneeNode.update(); + } break; case "Legs length": legsLength = newLength; leftKneeNode.localTransform.setTranslation(0, -(legsLength - kneeHeight), 0); rightKneeNode.localTransform.setTranslation(0, -(legsLength - kneeHeight), 0); + if (updatePose) { + leftKneeNode.update(); + rightKneeNode.update(); + } break; } } From 70f5228d1ccf41eac0deeb5fab4cbd6d2b862650 Mon Sep 17 00:00:00 2001 From: ButterscotchVanilla Date: Mon, 16 Aug 2021 04:49:16 -0400 Subject: [PATCH 46/60] AutoBone: Remove head offset, remove totalLength --- src/main/java/io/eiren/gui/autobone/AutoBone.java | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/main/java/io/eiren/gui/autobone/AutoBone.java b/src/main/java/io/eiren/gui/autobone/AutoBone.java index 8d3dafc423..e83a6c2141 100644 --- a/src/main/java/io/eiren/gui/autobone/AutoBone.java +++ b/src/main/java/io/eiren/gui/autobone/AutoBone.java @@ -20,12 +20,12 @@ public class AutoBone { public int cursorIncrement = 1; public int minDataDistance = 1; - public int maxDataDistance = 2; + public int maxDataDistance = 1; - public int numEpochs = 100; + public int numEpochs = 5; - public float initialAdjustRate = 2.5f; - public float adjustRateDecay = 1.03f; + public float initialAdjustRate = 1.0f; + public float adjustRateDecay = 1.05f; public float slideErrorFactor = 1.0f; public float offsetErrorFactor = 0.0f; @@ -72,7 +72,7 @@ public AutoBone(VRServer server) { public void reloadConfigValues() { // Load waist configs - configs.put("Head", server.config.getFloat("body.headShift", HumanSkeletonWithWaist.HEAD_SHIFT_DEFAULT)); + //configs.put("Head", server.config.getFloat("body.headShift", HumanSkeletonWithWaist.HEAD_SHIFT_DEFAULT)); configs.put("Neck", server.config.getFloat("body.neckLength", HumanSkeletonWithWaist.NECK_LENGTH_DEFAULT)); configs.put("Waist", server.config.getFloat("body.waistDistance", 0.85f)); @@ -243,7 +243,6 @@ public float processFrames(PoseFrame[] frames, boolean calcInitError, float targ skeleton1.setPoseFromFrame(frame1); skeleton2.setPoseFromFrame(frame2); - float totalLength = getLengthSum(configs); float curHeight = getHeight(configs); float errorDeriv = getErrorDeriv(skeleton1, skeleton2, targetHeight - curHeight); float error = errorFunc(errorDeriv); @@ -282,7 +281,7 @@ public float processFrames(PoseFrame[] frames, boolean calcInitError, float targ float finalNewLength = -1f; for (int i = 0; i < 2; i++) { // Scale by the ratio for smooth adjustment and more stable results - float curAdjustVal = (i == 0 ? adjustVal : -adjustVal) * (originalLength / totalLength); + float curAdjustVal = (i == 0 ? adjustVal : -adjustVal) * originalLength; float newLength = originalLength + curAdjustVal; // No small or negative numbers!!! Bad algorithm! From f835eeecdd3b477d7aeb5866e7761efbe8aac6eb Mon Sep 17 00:00:00 2001 From: ButterscotchVanilla Date: Mon, 16 Aug 2021 18:13:09 -0400 Subject: [PATCH 47/60] AutoBone: Use error derivative and add more foot offsets --- .../java/io/eiren/gui/autobone/AutoBone.java | 48 ++++++++++++------- 1 file changed, 32 insertions(+), 16 deletions(-) diff --git a/src/main/java/io/eiren/gui/autobone/AutoBone.java b/src/main/java/io/eiren/gui/autobone/AutoBone.java index e83a6c2141..697fc260f5 100644 --- a/src/main/java/io/eiren/gui/autobone/AutoBone.java +++ b/src/main/java/io/eiren/gui/autobone/AutoBone.java @@ -277,7 +277,7 @@ public float processFrames(PoseFrame[] frames, boolean calcInitError, float targ // Try positive and negative adjustments boolean isHeightVar = heightConfigs.contains(entry.getKey()); - float minError = error; + float minError = errorDeriv; float finalNewLength = -1f; for (int i = 0; i < 2; i++) { // Scale by the ratio for smooth adjustment and more stable results @@ -292,10 +292,10 @@ public float processFrames(PoseFrame[] frames, boolean calcInitError, float targ updateSkeletonBoneLength(skeleton1, skeleton2, entry.getKey(), newLength); float newHeight = isHeightVar ? curHeight + curAdjustVal : curHeight; - float newError = errorFunc(getErrorDeriv(skeleton1, skeleton2, targetHeight - newHeight)); + float newErrorDeriv = getErrorDeriv(skeleton1, skeleton2, targetHeight - newHeight); - if (newError < minError) { - minError = newError; + if (newErrorDeriv < minError) { + minError = newErrorDeriv; finalNewLength = newLength; } } @@ -328,23 +328,39 @@ public float processFrames(PoseFrame[] frames, boolean calcInitError, float targ } protected float getErrorDeriv(SimpleSkeleton skeleton1, SimpleSkeleton skeleton2, float heightChange) { - float slideLeft = skeleton1.getLeftFootPos().distance(skeleton2.getLeftFootPos()); - float slideRight = skeleton1.getRightFootPos().distance(skeleton2.getRightFootPos()); + float totalError = 0f; + float sumWeight = 0f; - // Averaged error - float slideError = (slideLeft + slideRight) / 2f; + if (slideErrorFactor > 0f) { + float slideLeft = skeleton1.getLeftFootPos().distance(skeleton2.getLeftFootPos()); + float slideRight = skeleton1.getRightFootPos().distance(skeleton2.getRightFootPos()); - float dist1 = skeleton1.getLeftFootPos().y - skeleton1.getRightFootPos().y; - float dist2 = skeleton2.getLeftFootPos().y - skeleton2.getRightFootPos().y; + totalError += Math.abs((slideLeft + slideRight) / 2f) * slideErrorFactor; + sumWeight += slideErrorFactor; + } + + if (offsetErrorFactor > 0f) { + float dist1 = skeleton1.getLeftFootPos().y - skeleton1.getRightFootPos().y; + float dist2 = skeleton2.getLeftFootPos().y - skeleton2.getRightFootPos().y; + + float dist3 = skeleton1.getLeftFootPos().y - skeleton2.getRightFootPos().y; + float dist4 = skeleton1.getLeftFootPos().y - skeleton2.getRightFootPos().y; + + float dist5 = skeleton1.getLeftFootPos().y - skeleton2.getLeftFootPos().y; + float dist6 = skeleton1.getRightFootPos().y - skeleton2.getRightFootPos().y; - // Averaged error - float distError = (dist1 + dist2) / 2f; + // Averaged error + totalError += Math.abs((dist1 + dist2 + dist3 + dist4 + dist5 + dist6) / 6f) * offsetErrorFactor; + sumWeight += offsetErrorFactor; + } + + if (heightErrorFactor > 0f) { + totalError += Math.abs(heightChange) * heightErrorFactor; + sumWeight += heightErrorFactor; + } // Minimize sliding, minimize foot height offset, minimize change in total height - return ((slideErrorFactor * Math.abs(slideError)) + - (offsetErrorFactor * Math.abs(distError)) + - (heightErrorFactor * Math.abs(heightChange))) / - (slideErrorFactor + offsetErrorFactor + heightErrorFactor); + return sumWeight > 0f ? totalError / sumWeight : 0f; } // Mean square error function From a8ca2fd6e6148d39dc8b9e659e7f1feafac12cf5 Mon Sep 17 00:00:00 2001 From: ButterscotchVanilla Date: Mon, 16 Aug 2021 19:37:13 -0400 Subject: [PATCH 48/60] AutoBone: Use abs dist for foot offset error, use total length again, and remove hips --- .../java/io/eiren/gui/autobone/AutoBone.java | 30 ++++++++++--------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/src/main/java/io/eiren/gui/autobone/AutoBone.java b/src/main/java/io/eiren/gui/autobone/AutoBone.java index 697fc260f5..d7f8b45663 100644 --- a/src/main/java/io/eiren/gui/autobone/AutoBone.java +++ b/src/main/java/io/eiren/gui/autobone/AutoBone.java @@ -22,10 +22,10 @@ public class AutoBone { public int minDataDistance = 1; public int maxDataDistance = 1; - public int numEpochs = 5; + public int numEpochs = 100; - public float initialAdjustRate = 1.0f; - public float adjustRateDecay = 1.05f; + public float initialAdjustRate = 2.0f; + public float adjustRateDecay = 1.04f; public float slideErrorFactor = 1.0f; public float offsetErrorFactor = 0.0f; @@ -86,7 +86,7 @@ public void reloadConfigValues() { } // Load leg configs - configs.put("Hips width", server.config.getFloat("body.hipsWidth", HumanSkeletonWithLegs.HIPS_WIDTH_DEFAULT)); + //configs.put("Hips width", server.config.getFloat("body.hipsWidth", HumanSkeletonWithLegs.HIPS_WIDTH_DEFAULT)); configs.put("Legs length", server.config.getFloat("body.legsLength", 0.84f)); configs.put("Knee height", server.config.getFloat("body.kneeHeight", 0.42f)); } @@ -243,6 +243,7 @@ public float processFrames(PoseFrame[] frames, boolean calcInitError, float targ skeleton1.setPoseFromFrame(frame1); skeleton2.setPoseFromFrame(frame2); + float totalLength = getLengthSum(configs); float curHeight = getHeight(configs); float errorDeriv = getErrorDeriv(skeleton1, skeleton2, targetHeight - curHeight); float error = errorFunc(errorDeriv); @@ -281,7 +282,7 @@ public float processFrames(PoseFrame[] frames, boolean calcInitError, float targ float finalNewLength = -1f; for (int i = 0; i < 2; i++) { // Scale by the ratio for smooth adjustment and more stable results - float curAdjustVal = (i == 0 ? adjustVal : -adjustVal) * originalLength; + float curAdjustVal = (i == 0 ? adjustVal : -adjustVal) * (originalLength / totalLength); float newLength = originalLength + curAdjustVal; // No small or negative numbers!!! Bad algorithm! @@ -335,22 +336,23 @@ protected float getErrorDeriv(SimpleSkeleton skeleton1, SimpleSkeleton skeleton2 float slideLeft = skeleton1.getLeftFootPos().distance(skeleton2.getLeftFootPos()); float slideRight = skeleton1.getRightFootPos().distance(skeleton2.getRightFootPos()); - totalError += Math.abs((slideLeft + slideRight) / 2f) * slideErrorFactor; + // Divide by 4 to halve and average, it's halved because you want to approach a midpoint, not the other point + totalError += ((slideLeft + slideRight) / 4f) * slideErrorFactor; sumWeight += slideErrorFactor; } if (offsetErrorFactor > 0f) { - float dist1 = skeleton1.getLeftFootPos().y - skeleton1.getRightFootPos().y; - float dist2 = skeleton2.getLeftFootPos().y - skeleton2.getRightFootPos().y; + float dist1 = Math.abs(skeleton1.getLeftFootPos().y - skeleton1.getRightFootPos().y); + float dist2 = Math.abs(skeleton2.getLeftFootPos().y - skeleton2.getRightFootPos().y); - float dist3 = skeleton1.getLeftFootPos().y - skeleton2.getRightFootPos().y; - float dist4 = skeleton1.getLeftFootPos().y - skeleton2.getRightFootPos().y; + float dist3 = Math.abs(skeleton1.getLeftFootPos().y - skeleton2.getRightFootPos().y); + float dist4 = Math.abs(skeleton1.getLeftFootPos().y - skeleton2.getRightFootPos().y); - float dist5 = skeleton1.getLeftFootPos().y - skeleton2.getLeftFootPos().y; - float dist6 = skeleton1.getRightFootPos().y - skeleton2.getRightFootPos().y; + float dist5 = Math.abs(skeleton1.getLeftFootPos().y - skeleton2.getLeftFootPos().y); + float dist6 = Math.abs(skeleton1.getRightFootPos().y - skeleton2.getRightFootPos().y); - // Averaged error - totalError += Math.abs((dist1 + dist2 + dist3 + dist4 + dist5 + dist6) / 6f) * offsetErrorFactor; + // Divide by 12 to halve and average, it's halved because you want to approach a midpoint, not the other point + totalError += ((dist1 + dist2 + dist3 + dist4 + dist5 + dist6) / 12f) * offsetErrorFactor; sumWeight += offsetErrorFactor; } From a1f709ca124620ade192b75f2361d4afe5fbe077 Mon Sep 17 00:00:00 2001 From: ButterscotchVanilla Date: Mon, 16 Aug 2021 23:55:20 -0400 Subject: [PATCH 49/60] AutoBone: Add unused configs to staticConfigs and split error function --- .../java/io/eiren/gui/SkeletonConfig.java | 14 +- .../java/io/eiren/gui/autobone/AutoBone.java | 130 ++++++++++-------- 2 files changed, 80 insertions(+), 64 deletions(-) diff --git a/src/main/java/io/eiren/gui/SkeletonConfig.java b/src/main/java/io/eiren/gui/SkeletonConfig.java index 9d15617f31..5e2c1cdfe5 100644 --- a/src/main/java/io/eiren/gui/SkeletonConfig.java +++ b/src/main/java/io/eiren/gui/SkeletonConfig.java @@ -200,23 +200,25 @@ public void run() { LogManager.log.info("[AutoBone] Done processing!"); //#region Stats/Values - Float neckLength = autoBone.configs.get("Neck"); - Float chestLength = autoBone.configs.get("Chest"); - Float waistLength = autoBone.configs.get("Waist"); - Float hipWidth = autoBone.configs.get("Hips width"); - Float legsLength = autoBone.configs.get("Legs length"); - Float kneeHeight = autoBone.configs.get("Knee height"); + Float neckLength = autoBone.getConfig("Neck"); + Float chestLength = autoBone.getConfig("Chest"); + Float waistLength = autoBone.getConfig("Waist"); + Float hipWidth = autoBone.getConfig("Hips width"); + Float legsLength = autoBone.getConfig("Legs length"); + Float kneeHeight = autoBone.getConfig("Knee height"); float neckWaist = neckLength != null && waistLength != null ? neckLength / waistLength : 0f; float chestWaist = chestLength != null && waistLength != null ? chestLength / waistLength : 0f; float hipWaist = hipWidth != null && waistLength != null ? hipWidth / waistLength : 0f; float legWaist = legsLength != null && waistLength != null ? legsLength / waistLength : 0f; + float legBody = legsLength != null && waistLength != null && neckLength != null ? legsLength / (waistLength + neckLength) : 0f; float kneeLeg = kneeHeight != null && legsLength != null ? kneeHeight / legsLength : 0f; LogManager.log.info("[AutoBone] Ratios: [{Neck-Waist: " + StringUtils.prettyNumber(neckWaist) + "}, {Chest-Waist: " + StringUtils.prettyNumber(chestWaist) + "}, {Hip-Waist: " + StringUtils.prettyNumber(hipWaist) + "}, {Leg-Waist: " + StringUtils.prettyNumber(legWaist) + + "}, {Leg-Body: " + StringUtils.prettyNumber(legBody) + "}, {Knee-Leg: " + StringUtils.prettyNumber(kneeLeg) + "}]"); boolean first = true; diff --git a/src/main/java/io/eiren/gui/autobone/AutoBone.java b/src/main/java/io/eiren/gui/autobone/AutoBone.java index d7f8b45663..3143f61a14 100644 --- a/src/main/java/io/eiren/gui/autobone/AutoBone.java +++ b/src/main/java/io/eiren/gui/autobone/AutoBone.java @@ -19,17 +19,17 @@ public class AutoBone { public int cursorIncrement = 1; - public int minDataDistance = 1; - public int maxDataDistance = 1; + public int minDataDistance = 2; + public int maxDataDistance = 32; - public int numEpochs = 100; + public int numEpochs = 50; public float initialAdjustRate = 2.0f; - public float adjustRateDecay = 1.04f; + public float adjustRateDecay = 1.01f; public float slideErrorFactor = 1.0f; public float offsetErrorFactor = 0.0f; - public float heightErrorFactor = 0.004f; + public float heightErrorFactor = 0.05f; /* public float NECK_WAIST_RATIO_MIN = 0.2f; @@ -55,6 +55,7 @@ public class AutoBone { // This is filled by reloadConfigValues() public final HashMap configs = new HashMap(); + public final HashMap staticConfigs = new HashMap(); public final FastList heightConfigs = new FastList(new String[] { "Neck", @@ -72,8 +73,8 @@ public AutoBone(VRServer server) { public void reloadConfigValues() { // Load waist configs - //configs.put("Head", server.config.getFloat("body.headShift", HumanSkeletonWithWaist.HEAD_SHIFT_DEFAULT)); - configs.put("Neck", server.config.getFloat("body.neckLength", HumanSkeletonWithWaist.NECK_LENGTH_DEFAULT)); + staticConfigs.put("Head", server.config.getFloat("body.headShift", HumanSkeletonWithWaist.HEAD_SHIFT_DEFAULT)); + staticConfigs.put("Neck", server.config.getFloat("body.neckLength", HumanSkeletonWithWaist.NECK_LENGTH_DEFAULT)); configs.put("Waist", server.config.getFloat("body.waistDistance", 0.85f)); if (server.config.getBoolean("autobone.forceChestTracker", false) || @@ -83,10 +84,11 @@ public void reloadConfigValues() { } else { // Otherwise, make sure it's not used configs.remove("Chest"); + staticConfigs.put("Chest", server.config.getFloat("body.chestDistance", 0.42f)); } // Load leg configs - //configs.put("Hips width", server.config.getFloat("body.hipsWidth", HumanSkeletonWithLegs.HIPS_WIDTH_DEFAULT)); + staticConfigs.put("Hips width", server.config.getFloat("body.hipsWidth", HumanSkeletonWithLegs.HIPS_WIDTH_DEFAULT)); configs.put("Legs length", server.config.getFloat("body.legsLength", 0.84f)); configs.put("Knee height", server.config.getFloat("body.kneeHeight", 0.42f)); } @@ -115,51 +117,49 @@ public boolean applyConfigToSkeleton(HumanSkeleton skeleton) { return true; } - // This doesn't require a skeleton, therefore can be used if skeleton is null - public void saveConfigs() { - Float headOffset = configs.get("Head"); - if (headOffset != null) { - server.config.setProperty("body.headShift", headOffset); - } - - Float neckLength = configs.get("Neck"); - if (neckLength != null) { - server.config.setProperty("body.neckLength", neckLength); - } - - Float waistLength = configs.get("Waist"); - if (waistLength != null) { - server.config.setProperty("body.waistDistance", waistLength); + private void setConfig(String name, String path) { + Float value = configs.get(name); + if (value != null) { + server.config.setProperty(path, value); } + } - Float chestDistance = configs.get("Chest"); - if (chestDistance != null) { - server.config.setProperty("body.chestDistance", chestDistance); - } + // This doesn't require a skeleton, therefore can be used if skeleton is null + public void saveConfigs() { + setConfig("Head", "body.headShift"); + setConfig("Neck", "body.neckLength"); + setConfig("Waist", "body.waistDistance"); + setConfig("Chest", "body.chestDistance"); + setConfig("Hips width", "body.hipsWidth"); + setConfig("Legs length", "body.legsLength"); + setConfig("Knee height", "body.kneeHeight"); - Float hipsWidth = configs.get("Hips width"); - if (hipsWidth != null) { - server.config.setProperty("body.hipsWidth", hipsWidth); - } + server.saveConfig(); + } - Float legsLength = configs.get("Legs length"); - if (legsLength != null) { - server.config.setProperty("body.legsLength", legsLength); - } + public Float getConfig(String config) { + Float configVal = configs.get(config); + return configVal != null ? configVal : staticConfigs.get(config); + } - Float kneeHeight = configs.get("Knee height"); - if (kneeHeight != null) { - server.config.setProperty("body.kneeHeight", kneeHeight); + public Float getConfig(String config, Map configs, Map configsAlt) { + if (configs == null) { + throw new NullPointerException("Argument \"configs\" must not be null"); } - server.saveConfig(); + Float configVal = configs.get(config); + return configVal != null || configsAlt == null ? configVal : configsAlt.get(config); } public float getHeight(Map configs) { + return getHeight(configs, null); + } + + public float getHeight(Map configs, Map configsAlt) { float height = 0f; for (String heightConfig : heightConfigs) { - Float length = configs.get(heightConfig); + Float length = getConfig(heightConfig, configs, configsAlt); if (length != null) { height += length; } @@ -220,6 +220,11 @@ public float processFrames(PoseFrame[] frames, boolean calcInitError, float targ } } + for (Entry entry : configSet) { + entry.setValue(1f); + } + + HashMap iterConfigs = new HashMap(configs); for (int epoch = calcInitError ? -1 : 0; epoch < numEpochs; epoch++) { float sumError = 0f; int errorCount = 0; @@ -244,7 +249,7 @@ public float processFrames(PoseFrame[] frames, boolean calcInitError, float targ skeleton2.setPoseFromFrame(frame2); float totalLength = getLengthSum(configs); - float curHeight = getHeight(configs); + float curHeight = getHeight(configs, staticConfigs); float errorDeriv = getErrorDeriv(skeleton1, skeleton2, targetHeight - curHeight); float error = errorFunc(errorDeriv); @@ -268,6 +273,7 @@ public float processFrames(PoseFrame[] frames, boolean calcInitError, float targ float adjustVal = error * adjustRate; + iterConfigs.putAll(configs); for (Entry entry : configs.entrySet()) { // Skip adjustment if the epoch is before starting (for logging only) if (epoch < 0) { @@ -282,7 +288,7 @@ public float processFrames(PoseFrame[] frames, boolean calcInitError, float targ float finalNewLength = -1f; for (int i = 0; i < 2; i++) { // Scale by the ratio for smooth adjustment and more stable results - float curAdjustVal = (i == 0 ? adjustVal : -adjustVal) * (originalLength / totalLength); + float curAdjustVal = ((i == 0 ? adjustVal : -adjustVal) * originalLength) / totalLength; float newLength = originalLength + curAdjustVal; // No small or negative numbers!!! Bad algorithm! @@ -328,31 +334,39 @@ public float processFrames(PoseFrame[] frames, boolean calcInitError, float targ return Math.abs(finalHeight - targetHeight); } + protected float getSlideErrorDeriv(SimpleSkeleton skeleton1, SimpleSkeleton skeleton2) { + float slideLeft = skeleton1.getLeftFootPos().distance(skeleton2.getLeftFootPos()); + float slideRight = skeleton1.getRightFootPos().distance(skeleton2.getRightFootPos()); + + // Divide by 4 to halve and average, it's halved because you want to approach a midpoint, not the other point + return (slideLeft + slideRight) / 4f; + } + + protected float getOffsetErrorDeriv(SimpleSkeleton skeleton1, SimpleSkeleton skeleton2) { + float dist1 = Math.abs(skeleton1.getLeftFootPos().y - skeleton1.getRightFootPos().y); + float dist2 = Math.abs(skeleton2.getLeftFootPos().y - skeleton2.getRightFootPos().y); + + float dist3 = Math.abs(skeleton1.getLeftFootPos().y - skeleton2.getRightFootPos().y); + float dist4 = Math.abs(skeleton1.getLeftFootPos().y - skeleton2.getRightFootPos().y); + + float dist5 = Math.abs(skeleton1.getLeftFootPos().y - skeleton2.getLeftFootPos().y); + float dist6 = Math.abs(skeleton1.getRightFootPos().y - skeleton2.getRightFootPos().y); + + // Divide by 12 to halve and average, it's halved because you want to approach a midpoint, not the other point + return (dist1 + dist2 + dist3 + dist4 + dist5 + dist6) / 12f; + } + protected float getErrorDeriv(SimpleSkeleton skeleton1, SimpleSkeleton skeleton2, float heightChange) { float totalError = 0f; float sumWeight = 0f; if (slideErrorFactor > 0f) { - float slideLeft = skeleton1.getLeftFootPos().distance(skeleton2.getLeftFootPos()); - float slideRight = skeleton1.getRightFootPos().distance(skeleton2.getRightFootPos()); - - // Divide by 4 to halve and average, it's halved because you want to approach a midpoint, not the other point - totalError += ((slideLeft + slideRight) / 4f) * slideErrorFactor; + totalError += getSlideErrorDeriv(skeleton1, skeleton2) * slideErrorFactor; sumWeight += slideErrorFactor; } if (offsetErrorFactor > 0f) { - float dist1 = Math.abs(skeleton1.getLeftFootPos().y - skeleton1.getRightFootPos().y); - float dist2 = Math.abs(skeleton2.getLeftFootPos().y - skeleton2.getRightFootPos().y); - - float dist3 = Math.abs(skeleton1.getLeftFootPos().y - skeleton2.getRightFootPos().y); - float dist4 = Math.abs(skeleton1.getLeftFootPos().y - skeleton2.getRightFootPos().y); - - float dist5 = Math.abs(skeleton1.getLeftFootPos().y - skeleton2.getLeftFootPos().y); - float dist6 = Math.abs(skeleton1.getRightFootPos().y - skeleton2.getRightFootPos().y); - - // Divide by 12 to halve and average, it's halved because you want to approach a midpoint, not the other point - totalError += ((dist1 + dist2 + dist3 + dist4 + dist5 + dist6) / 12f) * offsetErrorFactor; + totalError += getOffsetErrorDeriv(skeleton1, skeleton2) * offsetErrorFactor; sumWeight += offsetErrorFactor; } From 1e6448c61fc78ba5fc028e699a2b66c5a86eb519 Mon Sep 17 00:00:00 2001 From: ButterscotchVanilla Date: Tue, 17 Aug 2021 00:10:08 -0400 Subject: [PATCH 50/60] AutoBone: Let's pretend this didn't get committed --- src/main/java/io/eiren/gui/autobone/AutoBone.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/main/java/io/eiren/gui/autobone/AutoBone.java b/src/main/java/io/eiren/gui/autobone/AutoBone.java index 3143f61a14..88e6585ae3 100644 --- a/src/main/java/io/eiren/gui/autobone/AutoBone.java +++ b/src/main/java/io/eiren/gui/autobone/AutoBone.java @@ -220,10 +220,6 @@ public float processFrames(PoseFrame[] frames, boolean calcInitError, float targ } } - for (Entry entry : configSet) { - entry.setValue(1f); - } - HashMap iterConfigs = new HashMap(configs); for (int epoch = calcInitError ? -1 : 0; epoch < numEpochs; epoch++) { float sumError = 0f; From 3d90f0b28461be8d3bcbedf5f08af3fc9374e885 Mon Sep 17 00:00:00 2001 From: ButterscotchVanilla Date: Tue, 17 Aug 2021 04:12:21 -0400 Subject: [PATCH 51/60] AutoBone: Add proportion error --- .../java/io/eiren/gui/SkeletonConfig.java | 6 ++- .../java/io/eiren/gui/autobone/AutoBone.java | 50 +++++++++++++------ .../io/eiren/gui/autobone/SimpleSkeleton.java | 44 +++++++++++++++- 3 files changed, 84 insertions(+), 16 deletions(-) diff --git a/src/main/java/io/eiren/gui/SkeletonConfig.java b/src/main/java/io/eiren/gui/SkeletonConfig.java index 5e2c1cdfe5..44849f933a 100644 --- a/src/main/java/io/eiren/gui/SkeletonConfig.java +++ b/src/main/java/io/eiren/gui/SkeletonConfig.java @@ -146,7 +146,7 @@ public void run() { if (frameRecordings.size() > 0) { setText("Wait"); LogManager.log.info("[AutoBone] Done loading frames!"); - } else { + } else if (newSkeleton != null) { setText("Move"); // 1000 samples at 20 ms per sample is 20 seconds int sampleCount = server.config.getInt("autobone.sampleCount", 1000); @@ -176,6 +176,9 @@ public void run() { } } frameRecordings.add(Pair.of("", frames)); + } else { + LogManager.log.severe("[AutoBone] No recordings found in \"" + loadFolder.getPath() + "\" and unable to record..."); + return; } LogManager.log.info("[AutoBone] Processing frames..."); @@ -191,6 +194,7 @@ public void run() { autoBone.adjustRateDecay = server.config.getFloat("autobone.adjustRateDecay", autoBone.adjustRateDecay); autoBone.slideErrorFactor = server.config.getFloat("autobone.slideErrorFactor", autoBone.slideErrorFactor); autoBone.offsetErrorFactor = server.config.getFloat("autobone.offsetErrorFactor", autoBone.offsetErrorFactor); + autoBone.proportionErrorFactor = server.config.getFloat("autobone.proportionErrorFactor", autoBone.proportionErrorFactor); autoBone.heightErrorFactor = server.config.getFloat("autobone.heightErrorFactor", autoBone.heightErrorFactor); boolean calcInitError = server.config.getBoolean("autobone.calculateInitialError", true); diff --git a/src/main/java/io/eiren/gui/autobone/AutoBone.java b/src/main/java/io/eiren/gui/autobone/AutoBone.java index 88e6585ae3..6c936d1a14 100644 --- a/src/main/java/io/eiren/gui/autobone/AutoBone.java +++ b/src/main/java/io/eiren/gui/autobone/AutoBone.java @@ -2,7 +2,6 @@ import java.util.HashMap; import java.util.Map; -import java.util.Set; import java.util.Map.Entry; import io.eiren.util.ann.ThreadSafe; @@ -22,14 +21,15 @@ public class AutoBone { public int minDataDistance = 2; public int maxDataDistance = 32; - public int numEpochs = 50; + public int numEpochs = 5; - public float initialAdjustRate = 2.0f; + public float initialAdjustRate = 2.5f; public float adjustRateDecay = 1.01f; public float slideErrorFactor = 1.0f; public float offsetErrorFactor = 0.0f; - public float heightErrorFactor = 0.05f; + public float proportionErrorFactor = 0.2f; + public float heightErrorFactor = 0.1f; /* public float NECK_WAIST_RATIO_MIN = 0.2f; @@ -197,10 +197,8 @@ public void processFrames(PoseFrame[] frames, float targetHeight) { } public float processFrames(PoseFrame[] frames, boolean calcInitError, float targetHeight) { - Set> configSet = configs.entrySet(); - - SimpleSkeleton skeleton1 = new SimpleSkeleton(configSet); - SimpleSkeleton skeleton2 = new SimpleSkeleton(configSet); + SimpleSkeleton skeleton1 = new SimpleSkeleton(configs, staticConfigs); + SimpleSkeleton skeleton2 = new SimpleSkeleton(configs, staticConfigs); // If target height isn't specified, auto-detect if (targetHeight < 0f) { @@ -220,7 +218,6 @@ public float processFrames(PoseFrame[] frames, boolean calcInitError, float targ } } - HashMap iterConfigs = new HashMap(configs); for (int epoch = calcInitError ? -1 : 0; epoch < numEpochs; epoch++) { float sumError = 0f; int errorCount = 0; @@ -237,9 +234,8 @@ public float processFrames(PoseFrame[] frames, boolean calcInitError, float targ continue; } - configSet = configs.entrySet(); - skeleton1.setSkeletonConfigs(configSet); - skeleton2.setSkeletonConfigs(configSet); + skeleton1.setSkeletonConfigs(configs); + skeleton2.setSkeletonConfigs(configs); skeleton1.setPoseFromFrame(frame1); skeleton2.setPoseFromFrame(frame2); @@ -269,7 +265,6 @@ public float processFrames(PoseFrame[] frames, boolean calcInitError, float targ float adjustVal = error * adjustRate; - iterConfigs.putAll(configs); for (Entry entry : configs.entrySet()) { // Skip adjustment if the epoch is before starting (for logging only) if (epoch < 0) { @@ -318,7 +313,7 @@ public float processFrames(PoseFrame[] frames, boolean calcInitError, float targ LogManager.log.info("[AutoBone] Epoch " + (epoch + 1) + " average error: " + avgError); } - float finalHeight = getHeight(configs); + float finalHeight = getHeight(configs, staticConfigs); LogManager.log.info("[AutoBone] Target height: " + targetHeight + " New height: " + finalHeight); LogManager.log.info("[AutoBone] Done! Applying to skeleton..."); @@ -352,6 +347,28 @@ protected float getOffsetErrorDeriv(SimpleSkeleton skeleton1, SimpleSkeleton ske return (dist1 + dist2 + dist3 + dist4 + dist5 + dist6) / 12f; } + protected float getProportionErrorDeriv(SimpleSkeleton skeleton) { + Float neckLength = skeleton.getSkeletonConfig("Neck"); + Float chestLength = skeleton.getSkeletonConfig("Chest"); + Float waistLength = skeleton.getSkeletonConfig("Waist"); + Float legsLength = skeleton.getSkeletonConfig("Legs length"); + Float kneeHeight = skeleton.getSkeletonConfig("Knee height"); + + float chestWaist = chestLength != null && waistLength != null ? Math.abs((chestLength / waistLength) - 0.5f) : 0f; + float legBody = legsLength != null && waistLength != null && neckLength != null ? Math.abs((legsLength / (waistLength + neckLength)) - 1.1235f) : 0f; + float kneeLeg = kneeHeight != null && legsLength != null ? Math.abs((kneeHeight / legsLength) - 0.5f) : 0f; + + // SD of 0.07, capture 68% within range + float sdValue = 0.07f; + if (legBody <= sdValue) { + legBody = 0f; + } else { + legBody -= sdValue; + } + + return (chestWaist + legBody + kneeLeg) / 3f; + } + protected float getErrorDeriv(SimpleSkeleton skeleton1, SimpleSkeleton skeleton2, float heightChange) { float totalError = 0f; float sumWeight = 0f; @@ -366,6 +383,11 @@ protected float getErrorDeriv(SimpleSkeleton skeleton1, SimpleSkeleton skeleton2 sumWeight += offsetErrorFactor; } + if (proportionErrorFactor > 0f) { + totalError += getProportionErrorDeriv(skeleton1) * proportionErrorFactor; + sumWeight += proportionErrorFactor; + } + if (heightErrorFactor > 0f) { totalError += Math.abs(heightChange) * heightErrorFactor; sumWeight += heightErrorFactor; diff --git a/src/main/java/io/eiren/gui/autobone/SimpleSkeleton.java b/src/main/java/io/eiren/gui/autobone/SimpleSkeleton.java index 710fbf464b..07accdbba1 100644 --- a/src/main/java/io/eiren/gui/autobone/SimpleSkeleton.java +++ b/src/main/java/io/eiren/gui/autobone/SimpleSkeleton.java @@ -1,6 +1,7 @@ package io.eiren.gui.autobone; import java.util.HashMap; +import java.util.Map; import java.util.Map.Entry; import com.jme3.math.Quaternion; @@ -96,14 +97,30 @@ public SimpleSkeleton() { }); } - public SimpleSkeleton(Iterable> configs) { + public SimpleSkeleton(Iterable> configs, Iterable> altConfigs) { // Initialize this(); // Set configs + if (altConfigs != null) { + // Set alts first, so if there's any overlap it doesn't affect the values + setSkeletonConfigs(altConfigs); + } setSkeletonConfigs(configs); } + public SimpleSkeleton(Map configs, Map altConfigs) { + this(configs.entrySet(), altConfigs.entrySet()); + } + + public SimpleSkeleton(Iterable> configs) { + this(configs, null); + } + + public SimpleSkeleton(Map configs) { + this(configs.entrySet()); + } + public void setPoseFromSkeleton(HumanSkeletonWithLegs humanSkeleton) { TransformNode rootNode = humanSkeleton.getRootNode(); @@ -144,6 +161,10 @@ public void setSkeletonConfigs(Iterable> configs) { } } + public void setSkeletonConfigs(Map configs) { + setSkeletonConfigs(configs.entrySet()); + } + public void setSkeletonConfig(String joint, float newLength) { setSkeletonConfig(joint, newLength, false); } @@ -211,6 +232,27 @@ public void setSkeletonConfig(String joint, float newLength, boolean updatePose) } } + public Float getSkeletonConfig(String joint) { + switch(joint) { + case "Head": + return headShift; + case "Neck": + return neckLength; + case "Waist": + return waistDistance; + case "Chest": + return chestDistance; + case "Hips width": + return hipsWidth; + case "Knee height": + return kneeHeight; + case "Legs length": + return legsLength; + } + + return null; + } + public void updatePose() { hmdNode.update(); } From 23a3babf3310a495d15cf6542d8996d2e3391050 Mon Sep 17 00:00:00 2001 From: ButterscotchVanilla Date: Tue, 17 Aug 2021 16:28:13 -0400 Subject: [PATCH 52/60] AutoBone: Fix recording --- src/main/java/io/eiren/gui/SkeletonConfig.java | 2 +- src/main/java/io/eiren/gui/autobone/PoseRecorder.java | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/main/java/io/eiren/gui/SkeletonConfig.java b/src/main/java/io/eiren/gui/SkeletonConfig.java index 44849f933a..707fb5b1f5 100644 --- a/src/main/java/io/eiren/gui/SkeletonConfig.java +++ b/src/main/java/io/eiren/gui/SkeletonConfig.java @@ -146,7 +146,7 @@ public void run() { if (frameRecordings.size() > 0) { setText("Wait"); LogManager.log.info("[AutoBone] Done loading frames!"); - } else if (newSkeleton != null) { + } else if (poseRecorder.isReadyToRecord()) { setText("Move"); // 1000 samples at 20 ms per sample is 20 seconds int sampleCount = server.config.getInt("autobone.sampleCount", 1000); diff --git a/src/main/java/io/eiren/gui/autobone/PoseRecorder.java b/src/main/java/io/eiren/gui/autobone/PoseRecorder.java index a6499ea891..522265b015 100644 --- a/src/main/java/io/eiren/gui/autobone/PoseRecorder.java +++ b/src/main/java/io/eiren/gui/autobone/PoseRecorder.java @@ -85,6 +85,10 @@ public boolean isRecording() { return numFrames > frames.size(); } + public boolean isReadyToRecord() { + return skeleton != null; + } + public PoseFrame[] getFrames() throws ExecutionException, InterruptedException { CompletableFuture currentRecording = this.currentRecording; return currentRecording != null ? currentRecording.get() : null; From 32a29c8bc717180f821a800941e8b02372a8d456 Mon Sep 17 00:00:00 2001 From: ButterscotchVanilla Date: Wed, 18 Aug 2021 23:59:12 -0400 Subject: [PATCH 53/60] AutoBone: Add new dedicated AutoBone window --- .../java/io/eiren/gui/SkeletonConfig.java | 184 +--------- .../java/io/eiren/gui/autobone/AutoBone.java | 48 ++- .../io/eiren/gui/autobone/AutoBoneWindow.java | 318 ++++++++++++++++++ .../io/eiren/gui/autobone/PoseRecorder.java | 8 + 4 files changed, 372 insertions(+), 186 deletions(-) create mode 100644 src/main/java/io/eiren/gui/autobone/AutoBoneWindow.java diff --git a/src/main/java/io/eiren/gui/SkeletonConfig.java b/src/main/java/io/eiren/gui/SkeletonConfig.java index 707fb5b1f5..e01fea9d9e 100644 --- a/src/main/java/io/eiren/gui/SkeletonConfig.java +++ b/src/main/java/io/eiren/gui/SkeletonConfig.java @@ -3,46 +3,33 @@ import java.awt.event.ItemEvent; import java.awt.event.ItemListener; import java.awt.event.MouseEvent; -import java.io.File; import java.util.HashMap; import java.util.Map; -import java.util.Map.Entry; -import java.util.concurrent.Future; import javax.swing.JButton; import javax.swing.JCheckBox; import javax.swing.JLabel; import javax.swing.event.MouseInputAdapter; -import io.eiren.gui.autobone.PoseRecorder; -import org.apache.commons.lang3.tuple.Pair; - -import io.eiren.gui.autobone.AutoBone; -import io.eiren.gui.autobone.PoseFrame; -import io.eiren.gui.autobone.PoseRecordIO; +import io.eiren.gui.autobone.AutoBoneWindow; import io.eiren.util.StringUtils; import io.eiren.util.ann.ThreadSafe; -import io.eiren.util.collections.FastList; import io.eiren.vr.VRServer; import io.eiren.vr.processor.HumanSkeletonWithLegs; import io.eiren.vr.processor.HumanSkeleton; -import io.eiren.util.logging.LogManager; public class SkeletonConfig extends EJBag { private final VRServer server; private final VRServerGUI gui; - private final PoseRecorder poseRecorder; - private final AutoBone autoBone; - private Thread autoBoneThread = null; + private final AutoBoneWindow autoBone; private Map labels = new HashMap<>(); public SkeletonConfig(VRServer server, VRServerGUI gui) { super(); this.server = server; this.gui = gui; - this.poseRecorder = new PoseRecorder(server); - this.autoBone = new AutoBone(server); + this.autoBone = new AutoBoneWindow(server, this); setAlignmentY(TOP_ALIGNMENT); server.humanPoseProcessor.addSkeletonUpdatedCallback(this::skeletonUpdated); @@ -111,169 +98,8 @@ public void itemStateChanged(ItemEvent e) { addMouseListener(new MouseInputAdapter() { @Override public void mouseClicked(MouseEvent e) { - // Prevent running multiple times - if (autoBoneThread != null) { - return; - } - - Thread thread = new Thread() { - @Override - public void run() { - try { - FastList> frameRecordings = new FastList>(); - - File loadFolder = new File("LoadRecordings"); - if (loadFolder.isDirectory()) { - setText("Load"); - - File[] files = loadFolder.listFiles(); - if (files != null) { - for (File file : files) { - if (file.isFile() && org.apache.commons.lang3.StringUtils.endsWithIgnoreCase(file.getName(), ".abf")) { - LogManager.log.info("[AutoBone] Detected recording at \"" + file.getPath() + "\", loading frames..."); - PoseFrame[] frames = PoseRecordIO.readFromFile(file); - - if (frames == null) { - LogManager.log.severe("Reading frames from \"" + file.getPath() + "\" failed..."); - } else { - frameRecordings.add(Pair.of(file.getName(), frames)); - } - } - } - } - } - - if (frameRecordings.size() > 0) { - setText("Wait"); - LogManager.log.info("[AutoBone] Done loading frames!"); - } else if (poseRecorder.isReadyToRecord()) { - setText("Move"); - // 1000 samples at 20 ms per sample is 20 seconds - int sampleCount = server.config.getInt("autobone.sampleCount", 1000); - long sampleRate = server.config.getLong("autobone.sampleRateMs", 20L); - Future framesFuture = poseRecorder.startFrameRecording(sampleCount, sampleRate); - PoseFrame[] frames = framesFuture.get(); - LogManager.log.info("[AutoBone] Done recording!"); - - setText("Wait"); - if (server.config.getBoolean("autobone.saveRecordings", true)) { - File saveFolder = new File("Recordings"); - if (saveFolder.isDirectory() || saveFolder.mkdirs()) { - File saveRecording; - int recordingIndex = 1; - do { - saveRecording = new File(saveFolder, "ABRecording" + recordingIndex++ + ".abf"); - } while (saveRecording.exists()); - - LogManager.log.info("[AutoBone] Exporting frames to \"" + saveRecording.getPath() + "\"..."); - if (PoseRecordIO.writeToFile(saveRecording, frames)) { - LogManager.log.info("[AutoBone] Done exporting! Recording can be found at \"" + saveRecording.getPath() + "\"."); - } else { - LogManager.log.severe("[AutoBone] Failed to export the recording to \"" + saveRecording.getPath() + "\"."); - } - } else { - LogManager.log.severe("[AutoBone] Failed to create the recording directory \"" + saveFolder.getPath() + "\"."); - } - } - frameRecordings.add(Pair.of("", frames)); - } else { - LogManager.log.severe("[AutoBone] No recordings found in \"" + loadFolder.getPath() + "\" and unable to record..."); - return; - } - - LogManager.log.info("[AutoBone] Processing frames..."); - FastList heightPercentError = new FastList(frameRecordings.size()); - for (Pair recording : frameRecordings) { - LogManager.log.info("[AutoBone] Processing frames from \"" + recording.getKey() + "\"..."); - autoBone.reloadConfigValues(); - - autoBone.minDataDistance = server.config.getInt("autobone.minimumDataDistance", autoBone.minDataDistance); - autoBone.maxDataDistance = server.config.getInt("autobone.maximumDataDistance", autoBone.maxDataDistance); - autoBone.numEpochs = server.config.getInt("autobone.epochCount", autoBone.numEpochs); - autoBone.initialAdjustRate = server.config.getFloat("autobone.adjustRate", autoBone.initialAdjustRate); - autoBone.adjustRateDecay = server.config.getFloat("autobone.adjustRateDecay", autoBone.adjustRateDecay); - autoBone.slideErrorFactor = server.config.getFloat("autobone.slideErrorFactor", autoBone.slideErrorFactor); - autoBone.offsetErrorFactor = server.config.getFloat("autobone.offsetErrorFactor", autoBone.offsetErrorFactor); - autoBone.proportionErrorFactor = server.config.getFloat("autobone.proportionErrorFactor", autoBone.proportionErrorFactor); - autoBone.heightErrorFactor = server.config.getFloat("autobone.heightErrorFactor", autoBone.heightErrorFactor); - - boolean calcInitError = server.config.getBoolean("autobone.calculateInitialError", true); - float targetHeight = server.config.getFloat("autobone.manualTargetHeight", -1f); - heightPercentError.add(autoBone.processFrames(recording.getValue(), calcInitError, targetHeight)); - - LogManager.log.info("[AutoBone] Done processing!"); - - //#region Stats/Values - Float neckLength = autoBone.getConfig("Neck"); - Float chestLength = autoBone.getConfig("Chest"); - Float waistLength = autoBone.getConfig("Waist"); - Float hipWidth = autoBone.getConfig("Hips width"); - Float legsLength = autoBone.getConfig("Legs length"); - Float kneeHeight = autoBone.getConfig("Knee height"); - - float neckWaist = neckLength != null && waistLength != null ? neckLength / waistLength : 0f; - float chestWaist = chestLength != null && waistLength != null ? chestLength / waistLength : 0f; - float hipWaist = hipWidth != null && waistLength != null ? hipWidth / waistLength : 0f; - float legWaist = legsLength != null && waistLength != null ? legsLength / waistLength : 0f; - float legBody = legsLength != null && waistLength != null && neckLength != null ? legsLength / (waistLength + neckLength) : 0f; - float kneeLeg = kneeHeight != null && legsLength != null ? kneeHeight / legsLength : 0f; - - LogManager.log.info("[AutoBone] Ratios: [{Neck-Waist: " + StringUtils.prettyNumber(neckWaist) + - "}, {Chest-Waist: " + StringUtils.prettyNumber(chestWaist) + - "}, {Hip-Waist: " + StringUtils.prettyNumber(hipWaist) + - "}, {Leg-Waist: " + StringUtils.prettyNumber(legWaist) + - "}, {Leg-Body: " + StringUtils.prettyNumber(legBody) + - "}, {Knee-Leg: " + StringUtils.prettyNumber(kneeLeg) + "}]"); - - boolean first = true; - StringBuilder configInfo = new StringBuilder("["); - - for (Entry entry : autoBone.configs.entrySet()) { - if (!first) { - configInfo.append(", "); - } else { - first = false; - } - - configInfo.append("{" + entry.getKey() + ": " + StringUtils.prettyNumber(entry.getValue()) + "}"); - } - - configInfo.append(']'); - - LogManager.log.info("[AutoBone] Length values: " + configInfo.toString()); - } - - if (heightPercentError.size() > 0) { - float mean = 0f; - for (float val : heightPercentError) { - mean += val; - } - mean /= heightPercentError.size(); - - float std = 0f; - for (float val : heightPercentError) { - float stdVal = val - mean; - std += stdVal * stdVal; - } - std = (float)Math.sqrt(std / heightPercentError.size()); - - LogManager.log.info("[AutoBone] Average height error: " + StringUtils.prettyNumber(mean, 6) + " (SD " + StringUtils.prettyNumber(std, 6) + ")"); - } - //#endregion - - // Update GUI values after adjustment - refreshAll(); - } catch (Exception e1) { - LogManager.log.severe("[AutoBone] Failed adjustment!", e1); - } finally { - setText("Auto"); - autoBoneThread = null; - } - } - }; - - autoBoneThread = thread; - thread.start(); + autoBone.setVisible(true); + autoBone.toFront(); } }); }}, s(c(4, row, 1), 3, 1)); diff --git a/src/main/java/io/eiren/gui/autobone/AutoBone.java b/src/main/java/io/eiren/gui/autobone/AutoBone.java index 6c936d1a14..50af70c465 100644 --- a/src/main/java/io/eiren/gui/autobone/AutoBone.java +++ b/src/main/java/io/eiren/gui/autobone/AutoBone.java @@ -3,6 +3,7 @@ import java.util.HashMap; import java.util.Map; import java.util.Map.Entry; +import java.util.function.Consumer; import io.eiren.util.ann.ThreadSafe; import io.eiren.util.logging.LogManager; @@ -16,6 +17,22 @@ public class AutoBone { + public class Epoch { + + public final int epoch; + public final float epochError; + + public Epoch(int epoch, float epochError) { + this.epoch = epoch; + this.epochError = epochError; + } + + @Override + public String toString() { + return "Epoch: " + epoch + ", Epoch Error: " + epochError; + } + } + public int cursorIncrement = 1; public int minDataDistance = 2; @@ -51,7 +68,7 @@ public class AutoBone { protected final VRServer server; - HumanSkeletonWithLegs skeleton = null; + protected HumanSkeletonWithLegs skeleton = null; // This is filled by reloadConfigValues() public final HashMap configs = new HashMap(); @@ -102,6 +119,13 @@ public void skeletonUpdated(HumanSkeleton newSkeleton) { } } + public void applyConfig() { + if (!applyConfigToSkeleton(skeleton)) { + // Unable to apply to skeleton, save directly + saveConfigs(); + } + } + public boolean applyConfigToSkeleton(HumanSkeleton skeleton) { if (skeleton == null) { return false; @@ -192,11 +216,23 @@ public void processFrames(PoseFrame[] frames) { processFrames(frames, -1f); } + public void processFrames(PoseFrame[] frames, Consumer epochCallback) { + processFrames(frames, -1f, epochCallback); + } + public void processFrames(PoseFrame[] frames, float targetHeight) { processFrames(frames, true, targetHeight); } + public void processFrames(PoseFrame[] frames, float targetHeight, Consumer epochCallback) { + processFrames(frames, true, targetHeight, epochCallback); + } + public float processFrames(PoseFrame[] frames, boolean calcInitError, float targetHeight) { + return processFrames(frames, calcInitError, targetHeight, null); + } + + public float processFrames(PoseFrame[] frames, boolean calcInitError, float targetHeight, Consumer epochCallback) { SimpleSkeleton skeleton1 = new SimpleSkeleton(configs, staticConfigs); SimpleSkeleton skeleton2 = new SimpleSkeleton(configs, staticConfigs); @@ -311,17 +347,15 @@ public float processFrames(PoseFrame[] frames, boolean calcInitError, float targ // Calculate average error over the epoch float avgError = errorCount > 0 ? sumError / errorCount : -1f; LogManager.log.info("[AutoBone] Epoch " + (epoch + 1) + " average error: " + avgError); + + if (epochCallback != null) { + epochCallback.accept(new Epoch(epoch + 1, avgError)); + } } float finalHeight = getHeight(configs, staticConfigs); LogManager.log.info("[AutoBone] Target height: " + targetHeight + " New height: " + finalHeight); - LogManager.log.info("[AutoBone] Done! Applying to skeleton..."); - if (!applyConfigToSkeleton(skeleton)) { - LogManager.log.info("[AutoBone] Applying to skeleton failed, only saving configs..."); - saveConfigs(); - } - return Math.abs(finalHeight - targetHeight); } diff --git a/src/main/java/io/eiren/gui/autobone/AutoBoneWindow.java b/src/main/java/io/eiren/gui/autobone/AutoBoneWindow.java new file mode 100644 index 0000000000..3d199a8e59 --- /dev/null +++ b/src/main/java/io/eiren/gui/autobone/AutoBoneWindow.java @@ -0,0 +1,318 @@ +package io.eiren.gui.autobone; + +import java.awt.Container; + +import javax.swing.BoxLayout; +import javax.swing.JFrame; +import javax.swing.JLabel; +import javax.swing.JScrollPane; +import javax.swing.ScrollPaneConstants; +import javax.swing.JButton; +import javax.swing.border.EmptyBorder; +import java.awt.event.MouseEvent; +import java.io.File; +import java.util.Map.Entry; +import java.util.concurrent.Future; + +import io.eiren.gui.EJBox; +import io.eiren.gui.SkeletonConfig; +import io.eiren.util.StringUtils; +import io.eiren.util.ann.AWTThread; +import io.eiren.util.collections.FastList; +import io.eiren.util.logging.LogManager; +import io.eiren.vr.VRServer; +import javax.swing.event.MouseInputAdapter; + +import org.apache.commons.lang3.tuple.Pair; + +public class AutoBoneWindow extends JFrame { + + private EJBox pane; + + private final VRServer server; + private final SkeletonConfig skeletonConfig; + private final PoseRecorder poseRecorder; + private final AutoBone autoBone; + + private Thread recordingThread = null; + private Thread autoBoneThread = null; + + private JLabel processLabel; + private JLabel lengthsLabel; + + public AutoBoneWindow(VRServer server, SkeletonConfig skeletonConfig) { + super("Skeleton Auto-Configuration"); + + this.server = server; + this.skeletonConfig = skeletonConfig; + this.poseRecorder = new PoseRecorder(server); + this.autoBone = new AutoBone(server); + + getContentPane().setLayout(new BoxLayout(getContentPane(), BoxLayout.PAGE_AXIS)); + add(new JScrollPane(pane = new EJBox(BoxLayout.PAGE_AXIS), ScrollPaneConstants.VERTICAL_SCROLLBAR_AS_NEEDED, ScrollPaneConstants.HORIZONTAL_SCROLLBAR_AS_NEEDED)); + + build(); + } + + private String getLengthsString() { + boolean first = true; + StringBuilder configInfo = new StringBuilder(""); + for (Entry entry : autoBone.configs.entrySet()) { + if (!first) { + configInfo.append(", "); + } else { + first = false; + } + + configInfo.append(entry.getKey() + ": " + StringUtils.prettyNumber(entry.getValue() * 100f, 2)); + } + + return configInfo.toString(); + } + + @AWTThread + private void build() { + pane.add(new EJBox(BoxLayout.LINE_AXIS) {{ + setBorder(new EmptyBorder(i(5))); + add(new JButton("Start Recording") {{ + addMouseListener(new MouseInputAdapter() { + @Override + public void mouseClicked(MouseEvent e) { + // Prevent running multiple times + if (recordingThread != null) { + return; + } + + Thread thread = new Thread() { + @Override + public void run() { + try { + if (poseRecorder.isReadyToRecord()) { + setText("Recording..."); + // 1000 samples at 20 ms per sample is 20 seconds + int sampleCount = server.config.getInt("autobone.sampleCount", 1000); + long sampleRate = server.config.getLong("autobone.sampleRateMs", 20L); + Future framesFuture = poseRecorder.startFrameRecording(sampleCount, sampleRate); + PoseFrame[] frames = framesFuture.get(); + LogManager.log.info("[AutoBone] Done recording!"); + + setText("Saving..."); + if (server.config.getBoolean("autobone.saveRecordings", true)) { + File saveFolder = new File("Recordings"); + if (saveFolder.isDirectory() || saveFolder.mkdirs()) { + File saveRecording; + int recordingIndex = 1; + do { + saveRecording = new File(saveFolder, "ABRecording" + recordingIndex++ + ".abf"); + } while (saveRecording.exists()); + + LogManager.log.info("[AutoBone] Exporting frames to \"" + saveRecording.getPath() + "\"..."); + if (PoseRecordIO.writeToFile(saveRecording, frames)) { + LogManager.log.info("[AutoBone] Done exporting! Recording can be found at \"" + saveRecording.getPath() + "\"."); + } else { + LogManager.log.severe("[AutoBone] Failed to export the recording to \"" + saveRecording.getPath() + "\"."); + } + } else { + LogManager.log.severe("[AutoBone] Failed to create the recording directory \"" + saveFolder.getPath() + "\"."); + } + } + } else { + setText("Not Ready..."); + LogManager.log.severe("[AutoBone] Unable to record..."); + Thread.sleep(3000); // Wait for 3 seconds + return; + } + } catch (Exception e) { + setText("Recording Failed..."); + LogManager.log.severe("[AutoBone] Failed recording!", e); + try { + Thread.sleep(3000); // Wait for 3 seconds + } catch (Exception e1) { + // Ignore + } + } finally { + setText("Start Recording"); + recordingThread = null; + } + } + }; + + recordingThread = thread; + thread.start(); + } + }); + }}); + + add(new JButton("Auto-Adjust") {{ + addMouseListener(new MouseInputAdapter() { + @Override + public void mouseClicked(MouseEvent e) { + // Prevent running multiple times + if (autoBoneThread != null) { + return; + } + + Thread thread = new Thread() { + @Override + public void run() { + try { + FastList> frameRecordings = new FastList>(); + + File loadFolder = new File("LoadRecordings"); + if (loadFolder.isDirectory()) { + setText("Load..."); + + File[] files = loadFolder.listFiles(); + if (files != null) { + for (File file : files) { + if (file.isFile() && org.apache.commons.lang3.StringUtils.endsWithIgnoreCase(file.getName(), ".abf")) { + LogManager.log.info("[AutoBone] Detected recording at \"" + file.getPath() + "\", loading frames..."); + PoseFrame[] frames = PoseRecordIO.readFromFile(file); + + if (frames == null) { + LogManager.log.severe("Reading frames from \"" + file.getPath() + "\" failed..."); + } else { + frameRecordings.add(Pair.of(file.getName(), frames)); + } + } + } + } + } + + if (frameRecordings.size() > 0) { + LogManager.log.info("[AutoBone] Done loading frames!"); + } else { + Future frames = poseRecorder.getFramesAsync(); + if (frames != null) { + setText("Waiting for Recording..."); + frameRecordings.add(Pair.of("", frames.get())); + } else { + setText("No Recordings..."); + LogManager.log.severe("[AutoBone] No recordings found in \"" + loadFolder.getPath() + "\" and no recording was done..."); + try { + Thread.sleep(3000); // Wait for 3 seconds + } catch (Exception e1) { + // Ignore + } + return; + } + } + + setText("Processing..."); + LogManager.log.info("[AutoBone] Processing frames..."); + FastList heightPercentError = new FastList(frameRecordings.size()); + for (Pair recording : frameRecordings) { + LogManager.log.info("[AutoBone] Processing frames from \"" + recording.getKey() + "\"..."); + autoBone.reloadConfigValues(); + + autoBone.minDataDistance = server.config.getInt("autobone.minimumDataDistance", autoBone.minDataDistance); + autoBone.maxDataDistance = server.config.getInt("autobone.maximumDataDistance", autoBone.maxDataDistance); + autoBone.numEpochs = server.config.getInt("autobone.epochCount", autoBone.numEpochs); + autoBone.initialAdjustRate = server.config.getFloat("autobone.adjustRate", autoBone.initialAdjustRate); + autoBone.adjustRateDecay = server.config.getFloat("autobone.adjustRateDecay", autoBone.adjustRateDecay); + autoBone.slideErrorFactor = server.config.getFloat("autobone.slideErrorFactor", autoBone.slideErrorFactor); + autoBone.offsetErrorFactor = server.config.getFloat("autobone.offsetErrorFactor", autoBone.offsetErrorFactor); + autoBone.proportionErrorFactor = server.config.getFloat("autobone.proportionErrorFactor", autoBone.proportionErrorFactor); + autoBone.heightErrorFactor = server.config.getFloat("autobone.heightErrorFactor", autoBone.heightErrorFactor); + + boolean calcInitError = server.config.getBoolean("autobone.calculateInitialError", true); + float targetHeight = server.config.getFloat("autobone.manualTargetHeight", -1f); + heightPercentError.add(autoBone.processFrames(recording.getValue(), calcInitError, targetHeight, (epoch) -> { + processLabel.setText(epoch.toString()); + lengthsLabel.setText(getLengthsString()); + })); + + LogManager.log.info("[AutoBone] Done processing!"); + + //#region Stats/Values + Float neckLength = autoBone.getConfig("Neck"); + Float chestLength = autoBone.getConfig("Chest"); + Float waistLength = autoBone.getConfig("Waist"); + Float hipWidth = autoBone.getConfig("Hips width"); + Float legsLength = autoBone.getConfig("Legs length"); + Float kneeHeight = autoBone.getConfig("Knee height"); + + float neckWaist = neckLength != null && waistLength != null ? neckLength / waistLength : 0f; + float chestWaist = chestLength != null && waistLength != null ? chestLength / waistLength : 0f; + float hipWaist = hipWidth != null && waistLength != null ? hipWidth / waistLength : 0f; + float legWaist = legsLength != null && waistLength != null ? legsLength / waistLength : 0f; + float legBody = legsLength != null && waistLength != null && neckLength != null ? legsLength / (waistLength + neckLength) : 0f; + float kneeLeg = kneeHeight != null && legsLength != null ? kneeHeight / legsLength : 0f; + + LogManager.log.info("[AutoBone] Ratios: [{Neck-Waist: " + StringUtils.prettyNumber(neckWaist) + + "}, {Chest-Waist: " + StringUtils.prettyNumber(chestWaist) + + "}, {Hip-Waist: " + StringUtils.prettyNumber(hipWaist) + + "}, {Leg-Waist: " + StringUtils.prettyNumber(legWaist) + + "}, {Leg-Body: " + StringUtils.prettyNumber(legBody) + + "}, {Knee-Leg: " + StringUtils.prettyNumber(kneeLeg) + "}]"); + + String lengthsString = getLengthsString(); + LogManager.log.info("[AutoBone] Length values: " + lengthsString); + lengthsLabel.setText(lengthsString); + } + + if (heightPercentError.size() > 0) { + float mean = 0f; + for (float val : heightPercentError) { + mean += val; + } + mean /= heightPercentError.size(); + + float std = 0f; + for (float val : heightPercentError) { + float stdVal = val - mean; + std += stdVal * stdVal; + } + std = (float)Math.sqrt(std / heightPercentError.size()); + + LogManager.log.info("[AutoBone] Average height error: " + StringUtils.prettyNumber(mean, 6) + " (SD " + StringUtils.prettyNumber(std, 6) + ")"); + } + //#endregion + } catch (Exception e) { + setText("Failed..."); + LogManager.log.severe("[AutoBone] Failed adjustment!", e); + try { + Thread.sleep(3000); // Wait for 3 seconds + } catch (Exception e1) { + // Ignore + } + } finally { + setText("Auto-Adjust"); + autoBoneThread = null; + } + } + }; + + autoBoneThread = thread; + thread.start(); + } + }); + }}); + + add(new JButton("Apply Values") {{ + addMouseListener(new MouseInputAdapter() { + @Override + public void mouseClicked(MouseEvent e) { + autoBone.applyConfig(); + + // Update GUI values after applying + skeletonConfig.refreshAll(); + } + }); + }}); + }}); + + pane.add(new EJBox(BoxLayout.LINE_AXIS) {{ + add(processLabel = new JLabel("Processing has not been started...")); + }}); + + pane.add(new EJBox(BoxLayout.LINE_AXIS) {{ + add(lengthsLabel = new JLabel(getLengthsString())); + }}); + + // Pack and display + pack(); + setLocationRelativeTo(null); + setVisible(false); + } +} diff --git a/src/main/java/io/eiren/gui/autobone/PoseRecorder.java b/src/main/java/io/eiren/gui/autobone/PoseRecorder.java index 522265b015..fd517654c6 100644 --- a/src/main/java/io/eiren/gui/autobone/PoseRecorder.java +++ b/src/main/java/io/eiren/gui/autobone/PoseRecorder.java @@ -55,6 +55,10 @@ public void skeletonUpdated(HumanSkeleton newSkeleton) { } public synchronized Future startFrameRecording(int numFrames, long interval) { + if (!isReadyToRecord()) { + throw new IllegalStateException("PoseRecorder isn't ready to record!"); + } + stopFrameRecording(); // Clear old frames and ensure new size can be held @@ -89,6 +93,10 @@ public boolean isReadyToRecord() { return skeleton != null; } + public Future getFramesAsync() { + return currentRecording; + } + public PoseFrame[] getFrames() throws ExecutionException, InterruptedException { CompletableFuture currentRecording = this.currentRecording; return currentRecording != null ? currentRecording.get() : null; From a7a612aa9bd9b76f72462149a34b35a21ea8a476 Mon Sep 17 00:00:00 2001 From: ButterscotchVanilla Date: Thu, 19 Aug 2021 00:52:09 -0400 Subject: [PATCH 54/60] AutoBone: Add recording cancellation, always check if the recording is done and not submitted --- .../io/eiren/gui/autobone/AutoBoneWindow.java | 2 - .../io/eiren/gui/autobone/PoseRecorder.java | 38 +++++++++++++------ 2 files changed, 26 insertions(+), 14 deletions(-) diff --git a/src/main/java/io/eiren/gui/autobone/AutoBoneWindow.java b/src/main/java/io/eiren/gui/autobone/AutoBoneWindow.java index 3d199a8e59..88ad0ba757 100644 --- a/src/main/java/io/eiren/gui/autobone/AutoBoneWindow.java +++ b/src/main/java/io/eiren/gui/autobone/AutoBoneWindow.java @@ -1,7 +1,5 @@ package io.eiren.gui.autobone; -import java.awt.Container; - import javax.swing.BoxLayout; import javax.swing.JFrame; import javax.swing.JLabel; diff --git a/src/main/java/io/eiren/gui/autobone/PoseRecorder.java b/src/main/java/io/eiren/gui/autobone/PoseRecorder.java index fd517654c6..110851b3c0 100644 --- a/src/main/java/io/eiren/gui/autobone/PoseRecorder.java +++ b/src/main/java/io/eiren/gui/autobone/PoseRecorder.java @@ -34,15 +34,18 @@ public PoseRecorder(VRServer server) { @VRServerThread public void onTick() { HumanSkeletonWithLegs skeleton = this.skeleton; - if (skeleton != null && frames.size() < numFrames && System.currentTimeMillis() >= nextFrameTimeMs) { - nextFrameTimeMs = System.currentTimeMillis() + frameRecordingInterval; - - frames.add(new PoseFrame(skeleton)); - - // If done recording - CompletableFuture currentRecording = this.currentRecording; - if (currentRecording != null && frames.size() >= numFrames) { - currentRecording.complete(frames.toArray(new PoseFrame[0])); + if (skeleton != null) { + if (frames.size() < numFrames) { + if (System.currentTimeMillis() >= nextFrameTimeMs) { + nextFrameTimeMs = System.currentTimeMillis() + frameRecordingInterval; + frames.add(new PoseFrame(skeleton)); + } + } else { + // If done, send finished recording + CompletableFuture currentRecording = this.currentRecording; + if (currentRecording != null && !currentRecording.isDone()) { + currentRecording.complete(frames.toArray(new PoseFrame[0])); + } } } } @@ -59,7 +62,7 @@ public synchronized Future startFrameRecording(int numFrames, long throw new IllegalStateException("PoseRecorder isn't ready to record!"); } - stopFrameRecording(); + cancelFrameRecording(); // Clear old frames and ensure new size can be held frames.clear(); @@ -77,12 +80,23 @@ public synchronized Future startFrameRecording(int numFrames, long } public synchronized void stopFrameRecording() { + // Synchronized, this value can be expected to stay the same + if (currentRecording != null && !currentRecording.isDone()) { + // Stop the recording, returning the frames recorded + currentRecording.complete(frames.toArray(new PoseFrame[0])); + } + numFrames = -1; + } + public synchronized void cancelFrameRecording() { // Synchronized, this value can be expected to stay the same - if (currentRecording != null) { - currentRecording.complete(frames.toArray(new PoseFrame[0])); + if (currentRecording != null && !currentRecording.isDone()) { + // Cancel the current recording and return nothing + currentRecording.cancel(true); } + + numFrames = -1; } public boolean isRecording() { From b05d726ad0dcfedb9bab369e212e896e5ebde55f Mon Sep 17 00:00:00 2001 From: ButterscotchVanilla Date: Thu, 19 Aug 2021 01:54:28 -0400 Subject: [PATCH 55/60] AutoBone: Return recording ASAP and check if it's empty --- .../io/eiren/gui/autobone/AutoBoneWindow.java | 12 +++++++--- .../io/eiren/gui/autobone/PoseRecorder.java | 22 ++++++++++++------- 2 files changed, 23 insertions(+), 11 deletions(-) diff --git a/src/main/java/io/eiren/gui/autobone/AutoBoneWindow.java b/src/main/java/io/eiren/gui/autobone/AutoBoneWindow.java index 88ad0ba757..c1b66ff6a9 100644 --- a/src/main/java/io/eiren/gui/autobone/AutoBoneWindow.java +++ b/src/main/java/io/eiren/gui/autobone/AutoBoneWindow.java @@ -180,10 +180,16 @@ public void run() { if (frameRecordings.size() > 0) { LogManager.log.info("[AutoBone] Done loading frames!"); } else { - Future frames = poseRecorder.getFramesAsync(); - if (frames != null) { + Future framesFuture = poseRecorder.getFramesAsync(); + if (framesFuture != null) { setText("Waiting for Recording..."); - frameRecordings.add(Pair.of("", frames.get())); + PoseFrame[] frames = framesFuture.get(); + + if (frames.length <= 0) { + throw new IllegalStateException("Recording has no frames"); + } + + frameRecordings.add(Pair.of("", frames)); } else { setText("No Recordings..."); LogManager.log.severe("[AutoBone] No recordings found in \"" + loadFolder.getPath() + "\" and no recording was done..."); diff --git a/src/main/java/io/eiren/gui/autobone/PoseRecorder.java b/src/main/java/io/eiren/gui/autobone/PoseRecorder.java index 110851b3c0..dc2fb8c357 100644 --- a/src/main/java/io/eiren/gui/autobone/PoseRecorder.java +++ b/src/main/java/io/eiren/gui/autobone/PoseRecorder.java @@ -39,13 +39,15 @@ public void onTick() { if (System.currentTimeMillis() >= nextFrameTimeMs) { nextFrameTimeMs = System.currentTimeMillis() + frameRecordingInterval; frames.add(new PoseFrame(skeleton)); + + // If done, send finished recording + if (frames.size() >= numFrames) { + internalStopRecording(); + } } } else { - // If done, send finished recording - CompletableFuture currentRecording = this.currentRecording; - if (currentRecording != null && !currentRecording.isDone()) { - currentRecording.complete(frames.toArray(new PoseFrame[0])); - } + // If done and hasn't yet, send finished recording + internalStopRecording(); } } } @@ -79,8 +81,8 @@ public synchronized Future startFrameRecording(int numFrames, long return currentRecording; } - public synchronized void stopFrameRecording() { - // Synchronized, this value can be expected to stay the same + private void internalStopRecording() { + CompletableFuture currentRecording = this.currentRecording; if (currentRecording != null && !currentRecording.isDone()) { // Stop the recording, returning the frames recorded currentRecording.complete(frames.toArray(new PoseFrame[0])); @@ -89,8 +91,12 @@ public synchronized void stopFrameRecording() { numFrames = -1; } + public synchronized void stopFrameRecording() { + internalStopRecording(); + } + public synchronized void cancelFrameRecording() { - // Synchronized, this value can be expected to stay the same + CompletableFuture currentRecording = this.currentRecording; if (currentRecording != null && !currentRecording.isDone()) { // Cancel the current recording and return nothing currentRecording.cancel(true); From bc132b7757cdc17ce86ac305f2ce60b1059b11cc Mon Sep 17 00:00:00 2001 From: ButterscotchVanilla Date: Thu, 19 Aug 2021 02:38:18 -0400 Subject: [PATCH 56/60] AutoBone: Thow NullPointerException for missing frames --- src/main/java/io/eiren/gui/autobone/AutoBone.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/io/eiren/gui/autobone/AutoBone.java b/src/main/java/io/eiren/gui/autobone/AutoBone.java index 50af70c465..e516c1ab6b 100644 --- a/src/main/java/io/eiren/gui/autobone/AutoBone.java +++ b/src/main/java/io/eiren/gui/autobone/AutoBone.java @@ -265,9 +265,9 @@ public float processFrames(PoseFrame[] frames, boolean calcInitError, float targ PoseFrame frame1 = frames[frameCursor]; PoseFrame frame2 = frames[frameCursor + cursorOffset]; - // If there's missing data, skip it + // If there's missing data, throw an exception if (frame1 == null || frame2 == null) { - continue; + throw new NullPointerException("Frames are missing from processing data"); } skeleton1.setSkeletonConfigs(configs); From 9a6cb23659ef069e6499941bcf4859c4e248887a Mon Sep 17 00:00:00 2001 From: ButterscotchVanilla Date: Thu, 19 Aug 2021 04:11:23 -0400 Subject: [PATCH 57/60] AutoBone: Add support for absolute positions --- .../java/io/eiren/gui/autobone/AutoBone.java | 75 ++++++++++++++++- .../io/eiren/gui/autobone/AutoBoneWindow.java | 45 ++++++---- .../java/io/eiren/gui/autobone/PoseFrame.java | 6 +- .../io/eiren/gui/autobone/PoseRecordIO.java | 84 ++++++++++++++----- .../io/eiren/gui/autobone/SimpleSkeleton.java | 21 +++-- 5 files changed, 180 insertions(+), 51 deletions(-) diff --git a/src/main/java/io/eiren/gui/autobone/AutoBone.java b/src/main/java/io/eiren/gui/autobone/AutoBone.java index e516c1ab6b..a7c3cffa87 100644 --- a/src/main/java/io/eiren/gui/autobone/AutoBone.java +++ b/src/main/java/io/eiren/gui/autobone/AutoBone.java @@ -5,6 +5,8 @@ import java.util.Map.Entry; import java.util.function.Consumer; +import com.jme3.math.Vector3f; + import io.eiren.util.ann.ThreadSafe; import io.eiren.util.logging.LogManager; import io.eiren.util.collections.FastList; @@ -47,6 +49,8 @@ public String toString() { public float offsetErrorFactor = 0.0f; public float proportionErrorFactor = 0.2f; public float heightErrorFactor = 0.1f; + public float positionErrorFactor = 0.0f; + public float positionOffsetErrorFactor = 0.0f; /* public float NECK_WAIST_RATIO_MIN = 0.2f; @@ -278,7 +282,7 @@ public float processFrames(PoseFrame[] frames, boolean calcInitError, float targ float totalLength = getLengthSum(configs); float curHeight = getHeight(configs, staticConfigs); - float errorDeriv = getErrorDeriv(skeleton1, skeleton2, targetHeight - curHeight); + float errorDeriv = getErrorDeriv(frame1, frame2, skeleton1, skeleton2, targetHeight - curHeight); float error = errorFunc(errorDeriv); // In case of fire @@ -326,7 +330,7 @@ public float processFrames(PoseFrame[] frames, boolean calcInitError, float targ updateSkeletonBoneLength(skeleton1, skeleton2, entry.getKey(), newLength); float newHeight = isHeightVar ? curHeight + curAdjustVal : curHeight; - float newErrorDeriv = getErrorDeriv(skeleton1, skeleton2, targetHeight - newHeight); + float newErrorDeriv = getErrorDeriv(frame1, frame2, skeleton1, skeleton2, targetHeight - newHeight); if (newErrorDeriv < minError) { minError = newErrorDeriv; @@ -359,6 +363,7 @@ public float processFrames(PoseFrame[] frames, boolean calcInitError, float targ return Math.abs(finalHeight - targetHeight); } + // The change in position of the ankle over time protected float getSlideErrorDeriv(SimpleSkeleton skeleton1, SimpleSkeleton skeleton2) { float slideLeft = skeleton1.getLeftFootPos().distance(skeleton2.getLeftFootPos()); float slideRight = skeleton1.getRightFootPos().distance(skeleton2.getRightFootPos()); @@ -367,6 +372,7 @@ protected float getSlideErrorDeriv(SimpleSkeleton skeleton1, SimpleSkeleton skel return (slideLeft + slideRight) / 4f; } + // The offset between both feet at one instant and over time protected float getOffsetErrorDeriv(SimpleSkeleton skeleton1, SimpleSkeleton skeleton2) { float dist1 = Math.abs(skeleton1.getLeftFootPos().y - skeleton1.getRightFootPos().y); float dist2 = Math.abs(skeleton2.getLeftFootPos().y - skeleton2.getRightFootPos().y); @@ -381,6 +387,7 @@ protected float getOffsetErrorDeriv(SimpleSkeleton skeleton1, SimpleSkeleton ske return (dist1 + dist2 + dist3 + dist4 + dist5 + dist6) / 12f; } + // The distance from average human proportions protected float getProportionErrorDeriv(SimpleSkeleton skeleton) { Float neckLength = skeleton.getSkeletonConfig("Neck"); Float chestLength = skeleton.getSkeletonConfig("Chest"); @@ -403,7 +410,58 @@ protected float getProportionErrorDeriv(SimpleSkeleton skeleton) { return (chestWaist + legBody + kneeLeg) / 3f; } - protected float getErrorDeriv(SimpleSkeleton skeleton1, SimpleSkeleton skeleton2, float heightChange) { + // The distance of any points to the corresponding absolute position + protected float getPositionErrorDeriv(PoseFrame frame, SimpleSkeleton skeleton) { + float offset = 0f; + int offsetCount = 0; + + if (frame.positions != null) { + for (Entry entry : frame.positions.entrySet()) { + Vector3f nodePos = skeleton.getNodePosition(entry.getKey()); + if (nodePos != null) { + offset += Math.abs(nodePos.distance(entry.getValue())); + offsetCount++; + } + } + } + + return offsetCount > 0 ? offset / offsetCount : 0f; + } + + // The difference between offset of absolute position and the corresponding point over time + protected float getPositionOffsetErrorDeriv(PoseFrame frame1, PoseFrame frame2, SimpleSkeleton skeleton1, SimpleSkeleton skeleton2) { + float offset = 0f; + int offsetCount = 0; + + if (frame1.positions != null && frame2.positions != null) { + for (Entry entry : frame1.positions.entrySet()) { + Vector3f frame2Pos = frame2.positions.get(entry.getKey()); + if (frame2Pos == null) { + continue; + } + + Vector3f nodePos1 = skeleton1.getNodePosition(entry.getKey()); + if (nodePos1 == null) { + continue; + } + + Vector3f nodePos2 = skeleton2.getNodePosition(entry.getKey()); + if (nodePos2 == null) { + continue; + } + + float dist1 = Math.abs(nodePos1.distance(entry.getValue())); + float dist2 = Math.abs(nodePos2.distance(frame2Pos)); + + offset += Math.abs(dist2 - dist1); + offsetCount++; + } + } + + return offsetCount > 0 ? offset / offsetCount : 0f; + } + + protected float getErrorDeriv(PoseFrame frame1, PoseFrame frame2, SimpleSkeleton skeleton1, SimpleSkeleton skeleton2, float heightChange) { float totalError = 0f; float sumWeight = 0f; @@ -418,6 +476,7 @@ protected float getErrorDeriv(SimpleSkeleton skeleton1, SimpleSkeleton skeleton2 } if (proportionErrorFactor > 0f) { + // Either skeleton will work fine, skeleton1 is used as a default totalError += getProportionErrorDeriv(skeleton1) * proportionErrorFactor; sumWeight += proportionErrorFactor; } @@ -427,6 +486,16 @@ protected float getErrorDeriv(SimpleSkeleton skeleton1, SimpleSkeleton skeleton2 sumWeight += heightErrorFactor; } + if (positionErrorFactor > 0f) { + totalError += (getPositionErrorDeriv(frame1, skeleton1) + getPositionErrorDeriv(frame2, skeleton2) / 2f) * positionErrorFactor; + sumWeight += positionErrorFactor; + } + + if (positionOffsetErrorFactor > 0f) { + totalError += getPositionOffsetErrorDeriv(frame1, frame2, skeleton1, skeleton2) * positionOffsetErrorFactor; + sumWeight += positionOffsetErrorFactor; + } + // Minimize sliding, minimize foot height offset, minimize change in total height return sumWeight > 0f ? totalError / sumWeight : 0f; } diff --git a/src/main/java/io/eiren/gui/autobone/AutoBoneWindow.java b/src/main/java/io/eiren/gui/autobone/AutoBoneWindow.java index c1b66ff6a9..d6e1f9597c 100644 --- a/src/main/java/io/eiren/gui/autobone/AutoBoneWindow.java +++ b/src/main/java/io/eiren/gui/autobone/AutoBoneWindow.java @@ -68,6 +68,32 @@ private String getLengthsString() { return configInfo.toString(); } + private float processFrames(PoseFrame[] frames) { + autoBone.reloadConfigValues(); + + autoBone.minDataDistance = server.config.getInt("autobone.minimumDataDistance", autoBone.minDataDistance); + autoBone.maxDataDistance = server.config.getInt("autobone.maximumDataDistance", autoBone.maxDataDistance); + + autoBone.numEpochs = server.config.getInt("autobone.epochCount", autoBone.numEpochs); + + autoBone.initialAdjustRate = server.config.getFloat("autobone.adjustRate", autoBone.initialAdjustRate); + autoBone.adjustRateDecay = server.config.getFloat("autobone.adjustRateDecay", autoBone.adjustRateDecay); + + autoBone.slideErrorFactor = server.config.getFloat("autobone.slideErrorFactor", autoBone.slideErrorFactor); + autoBone.offsetErrorFactor = server.config.getFloat("autobone.offsetErrorFactor", autoBone.offsetErrorFactor); + autoBone.proportionErrorFactor = server.config.getFloat("autobone.proportionErrorFactor", autoBone.proportionErrorFactor); + autoBone.heightErrorFactor = server.config.getFloat("autobone.heightErrorFactor", autoBone.heightErrorFactor); + autoBone.positionErrorFactor = server.config.getFloat("autobone.positionErrorFactor", autoBone.positionErrorFactor); + autoBone.positionOffsetErrorFactor = server.config.getFloat("autobone.positionOffsetErrorFactor", autoBone.positionOffsetErrorFactor); + + boolean calcInitError = server.config.getBoolean("autobone.calculateInitialError", true); + float targetHeight = server.config.getFloat("autobone.manualTargetHeight", -1f); + return autoBone.processFrames(frames, calcInitError, targetHeight, (epoch) -> { + processLabel.setText(epoch.toString()); + lengthsLabel.setText(getLengthsString()); + }); + } + @AWTThread private void build() { pane.add(new EJBox(BoxLayout.LINE_AXIS) {{ @@ -207,25 +233,8 @@ public void run() { FastList heightPercentError = new FastList(frameRecordings.size()); for (Pair recording : frameRecordings) { LogManager.log.info("[AutoBone] Processing frames from \"" + recording.getKey() + "\"..."); - autoBone.reloadConfigValues(); - - autoBone.minDataDistance = server.config.getInt("autobone.minimumDataDistance", autoBone.minDataDistance); - autoBone.maxDataDistance = server.config.getInt("autobone.maximumDataDistance", autoBone.maxDataDistance); - autoBone.numEpochs = server.config.getInt("autobone.epochCount", autoBone.numEpochs); - autoBone.initialAdjustRate = server.config.getFloat("autobone.adjustRate", autoBone.initialAdjustRate); - autoBone.adjustRateDecay = server.config.getFloat("autobone.adjustRateDecay", autoBone.adjustRateDecay); - autoBone.slideErrorFactor = server.config.getFloat("autobone.slideErrorFactor", autoBone.slideErrorFactor); - autoBone.offsetErrorFactor = server.config.getFloat("autobone.offsetErrorFactor", autoBone.offsetErrorFactor); - autoBone.proportionErrorFactor = server.config.getFloat("autobone.proportionErrorFactor", autoBone.proportionErrorFactor); - autoBone.heightErrorFactor = server.config.getFloat("autobone.heightErrorFactor", autoBone.heightErrorFactor); - - boolean calcInitError = server.config.getBoolean("autobone.calculateInitialError", true); - float targetHeight = server.config.getFloat("autobone.manualTargetHeight", -1f); - heightPercentError.add(autoBone.processFrames(recording.getValue(), calcInitError, targetHeight, (epoch) -> { - processLabel.setText(epoch.toString()); - lengthsLabel.setText(getLengthsString()); - })); + heightPercentError.add(processFrames(recording.getValue())); LogManager.log.info("[AutoBone] Done processing!"); //#region Stats/Values diff --git a/src/main/java/io/eiren/gui/autobone/PoseFrame.java b/src/main/java/io/eiren/gui/autobone/PoseFrame.java index a7c584917e..81b1d75d56 100644 --- a/src/main/java/io/eiren/gui/autobone/PoseFrame.java +++ b/src/main/java/io/eiren/gui/autobone/PoseFrame.java @@ -12,10 +12,12 @@ public final class PoseFrame { public final Vector3f rootPos; public final HashMap rotations; + public final HashMap positions; - public PoseFrame(Vector3f rootPos, HashMap rotations) { + public PoseFrame(Vector3f rootPos, HashMap rotations, HashMap positions) { this.rootPos = rootPos; this.rotations = rotations; + this.positions = positions; } public PoseFrame(HumanSkeletonWithLegs skeleton) { @@ -29,5 +31,7 @@ public PoseFrame(HumanSkeletonWithLegs skeleton) { // Insert a copied quaternion so it isn't changed by reference rotations.put(visitor.getName(), new Quaternion(visitor.localTransform.getRotation())); }); + + this.positions = null; } } diff --git a/src/main/java/io/eiren/gui/autobone/PoseRecordIO.java b/src/main/java/io/eiren/gui/autobone/PoseRecordIO.java index 9ea3508f31..3fcfb81ddf 100644 --- a/src/main/java/io/eiren/gui/autobone/PoseRecordIO.java +++ b/src/main/java/io/eiren/gui/autobone/PoseRecordIO.java @@ -43,18 +43,39 @@ public static boolean writeFrame(DataOutputStream outputStream, PoseFrame frame) outputStream.writeFloat(frame.rootPos.y); outputStream.writeFloat(frame.rootPos.z); - // Write rotations - outputStream.writeInt(frame.rotations.size()); - for (Entry entry : frame.rotations.entrySet()) { - // Write the label string - outputStream.writeUTF(entry.getKey()); - - // Write the rotation quaternion - Quaternion quat = entry.getValue(); - outputStream.writeFloat(quat.getX()); - outputStream.writeFloat(quat.getY()); - outputStream.writeFloat(quat.getZ()); - outputStream.writeFloat(quat.getW()); + if (frame.rotations != null) { + // Write rotations + outputStream.writeInt(frame.rotations.size()); + for (Entry entry : frame.rotations.entrySet()) { + // Write the label string + outputStream.writeUTF(entry.getKey()); + + // Write the rotation quaternion + Quaternion quat = entry.getValue(); + outputStream.writeFloat(quat.getX()); + outputStream.writeFloat(quat.getY()); + outputStream.writeFloat(quat.getZ()); + outputStream.writeFloat(quat.getW()); + } + } else { + outputStream.writeInt(0); + } + + if (frame.positions != null) { + // Write positions + outputStream.writeInt(frame.positions.size()); + for (Entry entry : frame.positions.entrySet()) { + // Write the label string + outputStream.writeUTF(entry.getKey()); + + // Write the rotation quaternion + Vector3f vec = entry.getValue(); + outputStream.writeFloat(vec.getX()); + outputStream.writeFloat(vec.getY()); + outputStream.writeFloat(vec.getZ()); + } + } else { + outputStream.writeInt(0); } } catch (Exception e) { LogManager.log.severe("Error writing frame to stream", e); @@ -101,20 +122,39 @@ public static PoseFrame readFrame(DataInputStream inputStream) { Vector3f vector = new Vector3f(vecX, vecY, vecZ); int rotationCount = inputStream.readInt(); - HashMap rotations = new HashMap(rotationCount); - for (int j = 0; j < rotationCount; j++) { - String label = inputStream.readUTF(); + HashMap rotations = null; + if (rotationCount > 0) { + rotations = new HashMap(rotationCount); + for (int j = 0; j < rotationCount; j++) { + String label = inputStream.readUTF(); + + float quatX = inputStream.readFloat(); + float quatY = inputStream.readFloat(); + float quatZ = inputStream.readFloat(); + float quatW = inputStream.readFloat(); + Quaternion quaternion = new Quaternion(quatX, quatY, quatZ, quatW); + + rotations.put(label, quaternion); + } + } + + int positionCount = inputStream.readInt(); + HashMap positions = null; + if (positionCount > 0) { + positions = new HashMap(positionCount); + for (int j = 0; j < positionCount; j++) { + String label = inputStream.readUTF(); - float quatX = inputStream.readFloat(); - float quatY = inputStream.readFloat(); - float quatZ = inputStream.readFloat(); - float quatW = inputStream.readFloat(); - Quaternion quaternion = new Quaternion(quatX, quatY, quatZ, quatW); + float posX = inputStream.readFloat(); + float posY = inputStream.readFloat(); + float posZ = inputStream.readFloat(); + Vector3f position = new Vector3f(posX, posY, posZ); - rotations.put(label, quaternion); + positions.put(label, position); + } } - return new PoseFrame(vector, rotations); + return new PoseFrame(vector, rotations, positions); } catch (Exception e) { LogManager.log.severe("Error reading frame from stream", e); } diff --git a/src/main/java/io/eiren/gui/autobone/SimpleSkeleton.java b/src/main/java/io/eiren/gui/autobone/SimpleSkeleton.java index 07accdbba1..7cc4f3a890 100644 --- a/src/main/java/io/eiren/gui/autobone/SimpleSkeleton.java +++ b/src/main/java/io/eiren/gui/autobone/SimpleSkeleton.java @@ -56,7 +56,7 @@ public class SimpleSkeleton { */ protected float legsLength = 0.84f; - protected final HashMap nodes = new HashMap(); + protected final HashMap nodes = new HashMap(); public SimpleSkeleton() { // Assemble skeleton to waist @@ -142,13 +142,15 @@ public void setPoseFromFrame(PoseFrame frame) { // Copy headset position hmdNode.localTransform.setTranslation(frame.rootPos); - // Copy all rotations - for (Entry rotation : frame.rotations.entrySet()) { - TransformNode targetNode = nodes.get(rotation.getKey()); + if (frame.rotations != null) { + // Copy all rotations + for (Entry rotation : frame.rotations.entrySet()) { + TransformNode targetNode = nodes.get(rotation.getKey()); - // Handle unexpected nodes gracefully - if (targetNode != null) { - targetNode.localTransform.setRotation(rotation.getValue()); + // Handle unexpected nodes gracefully + if (targetNode != null) { + targetNode.localTransform.setRotation(rotation.getValue()); + } } } @@ -257,6 +259,11 @@ public void updatePose() { hmdNode.update(); } + public Vector3f getNodePosition(String node) { + TransformNode transformNode = nodes.get(node); + return transformNode != null ? transformNode.worldTransform.getTranslation() : null; + } + public Vector3f getHMDPos() { return hmdNode.worldTransform.getTranslation(); } From 7992526d2d7d4bf10091609b70ba924fa17ff5c0 Mon Sep 17 00:00:00 2001 From: ButterscotchVanilla Date: Thu, 19 Aug 2021 04:29:14 -0400 Subject: [PATCH 58/60] AutoBone: Rename PoseRecordIO to PoseFrameIO and separate recording load into a method --- .../io/eiren/gui/autobone/AutoBoneWindow.java | 64 +++++++++++-------- .../{PoseRecordIO.java => PoseFrameIO.java} | 4 +- 2 files changed, 38 insertions(+), 30 deletions(-) rename src/main/java/io/eiren/gui/autobone/{PoseRecordIO.java => PoseFrameIO.java} (98%) diff --git a/src/main/java/io/eiren/gui/autobone/AutoBoneWindow.java b/src/main/java/io/eiren/gui/autobone/AutoBoneWindow.java index d6e1f9597c..cdafb7b474 100644 --- a/src/main/java/io/eiren/gui/autobone/AutoBoneWindow.java +++ b/src/main/java/io/eiren/gui/autobone/AutoBoneWindow.java @@ -9,6 +9,7 @@ import javax.swing.border.EmptyBorder; import java.awt.event.MouseEvent; import java.io.File; +import java.util.List; import java.util.Map.Entry; import java.util.concurrent.Future; @@ -25,6 +26,9 @@ public class AutoBoneWindow extends JFrame { + private static File saveDir = new File("Recordings"); + private static File loadDir = new File("LoadRecordings"); + private EJBox pane; private final VRServer server; @@ -68,6 +72,29 @@ private String getLengthsString() { return configInfo.toString(); } + private List> loadRecordings() { + List> recordings = new FastList>(); + if (loadDir.isDirectory()) { + File[] files = loadDir.listFiles(); + if (files != null) { + for (File file : files) { + if (file.isFile() && org.apache.commons.lang3.StringUtils.endsWithIgnoreCase(file.getName(), ".abf")) { + LogManager.log.info("[AutoBone] Detected recording at \"" + file.getPath() + "\", loading frames..."); + PoseFrame[] frames = PoseFrameIO.readFromFile(file); + + if (frames == null) { + LogManager.log.severe("Reading frames from \"" + file.getPath() + "\" failed..."); + } else { + recordings.add(Pair.of(file.getName(), frames)); + } + } + } + } + } + + return recordings; + } + private float processFrames(PoseFrame[] frames) { autoBone.reloadConfigValues(); @@ -122,22 +149,21 @@ public void run() { setText("Saving..."); if (server.config.getBoolean("autobone.saveRecordings", true)) { - File saveFolder = new File("Recordings"); - if (saveFolder.isDirectory() || saveFolder.mkdirs()) { + if (saveDir.isDirectory() || saveDir.mkdirs()) { File saveRecording; int recordingIndex = 1; do { - saveRecording = new File(saveFolder, "ABRecording" + recordingIndex++ + ".abf"); + saveRecording = new File(saveDir, "ABRecording" + recordingIndex++ + ".abf"); } while (saveRecording.exists()); LogManager.log.info("[AutoBone] Exporting frames to \"" + saveRecording.getPath() + "\"..."); - if (PoseRecordIO.writeToFile(saveRecording, frames)) { + if (PoseFrameIO.writeToFile(saveRecording, frames)) { LogManager.log.info("[AutoBone] Done exporting! Recording can be found at \"" + saveRecording.getPath() + "\"."); } else { LogManager.log.severe("[AutoBone] Failed to export the recording to \"" + saveRecording.getPath() + "\"."); } } else { - LogManager.log.severe("[AutoBone] Failed to create the recording directory \"" + saveFolder.getPath() + "\"."); + LogManager.log.severe("[AutoBone] Failed to create the recording directory \"" + saveDir.getPath() + "\"."); } } } else { @@ -180,28 +206,8 @@ public void mouseClicked(MouseEvent e) { @Override public void run() { try { - FastList> frameRecordings = new FastList>(); - - File loadFolder = new File("LoadRecordings"); - if (loadFolder.isDirectory()) { - setText("Load..."); - - File[] files = loadFolder.listFiles(); - if (files != null) { - for (File file : files) { - if (file.isFile() && org.apache.commons.lang3.StringUtils.endsWithIgnoreCase(file.getName(), ".abf")) { - LogManager.log.info("[AutoBone] Detected recording at \"" + file.getPath() + "\", loading frames..."); - PoseFrame[] frames = PoseRecordIO.readFromFile(file); - - if (frames == null) { - LogManager.log.severe("Reading frames from \"" + file.getPath() + "\" failed..."); - } else { - frameRecordings.add(Pair.of(file.getName(), frames)); - } - } - } - } - } + setText("Load..."); + List> frameRecordings = loadRecordings(); if (frameRecordings.size() > 0) { LogManager.log.info("[AutoBone] Done loading frames!"); @@ -218,7 +224,7 @@ public void run() { frameRecordings.add(Pair.of("", frames)); } else { setText("No Recordings..."); - LogManager.log.severe("[AutoBone] No recordings found in \"" + loadFolder.getPath() + "\" and no recording was done..."); + LogManager.log.severe("[AutoBone] No recordings found in \"" + loadDir.getPath() + "\" and no recording was done..."); try { Thread.sleep(3000); // Wait for 3 seconds } catch (Exception e1) { @@ -316,10 +322,12 @@ public void mouseClicked(MouseEvent e) { }}); pane.add(new EJBox(BoxLayout.LINE_AXIS) {{ + setBorder(new EmptyBorder(i(5))); add(processLabel = new JLabel("Processing has not been started...")); }}); pane.add(new EJBox(BoxLayout.LINE_AXIS) {{ + setBorder(new EmptyBorder(i(5))); add(lengthsLabel = new JLabel(getLengthsString())); }}); diff --git a/src/main/java/io/eiren/gui/autobone/PoseRecordIO.java b/src/main/java/io/eiren/gui/autobone/PoseFrameIO.java similarity index 98% rename from src/main/java/io/eiren/gui/autobone/PoseRecordIO.java rename to src/main/java/io/eiren/gui/autobone/PoseFrameIO.java index 3fcfb81ddf..56cdf98754 100644 --- a/src/main/java/io/eiren/gui/autobone/PoseRecordIO.java +++ b/src/main/java/io/eiren/gui/autobone/PoseFrameIO.java @@ -15,9 +15,9 @@ import io.eiren.util.logging.LogManager; -public final class PoseRecordIO { +public final class PoseFrameIO { - private PoseRecordIO() { + private PoseFrameIO() { // Do not allow instantiating } From 3b61f8834370a72a5b7de66a44a8768086bf502e Mon Sep 17 00:00:00 2001 From: ButterscotchVanilla Date: Thu, 19 Aug 2021 18:14:57 -0400 Subject: [PATCH 59/60] AutoBone: Add save recording button & enable buttons as needed --- .../io/eiren/gui/autobone/AutoBoneWindow.java | 121 ++++++++++++++---- .../io/eiren/gui/autobone/PoseRecorder.java | 8 +- 2 files changed, 104 insertions(+), 25 deletions(-) diff --git a/src/main/java/io/eiren/gui/autobone/AutoBoneWindow.java b/src/main/java/io/eiren/gui/autobone/AutoBoneWindow.java index cdafb7b474..75a1d9841f 100644 --- a/src/main/java/io/eiren/gui/autobone/AutoBoneWindow.java +++ b/src/main/java/io/eiren/gui/autobone/AutoBoneWindow.java @@ -37,8 +37,13 @@ public class AutoBoneWindow extends JFrame { private final AutoBone autoBone; private Thread recordingThread = null; + private Thread saveRecordingThread = null; private Thread autoBoneThread = null; + private JButton saveRecordingButton; + private JButton adjustButton; + private JButton applyButton; + private JLabel processLabel; private JLabel lengthsLabel; @@ -72,6 +77,25 @@ private String getLengthsString() { return configInfo.toString(); } + private void saveRecording(PoseFrame[] frames) { + if (saveDir.isDirectory() || saveDir.mkdirs()) { + File saveRecording; + int recordingIndex = 1; + do { + saveRecording = new File(saveDir, "ABRecording" + recordingIndex++ + ".abf"); + } while (saveRecording.exists()); + + LogManager.log.info("[AutoBone] Exporting frames to \"" + saveRecording.getPath() + "\"..."); + if (PoseFrameIO.writeToFile(saveRecording, frames)) { + LogManager.log.info("[AutoBone] Done exporting! Recording can be found at \"" + saveRecording.getPath() + "\"."); + } else { + LogManager.log.severe("[AutoBone] Failed to export the recording to \"" + saveRecording.getPath() + "\"."); + } + } else { + LogManager.log.severe("[AutoBone] Failed to create the recording directory \"" + saveDir.getPath() + "\"."); + } + } + private List> loadRecordings() { List> recordings = new FastList>(); if (loadDir.isDirectory()) { @@ -130,7 +154,7 @@ private void build() { @Override public void mouseClicked(MouseEvent e) { // Prevent running multiple times - if (recordingThread != null) { + if (!isEnabled() || recordingThread != null) { return; } @@ -147,24 +171,12 @@ public void run() { PoseFrame[] frames = framesFuture.get(); LogManager.log.info("[AutoBone] Done recording!"); - setText("Saving..."); - if (server.config.getBoolean("autobone.saveRecordings", true)) { - if (saveDir.isDirectory() || saveDir.mkdirs()) { - File saveRecording; - int recordingIndex = 1; - do { - saveRecording = new File(saveDir, "ABRecording" + recordingIndex++ + ".abf"); - } while (saveRecording.exists()); - - LogManager.log.info("[AutoBone] Exporting frames to \"" + saveRecording.getPath() + "\"..."); - if (PoseFrameIO.writeToFile(saveRecording, frames)) { - LogManager.log.info("[AutoBone] Done exporting! Recording can be found at \"" + saveRecording.getPath() + "\"."); - } else { - LogManager.log.severe("[AutoBone] Failed to export the recording to \"" + saveRecording.getPath() + "\"."); - } - } else { - LogManager.log.severe("[AutoBone] Failed to create the recording directory \"" + saveDir.getPath() + "\"."); - } + saveRecordingButton.setEnabled(true); + adjustButton.setEnabled(true); + + if (server.config.getBoolean("autobone.saveRecordings", false)) { + setText("Saving..."); + saveRecording(frames); } } else { setText("Not Ready..."); @@ -193,12 +205,70 @@ public void run() { }); }}); - add(new JButton("Auto-Adjust") {{ + add(saveRecordingButton = new JButton("Save Recording") {{ + setEnabled(poseRecorder.hasRecording()); addMouseListener(new MouseInputAdapter() { @Override public void mouseClicked(MouseEvent e) { // Prevent running multiple times - if (autoBoneThread != null) { + if (!isEnabled() || saveRecordingThread != null) { + return; + } + + Thread thread = new Thread() { + @Override + public void run() { + try { + Future framesFuture = poseRecorder.getFramesAsync(); + if (framesFuture != null) { + setText("Waiting for Recording..."); + PoseFrame[] frames = framesFuture.get(); + + if (frames.length <= 0) { + throw new IllegalStateException("Recording has no frames"); + } + + setText("Saving..."); + saveRecording(frames); + } else { + setText("No Recording..."); + LogManager.log.severe("[AutoBone] Unable to save, no recording was done..."); + try { + Thread.sleep(3000); // Wait for 3 seconds + } catch (Exception e1) { + // Ignore + } + return; + } + } catch (Exception e) { + setText("Saving Failed..."); + LogManager.log.severe("[AutoBone] Failed to save recording!", e); + try { + Thread.sleep(3000); // Wait for 3 seconds + } catch (Exception e1) { + // Ignore + } + } finally { + setText("Save Recording"); + saveRecordingThread = null; + } + } + }; + + saveRecordingThread = thread; + thread.start(); + } + }); + }}); + + add(adjustButton = new JButton("Auto-Adjust") {{ + // If there are files to load, enable the button + setEnabled(poseRecorder.hasRecording() || (loadDir.isDirectory() && loadDir.list().length > 0)); + addMouseListener(new MouseInputAdapter() { + @Override + public void mouseClicked(MouseEvent e) { + // Prevent running multiple times + if (!isEnabled() || autoBoneThread != null) { return; } @@ -242,6 +312,7 @@ public void run() { heightPercentError.add(processFrames(recording.getValue())); LogManager.log.info("[AutoBone] Done processing!"); + applyButton.setEnabled(true); //#region Stats/Values Float neckLength = autoBone.getConfig("Neck"); @@ -308,12 +379,16 @@ public void run() { }); }}); - add(new JButton("Apply Values") {{ + add(applyButton = new JButton("Apply Values") {{ + setEnabled(false); addMouseListener(new MouseInputAdapter() { @Override public void mouseClicked(MouseEvent e) { - autoBone.applyConfig(); + if (!isEnabled()) { + return; + } + autoBone.applyConfig(); // Update GUI values after applying skeletonConfig.refreshAll(); } diff --git a/src/main/java/io/eiren/gui/autobone/PoseRecorder.java b/src/main/java/io/eiren/gui/autobone/PoseRecorder.java index dc2fb8c357..0d272d445d 100644 --- a/src/main/java/io/eiren/gui/autobone/PoseRecorder.java +++ b/src/main/java/io/eiren/gui/autobone/PoseRecorder.java @@ -105,12 +105,16 @@ public synchronized void cancelFrameRecording() { numFrames = -1; } + public boolean isReadyToRecord() { + return skeleton != null; + } + public boolean isRecording() { return numFrames > frames.size(); } - public boolean isReadyToRecord() { - return skeleton != null; + public boolean hasRecording() { + return currentRecording != null; } public Future getFramesAsync() { From 337912f3d0eddcffd821e83be1bd8a5fc200db18 Mon Sep 17 00:00:00 2001 From: ButterscotchVanilla Date: Thu, 19 Aug 2021 22:46:41 -0400 Subject: [PATCH 60/60] AutoBone: Optimize PoseRecorder tick --- .../io/eiren/gui/autobone/PoseRecorder.java | 34 ++++++++++++------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/src/main/java/io/eiren/gui/autobone/PoseRecorder.java b/src/main/java/io/eiren/gui/autobone/PoseRecorder.java index 0d272d445d..5af0ba1866 100644 --- a/src/main/java/io/eiren/gui/autobone/PoseRecorder.java +++ b/src/main/java/io/eiren/gui/autobone/PoseRecorder.java @@ -33,21 +33,23 @@ public PoseRecorder(VRServer server) { @VRServerThread public void onTick() { - HumanSkeletonWithLegs skeleton = this.skeleton; - if (skeleton != null) { - if (frames.size() < numFrames) { - if (System.currentTimeMillis() >= nextFrameTimeMs) { - nextFrameTimeMs = System.currentTimeMillis() + frameRecordingInterval; - frames.add(new PoseFrame(skeleton)); - - // If done, send finished recording - if (frames.size() >= numFrames) { - internalStopRecording(); + if (numFrames > 0) { + HumanSkeletonWithLegs skeleton = this.skeleton; + if (skeleton != null) { + if (frames.size() < numFrames) { + if (System.currentTimeMillis() >= nextFrameTimeMs) { + nextFrameTimeMs = System.currentTimeMillis() + frameRecordingInterval; + frames.add(new PoseFrame(skeleton)); + + // If done, send finished recording + if (frames.size() >= numFrames) { + internalStopRecording(); + } } + } else { + // If done and hasn't yet, send finished recording + internalStopRecording(); } - } else { - // If done and hasn't yet, send finished recording - internalStopRecording(); } } } @@ -60,6 +62,12 @@ public void skeletonUpdated(HumanSkeleton newSkeleton) { } public synchronized Future startFrameRecording(int numFrames, long interval) { + if (numFrames < 1) { + throw new IllegalArgumentException("numFrames must at least have a value of 1"); + } + if (interval < 1) { + throw new IllegalArgumentException("interval must at least have a value of 1"); + } if (!isReadyToRecord()) { throw new IllegalStateException("PoseRecorder isn't ready to record!"); }