diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index 83f5da4368..2418c459bb 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -3,11 +3,7 @@ name: SlimeVR Server -on: - push: - branches: [ main ] - pull_request: - branches: [ main ] +on: [push, pull_request] jobs: test: diff --git a/build.gradle b/build.gradle index 73afa976c7..3f8c75b1bd 100644 --- a/build.gradle +++ b/build.gradle @@ -11,6 +11,24 @@ plugins { id 'java-library' } +sourceCompatibility = 1.8 +targetCompatibility = 1.8 + +// Set compiler to use UTF-8 +compileJava.options.encoding = 'UTF-8' +compileTestJava.options.encoding = 'UTF-8' +javadoc.options.encoding = 'UTF-8' + +tasks.withType(JavaCompile) { + options.encoding = 'UTF-8' +} +tasks.withType(Test) { + systemProperty('file.encoding', 'UTF-8') +} +tasks.withType(Javadoc){ + options.encoding = 'UTF-8' +} + repositories { // Use jcenter for resolving dependencies. // You can declare any Maven/Ivy/file repository here. @@ -31,7 +49,7 @@ dependencies { // This dependency is used internally, and not exposed to consumers on their own compile classpath. implementation 'com.google.guava:guava:28.2-jre' - + // Use JUnit test framework testImplementation platform('org.junit:junit-bom:5.7.2') @@ -48,7 +66,7 @@ task serverJar (type: Jar, dependsOn: subprojects.tasks['build']) { manifest { attributes 'Main-Class': 'io.eiren.vr.Main' } - + // Pack all dependencies within the JAR from { configurations.compile.collect { it.isDirectory() ? it : zipTree(it) } diff --git a/src/main/java/io/eiren/gui/SkeletonConfig.java b/src/main/java/io/eiren/gui/SkeletonConfig.java index fa1f518781..e01fea9d9e 100644 --- a/src/main/java/io/eiren/gui/SkeletonConfig.java +++ b/src/main/java/io/eiren/gui/SkeletonConfig.java @@ -11,6 +11,7 @@ import javax.swing.JLabel; import javax.swing.event.MouseInputAdapter; +import io.eiren.gui.autobone.AutoBoneWindow; import io.eiren.util.StringUtils; import io.eiren.util.ann.ThreadSafe; import io.eiren.vr.VRServer; @@ -18,26 +19,28 @@ import io.eiren.vr.processor.HumanSkeleton; public class SkeletonConfig extends EJBag { - + private final VRServer server; private final VRServerGUI gui; + private final AutoBoneWindow autoBone; private Map labels = new HashMap<>(); - + public SkeletonConfig(VRServer server, VRServerGUI gui) { super(); this.server = server; this.gui = gui; + this.autoBone = new AutoBoneWindow(server, this); setAlignmentY(TOP_ALIGNMENT); server.humanPoseProcessor.addSkeletonUpdatedCallback(this::skeletonUpdated); skeletonUpdated(null); } - + @ThreadSafe public void skeletonUpdated(HumanSkeleton newSkeleton) { java.awt.EventQueue.invokeLater(() -> { removeAll(); - + int row = 0; add(new JCheckBox("Extended pelvis model") {{ @@ -63,7 +66,7 @@ public void itemStateChanged(ItemEvent e) { } }}, s(c(0, row, 1), 3, 1)); row++; - + /* add(new JCheckBox("Extended knee model") {{ addItemListener(new ItemListener() { @@ -89,17 +92,26 @@ public void itemStateChanged(ItemEvent e) { }}, s(c(0, row, 1), 3, 1)); row++; //*/ - + add(new TimedResetButton("Reset All", "All"), s(c(1, row, 1), 3, 1)); + add(new JButton("Auto") {{ + addMouseListener(new MouseInputAdapter() { + @Override + public void mouseClicked(MouseEvent e) { + autoBone.setVisible(true); + autoBone.toFront(); + } + }); + }}, s(c(4, row, 1), 3, 1)); row++; - + add(new JLabel("Chest"), c(0, row, 1)); add(new AdjButton("+", "Chest", 0.01f), c(1, row, 1)); add(new SkeletonLabel("Chest"), c(2, row, 1)); add(new AdjButton("-", "Chest", -0.01f), c(3, row, 1)); add(new ResetButton("Reset", "Chest"), c(4, row, 1)); row++; - + add(new JLabel("Waist"), c(0, row, 1)); add(new AdjButton("+", "Waist", 0.01f), c(1, row, 1)); add(new SkeletonLabel("Waist"), c(2, row, 1)); @@ -148,14 +160,14 @@ public void itemStateChanged(ItemEvent e) { add(new AdjButton("-", "Neck", -0.01f), c(3, row, 1)); add(new ResetButton("Reset", "Neck"), c(4, row, 1)); row++; - + add(new JLabel("Virtual waist"), c(0, row, 1)); add(new AdjButton("+", "Virtual waist", 0.01f), c(1, row, 1)); add(new SkeletonLabel("Virtual waist"), c(2, row, 1)); add(new AdjButton("-", "Virtual waist", -0.01f), c(3, row, 1)); add(new ResetButton("Reset", "Virtual waist"), c(4, row, 1)); row++; - + gui.refresh(); }); } @@ -168,14 +180,14 @@ public void refreshAll() { }); }); } - + private void change(String joint, float diff) { float current = server.humanPoseProcessor.getSkeletonConfig(joint); server.humanPoseProcessor.setSkeletonConfig(joint, current + diff); server.saveConfig(); labels.get(joint).setText(StringUtils.prettyNumber((current + diff) * 100, 0)); } - + private void reset(String joint) { server.humanPoseProcessor.resetSkeletonConfig(joint); server.saveConfig(); @@ -189,17 +201,17 @@ private void reset(String joint) { }); } } - + private class SkeletonLabel extends JLabel { - + public SkeletonLabel(String joint) { super(StringUtils.prettyNumber(server.humanPoseProcessor.getSkeletonConfig(joint) * 100, 0)); labels.put(joint, this); } } - + private class AdjButton extends JButton { - + public AdjButton(String text, String joint, float diff) { super(text); addMouseListener(new MouseInputAdapter() { @@ -210,9 +222,9 @@ public void mouseClicked(MouseEvent e) { }); } } - + private class ResetButton extends JButton { - + public ResetButton(String text, String joint) { super(text); addMouseListener(new MouseInputAdapter() { @@ -223,9 +235,9 @@ public void mouseClicked(MouseEvent e) { }); } } - + private class TimedResetButton extends JButton { - + public TimedResetButton(String text, String joint) { super(text); addMouseListener(new MouseInputAdapter() { diff --git a/src/main/java/io/eiren/gui/autobone/AutoBone.java b/src/main/java/io/eiren/gui/autobone/AutoBone.java new file mode 100644 index 0000000000..a7c3cffa87 --- /dev/null +++ b/src/main/java/io/eiren/gui/autobone/AutoBone.java @@ -0,0 +1,512 @@ +package io.eiren.gui.autobone; + +import java.util.HashMap; +import java.util.Map; +import java.util.Map.Entry; +import java.util.function.Consumer; + +import com.jme3.math.Vector3f; + +import io.eiren.util.ann.ThreadSafe; +import io.eiren.util.logging.LogManager; +import io.eiren.util.collections.FastList; +import io.eiren.vr.VRServer; +import io.eiren.vr.processor.HumanSkeleton; +import io.eiren.vr.processor.HumanSkeletonWithLegs; +import io.eiren.vr.processor.HumanSkeletonWithWaist; +import io.eiren.vr.processor.TrackerBodyPosition; +import io.eiren.vr.trackers.TrackerUtils; + +public class AutoBone { + + public class Epoch { + + public final int epoch; + public final float epochError; + + public Epoch(int epoch, float epochError) { + this.epoch = epoch; + this.epochError = epochError; + } + + @Override + public String toString() { + return "Epoch: " + epoch + ", Epoch Error: " + epochError; + } + } + + public int cursorIncrement = 1; + + public int minDataDistance = 2; + public int maxDataDistance = 32; + + public int numEpochs = 5; + + public float initialAdjustRate = 2.5f; + public float adjustRateDecay = 1.01f; + + public float slideErrorFactor = 1.0f; + public float offsetErrorFactor = 0.0f; + public float proportionErrorFactor = 0.2f; + public float heightErrorFactor = 0.1f; + public float positionErrorFactor = 0.0f; + public float positionOffsetErrorFactor = 0.0f; + + /* + public float NECK_WAIST_RATIO_MIN = 0.2f; + public float NECK_WAIST_RATIO_MAX = 0.3f; + + public float CHEST_WAIST_RATIO_MIN = 0.35f; + public float CHEST_WAIST_RATIO_MAX = 0.6f; + + public float HIP_MIN = 0.08f; + public float HIP_WAIST_RATIO_MAX = 0.4f; + + // Human average is 1.1235 (SD 0.07) + public float LEG_WAIST_RATIO_MIN = 1.1235f - ((0.07f * 3f) + 0.05f); + public float LEG_WAIST_RATIO_MAX = 1.1235f + ((0.07f * 3f) + 0.05f); + + public float KNEE_LEG_RATIO_MIN = 0.42f; + public float KNEE_LEG_RATIO_MAX = 0.58f; + */ + + protected final VRServer server; + + protected HumanSkeletonWithLegs skeleton = null; + + // This is filled by reloadConfigValues() + public final HashMap configs = new HashMap(); + public final HashMap staticConfigs = new HashMap(); + + public final FastList heightConfigs = new FastList(new String[] { + "Neck", + "Waist", + "Legs length" + }); + + public AutoBone(VRServer server) { + this.server = server; + + reloadConfigValues(); + + server.addSkeletonUpdatedCallback(this::skeletonUpdated); + } + + public void reloadConfigValues() { + // Load waist configs + staticConfigs.put("Head", server.config.getFloat("body.headShift", HumanSkeletonWithWaist.HEAD_SHIFT_DEFAULT)); + staticConfigs.put("Neck", server.config.getFloat("body.neckLength", HumanSkeletonWithWaist.NECK_LENGTH_DEFAULT)); + configs.put("Waist", server.config.getFloat("body.waistDistance", 0.85f)); + + if (server.config.getBoolean("autobone.forceChestTracker", false) || + TrackerUtils.findTrackerForBodyPosition(server.getAllTrackers(), TrackerBodyPosition.CHEST) != null) { + // If force enabled or has a chest tracker + configs.put("Chest", server.config.getFloat("body.chestDistance", 0.42f)); + } else { + // Otherwise, make sure it's not used + configs.remove("Chest"); + staticConfigs.put("Chest", server.config.getFloat("body.chestDistance", 0.42f)); + } + + // Load leg configs + staticConfigs.put("Hips width", server.config.getFloat("body.hipsWidth", HumanSkeletonWithLegs.HIPS_WIDTH_DEFAULT)); + configs.put("Legs length", server.config.getFloat("body.legsLength", 0.84f)); + configs.put("Knee height", server.config.getFloat("body.kneeHeight", 0.42f)); + } + + @ThreadSafe + public void skeletonUpdated(HumanSkeleton newSkeleton) { + if (newSkeleton instanceof HumanSkeletonWithLegs) { + skeleton = (HumanSkeletonWithLegs)newSkeleton; + applyConfigToSkeleton(newSkeleton); + LogManager.log.info("[AutoBone] Received updated skeleton"); + } + } + + public void applyConfig() { + if (!applyConfigToSkeleton(skeleton)) { + // Unable to apply to skeleton, save directly + saveConfigs(); + } + } + + public boolean applyConfigToSkeleton(HumanSkeleton skeleton) { + if (skeleton == null) { + return false; + } + + for (Entry entry : configs.entrySet()) { + skeleton.setSkeletonConfig(entry.getKey(), entry.getValue()); + } + + server.saveConfig(); + + LogManager.log.info("[AutoBone] Configured skeleton bone lengths"); + return true; + } + + private void setConfig(String name, String path) { + Float value = configs.get(name); + if (value != null) { + server.config.setProperty(path, value); + } + } + + // This doesn't require a skeleton, therefore can be used if skeleton is null + public void saveConfigs() { + setConfig("Head", "body.headShift"); + setConfig("Neck", "body.neckLength"); + setConfig("Waist", "body.waistDistance"); + setConfig("Chest", "body.chestDistance"); + setConfig("Hips width", "body.hipsWidth"); + setConfig("Legs length", "body.legsLength"); + setConfig("Knee height", "body.kneeHeight"); + + server.saveConfig(); + } + + public Float getConfig(String config) { + Float configVal = configs.get(config); + return configVal != null ? configVal : staticConfigs.get(config); + } + + public Float getConfig(String config, Map configs, Map configsAlt) { + if (configs == null) { + throw new NullPointerException("Argument \"configs\" must not be null"); + } + + Float configVal = configs.get(config); + return configVal != null || configsAlt == null ? configVal : configsAlt.get(config); + } + + public float getHeight(Map configs) { + return getHeight(configs, null); + } + + public float getHeight(Map configs, Map configsAlt) { + float height = 0f; + + for (String heightConfig : heightConfigs) { + Float length = getConfig(heightConfig, configs, configsAlt); + if (length != null) { + height += length; + } + } + + return height; + } + + public float getLengthSum(Map configs) { + float length = 0f; + + for (float boneLength : configs.values()) { + length += boneLength; + } + + return length; + } + + public float getMaxHmdHeight(PoseFrame[] frames) { + float maxHeight = 0f; + for (PoseFrame frame : frames) { + if (frame.rootPos.y > maxHeight) { + maxHeight = frame.rootPos.y; + } + } + return maxHeight; + } + + public void processFrames(PoseFrame[] frames) { + processFrames(frames, -1f); + } + + public void processFrames(PoseFrame[] frames, Consumer epochCallback) { + processFrames(frames, -1f, epochCallback); + } + + public void processFrames(PoseFrame[] frames, float targetHeight) { + processFrames(frames, true, targetHeight); + } + + public void processFrames(PoseFrame[] frames, float targetHeight, Consumer epochCallback) { + processFrames(frames, true, targetHeight, epochCallback); + } + + public float processFrames(PoseFrame[] frames, boolean calcInitError, float targetHeight) { + return processFrames(frames, calcInitError, targetHeight, null); + } + + public float processFrames(PoseFrame[] frames, boolean calcInitError, float targetHeight, Consumer epochCallback) { + SimpleSkeleton skeleton1 = new SimpleSkeleton(configs, staticConfigs); + SimpleSkeleton skeleton2 = new SimpleSkeleton(configs, staticConfigs); + + // If target height isn't specified, auto-detect + if (targetHeight < 0f) { + if (skeleton != null) { + targetHeight = getHeight(skeleton.getSkeletonConfig()); + LogManager.log.warning("[AutoBone] Target height loaded from skeleton (Make sure you reset before running!): " + targetHeight); + } else { + float hmdHeight = getMaxHmdHeight(frames); + if (hmdHeight <= 0.50f) { + LogManager.log.warning("[AutoBone] Max headset height detected (Value seems too low, did you not stand up straight while measuring?): " + hmdHeight); + } else { + LogManager.log.info("[AutoBone] Max headset height detected: " + hmdHeight); + } + + // Estimate target height from HMD height + targetHeight = hmdHeight; + } + } + + for (int epoch = calcInitError ? -1 : 0; epoch < numEpochs; epoch++) { + float sumError = 0f; + int errorCount = 0; + + float adjustRate = epoch >= 0 ? (float)(initialAdjustRate / Math.pow(adjustRateDecay, epoch)) : 0f; + + for (int cursorOffset = minDataDistance; cursorOffset <= maxDataDistance && cursorOffset < frames.length; cursorOffset++) { + for (int frameCursor = 0; frameCursor < frames.length - cursorOffset; frameCursor += cursorIncrement) { + PoseFrame frame1 = frames[frameCursor]; + PoseFrame frame2 = frames[frameCursor + cursorOffset]; + + // If there's missing data, throw an exception + if (frame1 == null || frame2 == null) { + throw new NullPointerException("Frames are missing from processing data"); + } + + skeleton1.setSkeletonConfigs(configs); + skeleton2.setSkeletonConfigs(configs); + + skeleton1.setPoseFromFrame(frame1); + skeleton2.setPoseFromFrame(frame2); + + float totalLength = getLengthSum(configs); + float curHeight = getHeight(configs, staticConfigs); + float errorDeriv = getErrorDeriv(frame1, frame2, skeleton1, skeleton2, targetHeight - curHeight); + float error = errorFunc(errorDeriv); + + // In case of fire + if (Float.isNaN(error) || Float.isInfinite(error)) { + // Extinguish + LogManager.log.warning("[AutoBone] Error value is invalid, resetting variables to recover"); + reloadConfigValues(); + + // Reset error sum values + sumError = 0f; + errorCount = 0; + + // Continue on new data + continue; + } + + // Store the error count for logging purposes + sumError += errorDeriv; + errorCount++; + + float adjustVal = error * adjustRate; + + for (Entry entry : configs.entrySet()) { + // Skip adjustment if the epoch is before starting (for logging only) + if (epoch < 0) { + break; + } + + float originalLength = entry.getValue(); + + // Try positive and negative adjustments + boolean isHeightVar = heightConfigs.contains(entry.getKey()); + float minError = errorDeriv; + float finalNewLength = -1f; + for (int i = 0; i < 2; i++) { + // Scale by the ratio for smooth adjustment and more stable results + float curAdjustVal = ((i == 0 ? adjustVal : -adjustVal) * originalLength) / totalLength; + float newLength = originalLength + curAdjustVal; + + // No small or negative numbers!!! Bad algorithm! + if (newLength < 0.01f) { + continue; + } + + updateSkeletonBoneLength(skeleton1, skeleton2, entry.getKey(), newLength); + + float newHeight = isHeightVar ? curHeight + curAdjustVal : curHeight; + float newErrorDeriv = getErrorDeriv(frame1, frame2, skeleton1, skeleton2, targetHeight - newHeight); + + if (newErrorDeriv < minError) { + minError = newErrorDeriv; + finalNewLength = newLength; + } + } + + if (finalNewLength > 0f) { + entry.setValue(finalNewLength); + } + + // Reset the length to minimize bias in other variables, it's applied later + updateSkeletonBoneLength(skeleton1, skeleton2, entry.getKey(), originalLength); + } + } + } + + // Calculate average error over the epoch + float avgError = errorCount > 0 ? sumError / errorCount : -1f; + LogManager.log.info("[AutoBone] Epoch " + (epoch + 1) + " average error: " + avgError); + + if (epochCallback != null) { + epochCallback.accept(new Epoch(epoch + 1, avgError)); + } + } + + float finalHeight = getHeight(configs, staticConfigs); + LogManager.log.info("[AutoBone] Target height: " + targetHeight + " New height: " + finalHeight); + + return Math.abs(finalHeight - targetHeight); + } + + // The change in position of the ankle over time + protected float getSlideErrorDeriv(SimpleSkeleton skeleton1, SimpleSkeleton skeleton2) { + float slideLeft = skeleton1.getLeftFootPos().distance(skeleton2.getLeftFootPos()); + float slideRight = skeleton1.getRightFootPos().distance(skeleton2.getRightFootPos()); + + // Divide by 4 to halve and average, it's halved because you want to approach a midpoint, not the other point + return (slideLeft + slideRight) / 4f; + } + + // The offset between both feet at one instant and over time + protected float getOffsetErrorDeriv(SimpleSkeleton skeleton1, SimpleSkeleton skeleton2) { + float dist1 = Math.abs(skeleton1.getLeftFootPos().y - skeleton1.getRightFootPos().y); + float dist2 = Math.abs(skeleton2.getLeftFootPos().y - skeleton2.getRightFootPos().y); + + float dist3 = Math.abs(skeleton1.getLeftFootPos().y - skeleton2.getRightFootPos().y); + float dist4 = Math.abs(skeleton1.getLeftFootPos().y - skeleton2.getRightFootPos().y); + + float dist5 = Math.abs(skeleton1.getLeftFootPos().y - skeleton2.getLeftFootPos().y); + float dist6 = Math.abs(skeleton1.getRightFootPos().y - skeleton2.getRightFootPos().y); + + // Divide by 12 to halve and average, it's halved because you want to approach a midpoint, not the other point + return (dist1 + dist2 + dist3 + dist4 + dist5 + dist6) / 12f; + } + + // The distance from average human proportions + protected float getProportionErrorDeriv(SimpleSkeleton skeleton) { + Float neckLength = skeleton.getSkeletonConfig("Neck"); + Float chestLength = skeleton.getSkeletonConfig("Chest"); + Float waistLength = skeleton.getSkeletonConfig("Waist"); + Float legsLength = skeleton.getSkeletonConfig("Legs length"); + Float kneeHeight = skeleton.getSkeletonConfig("Knee height"); + + float chestWaist = chestLength != null && waistLength != null ? Math.abs((chestLength / waistLength) - 0.5f) : 0f; + float legBody = legsLength != null && waistLength != null && neckLength != null ? Math.abs((legsLength / (waistLength + neckLength)) - 1.1235f) : 0f; + float kneeLeg = kneeHeight != null && legsLength != null ? Math.abs((kneeHeight / legsLength) - 0.5f) : 0f; + + // SD of 0.07, capture 68% within range + float sdValue = 0.07f; + if (legBody <= sdValue) { + legBody = 0f; + } else { + legBody -= sdValue; + } + + return (chestWaist + legBody + kneeLeg) / 3f; + } + + // The distance of any points to the corresponding absolute position + protected float getPositionErrorDeriv(PoseFrame frame, SimpleSkeleton skeleton) { + float offset = 0f; + int offsetCount = 0; + + if (frame.positions != null) { + for (Entry entry : frame.positions.entrySet()) { + Vector3f nodePos = skeleton.getNodePosition(entry.getKey()); + if (nodePos != null) { + offset += Math.abs(nodePos.distance(entry.getValue())); + offsetCount++; + } + } + } + + return offsetCount > 0 ? offset / offsetCount : 0f; + } + + // The difference between offset of absolute position and the corresponding point over time + protected float getPositionOffsetErrorDeriv(PoseFrame frame1, PoseFrame frame2, SimpleSkeleton skeleton1, SimpleSkeleton skeleton2) { + float offset = 0f; + int offsetCount = 0; + + if (frame1.positions != null && frame2.positions != null) { + for (Entry entry : frame1.positions.entrySet()) { + Vector3f frame2Pos = frame2.positions.get(entry.getKey()); + if (frame2Pos == null) { + continue; + } + + Vector3f nodePos1 = skeleton1.getNodePosition(entry.getKey()); + if (nodePos1 == null) { + continue; + } + + Vector3f nodePos2 = skeleton2.getNodePosition(entry.getKey()); + if (nodePos2 == null) { + continue; + } + + float dist1 = Math.abs(nodePos1.distance(entry.getValue())); + float dist2 = Math.abs(nodePos2.distance(frame2Pos)); + + offset += Math.abs(dist2 - dist1); + offsetCount++; + } + } + + return offsetCount > 0 ? offset / offsetCount : 0f; + } + + protected float getErrorDeriv(PoseFrame frame1, PoseFrame frame2, SimpleSkeleton skeleton1, SimpleSkeleton skeleton2, float heightChange) { + float totalError = 0f; + float sumWeight = 0f; + + if (slideErrorFactor > 0f) { + totalError += getSlideErrorDeriv(skeleton1, skeleton2) * slideErrorFactor; + sumWeight += slideErrorFactor; + } + + if (offsetErrorFactor > 0f) { + totalError += getOffsetErrorDeriv(skeleton1, skeleton2) * offsetErrorFactor; + sumWeight += offsetErrorFactor; + } + + if (proportionErrorFactor > 0f) { + // Either skeleton will work fine, skeleton1 is used as a default + totalError += getProportionErrorDeriv(skeleton1) * proportionErrorFactor; + sumWeight += proportionErrorFactor; + } + + if (heightErrorFactor > 0f) { + totalError += Math.abs(heightChange) * heightErrorFactor; + sumWeight += heightErrorFactor; + } + + if (positionErrorFactor > 0f) { + totalError += (getPositionErrorDeriv(frame1, skeleton1) + getPositionErrorDeriv(frame2, skeleton2) / 2f) * positionErrorFactor; + sumWeight += positionErrorFactor; + } + + if (positionOffsetErrorFactor > 0f) { + totalError += getPositionOffsetErrorDeriv(frame1, frame2, skeleton1, skeleton2) * positionOffsetErrorFactor; + sumWeight += positionOffsetErrorFactor; + } + + // Minimize sliding, minimize foot height offset, minimize change in total height + return sumWeight > 0f ? totalError / sumWeight : 0f; + } + + // Mean square error function + protected static float errorFunc(float errorDeriv) { + return 0.5f * (errorDeriv * errorDeriv); + } + + protected void updateSkeletonBoneLength(SimpleSkeleton skeleton1, SimpleSkeleton skeleton2, String joint, float newLength) { + skeleton1.setSkeletonConfig(joint, newLength, true); + skeleton2.setSkeletonConfig(joint, newLength, true); + } +} diff --git a/src/main/java/io/eiren/gui/autobone/AutoBoneWindow.java b/src/main/java/io/eiren/gui/autobone/AutoBoneWindow.java new file mode 100644 index 0000000000..75a1d9841f --- /dev/null +++ b/src/main/java/io/eiren/gui/autobone/AutoBoneWindow.java @@ -0,0 +1,414 @@ +package io.eiren.gui.autobone; + +import javax.swing.BoxLayout; +import javax.swing.JFrame; +import javax.swing.JLabel; +import javax.swing.JScrollPane; +import javax.swing.ScrollPaneConstants; +import javax.swing.JButton; +import javax.swing.border.EmptyBorder; +import java.awt.event.MouseEvent; +import java.io.File; +import java.util.List; +import java.util.Map.Entry; +import java.util.concurrent.Future; + +import io.eiren.gui.EJBox; +import io.eiren.gui.SkeletonConfig; +import io.eiren.util.StringUtils; +import io.eiren.util.ann.AWTThread; +import io.eiren.util.collections.FastList; +import io.eiren.util.logging.LogManager; +import io.eiren.vr.VRServer; +import javax.swing.event.MouseInputAdapter; + +import org.apache.commons.lang3.tuple.Pair; + +public class AutoBoneWindow extends JFrame { + + private static File saveDir = new File("Recordings"); + private static File loadDir = new File("LoadRecordings"); + + private EJBox pane; + + private final VRServer server; + private final SkeletonConfig skeletonConfig; + private final PoseRecorder poseRecorder; + private final AutoBone autoBone; + + private Thread recordingThread = null; + private Thread saveRecordingThread = null; + private Thread autoBoneThread = null; + + private JButton saveRecordingButton; + private JButton adjustButton; + private JButton applyButton; + + private JLabel processLabel; + private JLabel lengthsLabel; + + public AutoBoneWindow(VRServer server, SkeletonConfig skeletonConfig) { + super("Skeleton Auto-Configuration"); + + this.server = server; + this.skeletonConfig = skeletonConfig; + this.poseRecorder = new PoseRecorder(server); + this.autoBone = new AutoBone(server); + + getContentPane().setLayout(new BoxLayout(getContentPane(), BoxLayout.PAGE_AXIS)); + add(new JScrollPane(pane = new EJBox(BoxLayout.PAGE_AXIS), ScrollPaneConstants.VERTICAL_SCROLLBAR_AS_NEEDED, ScrollPaneConstants.HORIZONTAL_SCROLLBAR_AS_NEEDED)); + + build(); + } + + private String getLengthsString() { + boolean first = true; + StringBuilder configInfo = new StringBuilder(""); + for (Entry entry : autoBone.configs.entrySet()) { + if (!first) { + configInfo.append(", "); + } else { + first = false; + } + + configInfo.append(entry.getKey() + ": " + StringUtils.prettyNumber(entry.getValue() * 100f, 2)); + } + + return configInfo.toString(); + } + + private void saveRecording(PoseFrame[] frames) { + if (saveDir.isDirectory() || saveDir.mkdirs()) { + File saveRecording; + int recordingIndex = 1; + do { + saveRecording = new File(saveDir, "ABRecording" + recordingIndex++ + ".abf"); + } while (saveRecording.exists()); + + LogManager.log.info("[AutoBone] Exporting frames to \"" + saveRecording.getPath() + "\"..."); + if (PoseFrameIO.writeToFile(saveRecording, frames)) { + LogManager.log.info("[AutoBone] Done exporting! Recording can be found at \"" + saveRecording.getPath() + "\"."); + } else { + LogManager.log.severe("[AutoBone] Failed to export the recording to \"" + saveRecording.getPath() + "\"."); + } + } else { + LogManager.log.severe("[AutoBone] Failed to create the recording directory \"" + saveDir.getPath() + "\"."); + } + } + + private List> loadRecordings() { + List> recordings = new FastList>(); + if (loadDir.isDirectory()) { + File[] files = loadDir.listFiles(); + if (files != null) { + for (File file : files) { + if (file.isFile() && org.apache.commons.lang3.StringUtils.endsWithIgnoreCase(file.getName(), ".abf")) { + LogManager.log.info("[AutoBone] Detected recording at \"" + file.getPath() + "\", loading frames..."); + PoseFrame[] frames = PoseFrameIO.readFromFile(file); + + if (frames == null) { + LogManager.log.severe("Reading frames from \"" + file.getPath() + "\" failed..."); + } else { + recordings.add(Pair.of(file.getName(), frames)); + } + } + } + } + } + + return recordings; + } + + private float processFrames(PoseFrame[] frames) { + autoBone.reloadConfigValues(); + + autoBone.minDataDistance = server.config.getInt("autobone.minimumDataDistance", autoBone.minDataDistance); + autoBone.maxDataDistance = server.config.getInt("autobone.maximumDataDistance", autoBone.maxDataDistance); + + autoBone.numEpochs = server.config.getInt("autobone.epochCount", autoBone.numEpochs); + + autoBone.initialAdjustRate = server.config.getFloat("autobone.adjustRate", autoBone.initialAdjustRate); + autoBone.adjustRateDecay = server.config.getFloat("autobone.adjustRateDecay", autoBone.adjustRateDecay); + + autoBone.slideErrorFactor = server.config.getFloat("autobone.slideErrorFactor", autoBone.slideErrorFactor); + autoBone.offsetErrorFactor = server.config.getFloat("autobone.offsetErrorFactor", autoBone.offsetErrorFactor); + autoBone.proportionErrorFactor = server.config.getFloat("autobone.proportionErrorFactor", autoBone.proportionErrorFactor); + autoBone.heightErrorFactor = server.config.getFloat("autobone.heightErrorFactor", autoBone.heightErrorFactor); + autoBone.positionErrorFactor = server.config.getFloat("autobone.positionErrorFactor", autoBone.positionErrorFactor); + autoBone.positionOffsetErrorFactor = server.config.getFloat("autobone.positionOffsetErrorFactor", autoBone.positionOffsetErrorFactor); + + boolean calcInitError = server.config.getBoolean("autobone.calculateInitialError", true); + float targetHeight = server.config.getFloat("autobone.manualTargetHeight", -1f); + return autoBone.processFrames(frames, calcInitError, targetHeight, (epoch) -> { + processLabel.setText(epoch.toString()); + lengthsLabel.setText(getLengthsString()); + }); + } + + @AWTThread + private void build() { + pane.add(new EJBox(BoxLayout.LINE_AXIS) {{ + setBorder(new EmptyBorder(i(5))); + add(new JButton("Start Recording") {{ + addMouseListener(new MouseInputAdapter() { + @Override + public void mouseClicked(MouseEvent e) { + // Prevent running multiple times + if (!isEnabled() || recordingThread != null) { + return; + } + + Thread thread = new Thread() { + @Override + public void run() { + try { + if (poseRecorder.isReadyToRecord()) { + setText("Recording..."); + // 1000 samples at 20 ms per sample is 20 seconds + int sampleCount = server.config.getInt("autobone.sampleCount", 1000); + long sampleRate = server.config.getLong("autobone.sampleRateMs", 20L); + Future framesFuture = poseRecorder.startFrameRecording(sampleCount, sampleRate); + PoseFrame[] frames = framesFuture.get(); + LogManager.log.info("[AutoBone] Done recording!"); + + saveRecordingButton.setEnabled(true); + adjustButton.setEnabled(true); + + if (server.config.getBoolean("autobone.saveRecordings", false)) { + setText("Saving..."); + saveRecording(frames); + } + } else { + setText("Not Ready..."); + LogManager.log.severe("[AutoBone] Unable to record..."); + Thread.sleep(3000); // Wait for 3 seconds + return; + } + } catch (Exception e) { + setText("Recording Failed..."); + LogManager.log.severe("[AutoBone] Failed recording!", e); + try { + Thread.sleep(3000); // Wait for 3 seconds + } catch (Exception e1) { + // Ignore + } + } finally { + setText("Start Recording"); + recordingThread = null; + } + } + }; + + recordingThread = thread; + thread.start(); + } + }); + }}); + + add(saveRecordingButton = new JButton("Save Recording") {{ + setEnabled(poseRecorder.hasRecording()); + addMouseListener(new MouseInputAdapter() { + @Override + public void mouseClicked(MouseEvent e) { + // Prevent running multiple times + if (!isEnabled() || saveRecordingThread != null) { + return; + } + + Thread thread = new Thread() { + @Override + public void run() { + try { + Future framesFuture = poseRecorder.getFramesAsync(); + if (framesFuture != null) { + setText("Waiting for Recording..."); + PoseFrame[] frames = framesFuture.get(); + + if (frames.length <= 0) { + throw new IllegalStateException("Recording has no frames"); + } + + setText("Saving..."); + saveRecording(frames); + } else { + setText("No Recording..."); + LogManager.log.severe("[AutoBone] Unable to save, no recording was done..."); + try { + Thread.sleep(3000); // Wait for 3 seconds + } catch (Exception e1) { + // Ignore + } + return; + } + } catch (Exception e) { + setText("Saving Failed..."); + LogManager.log.severe("[AutoBone] Failed to save recording!", e); + try { + Thread.sleep(3000); // Wait for 3 seconds + } catch (Exception e1) { + // Ignore + } + } finally { + setText("Save Recording"); + saveRecordingThread = null; + } + } + }; + + saveRecordingThread = thread; + thread.start(); + } + }); + }}); + + add(adjustButton = new JButton("Auto-Adjust") {{ + // If there are files to load, enable the button + setEnabled(poseRecorder.hasRecording() || (loadDir.isDirectory() && loadDir.list().length > 0)); + addMouseListener(new MouseInputAdapter() { + @Override + public void mouseClicked(MouseEvent e) { + // Prevent running multiple times + if (!isEnabled() || autoBoneThread != null) { + return; + } + + Thread thread = new Thread() { + @Override + public void run() { + try { + setText("Load..."); + List> frameRecordings = loadRecordings(); + + if (frameRecordings.size() > 0) { + LogManager.log.info("[AutoBone] Done loading frames!"); + } else { + Future framesFuture = poseRecorder.getFramesAsync(); + if (framesFuture != null) { + setText("Waiting for Recording..."); + PoseFrame[] frames = framesFuture.get(); + + if (frames.length <= 0) { + throw new IllegalStateException("Recording has no frames"); + } + + frameRecordings.add(Pair.of("", frames)); + } else { + setText("No Recordings..."); + LogManager.log.severe("[AutoBone] No recordings found in \"" + loadDir.getPath() + "\" and no recording was done..."); + try { + Thread.sleep(3000); // Wait for 3 seconds + } catch (Exception e1) { + // Ignore + } + return; + } + } + + setText("Processing..."); + LogManager.log.info("[AutoBone] Processing frames..."); + FastList heightPercentError = new FastList(frameRecordings.size()); + for (Pair recording : frameRecordings) { + LogManager.log.info("[AutoBone] Processing frames from \"" + recording.getKey() + "\"..."); + + heightPercentError.add(processFrames(recording.getValue())); + LogManager.log.info("[AutoBone] Done processing!"); + applyButton.setEnabled(true); + + //#region Stats/Values + Float neckLength = autoBone.getConfig("Neck"); + Float chestLength = autoBone.getConfig("Chest"); + Float waistLength = autoBone.getConfig("Waist"); + Float hipWidth = autoBone.getConfig("Hips width"); + Float legsLength = autoBone.getConfig("Legs length"); + Float kneeHeight = autoBone.getConfig("Knee height"); + + float neckWaist = neckLength != null && waistLength != null ? neckLength / waistLength : 0f; + float chestWaist = chestLength != null && waistLength != null ? chestLength / waistLength : 0f; + float hipWaist = hipWidth != null && waistLength != null ? hipWidth / waistLength : 0f; + float legWaist = legsLength != null && waistLength != null ? legsLength / waistLength : 0f; + float legBody = legsLength != null && waistLength != null && neckLength != null ? legsLength / (waistLength + neckLength) : 0f; + float kneeLeg = kneeHeight != null && legsLength != null ? kneeHeight / legsLength : 0f; + + LogManager.log.info("[AutoBone] Ratios: [{Neck-Waist: " + StringUtils.prettyNumber(neckWaist) + + "}, {Chest-Waist: " + StringUtils.prettyNumber(chestWaist) + + "}, {Hip-Waist: " + StringUtils.prettyNumber(hipWaist) + + "}, {Leg-Waist: " + StringUtils.prettyNumber(legWaist) + + "}, {Leg-Body: " + StringUtils.prettyNumber(legBody) + + "}, {Knee-Leg: " + StringUtils.prettyNumber(kneeLeg) + "}]"); + + String lengthsString = getLengthsString(); + LogManager.log.info("[AutoBone] Length values: " + lengthsString); + lengthsLabel.setText(lengthsString); + } + + if (heightPercentError.size() > 0) { + float mean = 0f; + for (float val : heightPercentError) { + mean += val; + } + mean /= heightPercentError.size(); + + float std = 0f; + for (float val : heightPercentError) { + float stdVal = val - mean; + std += stdVal * stdVal; + } + std = (float)Math.sqrt(std / heightPercentError.size()); + + LogManager.log.info("[AutoBone] Average height error: " + StringUtils.prettyNumber(mean, 6) + " (SD " + StringUtils.prettyNumber(std, 6) + ")"); + } + //#endregion + } catch (Exception e) { + setText("Failed..."); + LogManager.log.severe("[AutoBone] Failed adjustment!", e); + try { + Thread.sleep(3000); // Wait for 3 seconds + } catch (Exception e1) { + // Ignore + } + } finally { + setText("Auto-Adjust"); + autoBoneThread = null; + } + } + }; + + autoBoneThread = thread; + thread.start(); + } + }); + }}); + + add(applyButton = new JButton("Apply Values") {{ + setEnabled(false); + addMouseListener(new MouseInputAdapter() { + @Override + public void mouseClicked(MouseEvent e) { + if (!isEnabled()) { + return; + } + + autoBone.applyConfig(); + // Update GUI values after applying + skeletonConfig.refreshAll(); + } + }); + }}); + }}); + + pane.add(new EJBox(BoxLayout.LINE_AXIS) {{ + setBorder(new EmptyBorder(i(5))); + add(processLabel = new JLabel("Processing has not been started...")); + }}); + + pane.add(new EJBox(BoxLayout.LINE_AXIS) {{ + setBorder(new EmptyBorder(i(5))); + add(lengthsLabel = new JLabel(getLengthsString())); + }}); + + // Pack and display + pack(); + setLocationRelativeTo(null); + setVisible(false); + } +} diff --git a/src/main/java/io/eiren/gui/autobone/PoseFrame.java b/src/main/java/io/eiren/gui/autobone/PoseFrame.java new file mode 100644 index 0000000000..81b1d75d56 --- /dev/null +++ b/src/main/java/io/eiren/gui/autobone/PoseFrame.java @@ -0,0 +1,37 @@ +package io.eiren.gui.autobone; + +import java.util.HashMap; + +import com.jme3.math.Quaternion; +import com.jme3.math.Vector3f; + +import io.eiren.vr.processor.HumanSkeletonWithLegs; +import io.eiren.vr.processor.TransformNode; + +public final class PoseFrame { + + public final Vector3f rootPos; + public final HashMap rotations; + public final HashMap positions; + + public PoseFrame(Vector3f rootPos, HashMap rotations, HashMap positions) { + this.rootPos = rootPos; + this.rotations = rotations; + this.positions = positions; + } + + public PoseFrame(HumanSkeletonWithLegs skeleton) { + // Copy headset position + TransformNode rootNode = skeleton.getRootNode(); + this.rootPos = new Vector3f(rootNode.localTransform.getTranslation()); + + // Copy all rotations + this.rotations = new HashMap(); + rootNode.depthFirstTraversal(visitor -> { + // Insert a copied quaternion so it isn't changed by reference + rotations.put(visitor.getName(), new Quaternion(visitor.localTransform.getRotation())); + }); + + this.positions = null; + } +} diff --git a/src/main/java/io/eiren/gui/autobone/PoseFrameIO.java b/src/main/java/io/eiren/gui/autobone/PoseFrameIO.java new file mode 100644 index 0000000000..56cdf98754 --- /dev/null +++ b/src/main/java/io/eiren/gui/autobone/PoseFrameIO.java @@ -0,0 +1,174 @@ +package io.eiren.gui.autobone; + +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.util.HashMap; +import java.util.Map.Entry; + +import com.jme3.math.Quaternion; +import com.jme3.math.Vector3f; + +import io.eiren.util.logging.LogManager; + +public final class PoseFrameIO { + + private PoseFrameIO() { + // Do not allow instantiating + } + + public static boolean writeToFile(File file, PoseFrame[] frames) { + try (DataOutputStream outputStream = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(file)))) { + // Write every frame + outputStream.writeInt(frames.length); + for (PoseFrame frame : frames) { + writeFrame(outputStream, frame); + } + } catch (Exception e) { + LogManager.log.severe("Error writing frames to file", e); + return false; + } + + return true; + } + + public static boolean writeFrame(DataOutputStream outputStream, PoseFrame frame) { + try { + // Write root position vector + outputStream.writeFloat(frame.rootPos.x); + outputStream.writeFloat(frame.rootPos.y); + outputStream.writeFloat(frame.rootPos.z); + + if (frame.rotations != null) { + // Write rotations + outputStream.writeInt(frame.rotations.size()); + for (Entry entry : frame.rotations.entrySet()) { + // Write the label string + outputStream.writeUTF(entry.getKey()); + + // Write the rotation quaternion + Quaternion quat = entry.getValue(); + outputStream.writeFloat(quat.getX()); + outputStream.writeFloat(quat.getY()); + outputStream.writeFloat(quat.getZ()); + outputStream.writeFloat(quat.getW()); + } + } else { + outputStream.writeInt(0); + } + + if (frame.positions != null) { + // Write positions + outputStream.writeInt(frame.positions.size()); + for (Entry entry : frame.positions.entrySet()) { + // Write the label string + outputStream.writeUTF(entry.getKey()); + + // Write the rotation quaternion + Vector3f vec = entry.getValue(); + outputStream.writeFloat(vec.getX()); + outputStream.writeFloat(vec.getY()); + outputStream.writeFloat(vec.getZ()); + } + } else { + outputStream.writeInt(0); + } + } catch (Exception e) { + LogManager.log.severe("Error writing frame to stream", e); + return false; + } + + return true; + } + + public static boolean writeFrame(File file, PoseFrame frame) { + try (DataOutputStream outputStream = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(file)))) { + writeFrame(outputStream, frame); + } catch (Exception e) { + LogManager.log.severe("Error writing frame to file", e); + return false; + } + + return true; + } + + public static PoseFrame[] readFromFile(File file) { + try (DataInputStream inputStream = new DataInputStream(new BufferedInputStream(new FileInputStream(file)))) { + int frameCount = inputStream.readInt(); + + PoseFrame[] frames = new PoseFrame[frameCount]; + for (int i = 0; i < frameCount; i++) { + frames[i] = readFrame(inputStream); + } + + return frames; + } catch (Exception e) { + LogManager.log.severe("Error reading frames from file", e); + } + + return null; + } + + public static PoseFrame readFrame(DataInputStream inputStream) { + try { + float vecX = inputStream.readFloat(); + float vecY = inputStream.readFloat(); + float vecZ = inputStream.readFloat(); + + Vector3f vector = new Vector3f(vecX, vecY, vecZ); + + int rotationCount = inputStream.readInt(); + HashMap rotations = null; + if (rotationCount > 0) { + rotations = new HashMap(rotationCount); + for (int j = 0; j < rotationCount; j++) { + String label = inputStream.readUTF(); + + float quatX = inputStream.readFloat(); + float quatY = inputStream.readFloat(); + float quatZ = inputStream.readFloat(); + float quatW = inputStream.readFloat(); + Quaternion quaternion = new Quaternion(quatX, quatY, quatZ, quatW); + + rotations.put(label, quaternion); + } + } + + int positionCount = inputStream.readInt(); + HashMap positions = null; + if (positionCount > 0) { + positions = new HashMap(positionCount); + for (int j = 0; j < positionCount; j++) { + String label = inputStream.readUTF(); + + float posX = inputStream.readFloat(); + float posY = inputStream.readFloat(); + float posZ = inputStream.readFloat(); + Vector3f position = new Vector3f(posX, posY, posZ); + + positions.put(label, position); + } + } + + return new PoseFrame(vector, rotations, positions); + } catch (Exception e) { + LogManager.log.severe("Error reading frame from stream", e); + } + + return null; + } + + public static PoseFrame readFrame(File file) { + try { + return readFrame(new DataInputStream(new BufferedInputStream(new FileInputStream(file)))); + } catch (Exception e) { + LogManager.log.severe("Error reading frame from file", e); + } + + return null; + } +} diff --git a/src/main/java/io/eiren/gui/autobone/PoseRecorder.java b/src/main/java/io/eiren/gui/autobone/PoseRecorder.java new file mode 100644 index 0000000000..5af0ba1866 --- /dev/null +++ b/src/main/java/io/eiren/gui/autobone/PoseRecorder.java @@ -0,0 +1,136 @@ +package io.eiren.gui.autobone; + +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; + +import io.eiren.util.ann.ThreadSafe; +import io.eiren.util.ann.VRServerThread; +import io.eiren.util.collections.FastList; +import io.eiren.util.logging.LogManager; +import io.eiren.vr.VRServer; +import io.eiren.vr.processor.HumanSkeleton; +import io.eiren.vr.processor.HumanSkeletonWithLegs; + +public class PoseRecorder { + + protected final FastList frames = new FastList(); + + protected int numFrames = -1; + protected long frameRecordingInterval = 60L; + protected long nextFrameTimeMs = -1L; + + protected CompletableFuture currentRecording; + + protected final VRServer server; + HumanSkeletonWithLegs skeleton = null; + + public PoseRecorder(VRServer server) { + this.server = server; + server.addOnTick(this::onTick); + server.addSkeletonUpdatedCallback(this::skeletonUpdated); + } + + @VRServerThread + public void onTick() { + if (numFrames > 0) { + HumanSkeletonWithLegs skeleton = this.skeleton; + if (skeleton != null) { + if (frames.size() < numFrames) { + if (System.currentTimeMillis() >= nextFrameTimeMs) { + nextFrameTimeMs = System.currentTimeMillis() + frameRecordingInterval; + frames.add(new PoseFrame(skeleton)); + + // If done, send finished recording + if (frames.size() >= numFrames) { + internalStopRecording(); + } + } + } else { + // If done and hasn't yet, send finished recording + internalStopRecording(); + } + } + } + } + + @ThreadSafe + public void skeletonUpdated(HumanSkeleton newSkeleton) { + if (newSkeleton instanceof HumanSkeletonWithLegs) { + skeleton = (HumanSkeletonWithLegs) newSkeleton; + } + } + + public synchronized Future startFrameRecording(int numFrames, long interval) { + if (numFrames < 1) { + throw new IllegalArgumentException("numFrames must at least have a value of 1"); + } + if (interval < 1) { + throw new IllegalArgumentException("interval must at least have a value of 1"); + } + if (!isReadyToRecord()) { + throw new IllegalStateException("PoseRecorder isn't ready to record!"); + } + + cancelFrameRecording(); + + // Clear old frames and ensure new size can be held + frames.clear(); + frames.ensureCapacity(numFrames); + + this.numFrames = numFrames; + + frameRecordingInterval = interval; + nextFrameTimeMs = -1L; + + LogManager.log.info("[PoseRecorder] Recording " + numFrames + " samples at a " + interval + " ms frame interval"); + + currentRecording = new CompletableFuture(); + return currentRecording; + } + + private void internalStopRecording() { + CompletableFuture currentRecording = this.currentRecording; + if (currentRecording != null && !currentRecording.isDone()) { + // Stop the recording, returning the frames recorded + currentRecording.complete(frames.toArray(new PoseFrame[0])); + } + + numFrames = -1; + } + + public synchronized void stopFrameRecording() { + internalStopRecording(); + } + + public synchronized void cancelFrameRecording() { + CompletableFuture currentRecording = this.currentRecording; + if (currentRecording != null && !currentRecording.isDone()) { + // Cancel the current recording and return nothing + currentRecording.cancel(true); + } + + numFrames = -1; + } + + public boolean isReadyToRecord() { + return skeleton != null; + } + + public boolean isRecording() { + return numFrames > frames.size(); + } + + public boolean hasRecording() { + return currentRecording != null; + } + + public Future getFramesAsync() { + return currentRecording; + } + + public PoseFrame[] getFrames() throws ExecutionException, InterruptedException { + CompletableFuture currentRecording = this.currentRecording; + return currentRecording != null ? currentRecording.get() : null; + } +} diff --git a/src/main/java/io/eiren/gui/autobone/SimpleSkeleton.java b/src/main/java/io/eiren/gui/autobone/SimpleSkeleton.java new file mode 100644 index 0000000000..7cc4f3a890 --- /dev/null +++ b/src/main/java/io/eiren/gui/autobone/SimpleSkeleton.java @@ -0,0 +1,291 @@ +package io.eiren.gui.autobone; + +import java.util.HashMap; +import java.util.Map; +import java.util.Map.Entry; + +import com.jme3.math.Quaternion; +import com.jme3.math.Vector3f; + +import io.eiren.vr.processor.HumanSkeletonWithLegs; +import io.eiren.vr.processor.HumanSkeletonWithWaist; +import io.eiren.vr.processor.TransformNode; +import io.eiren.yaml.YamlFile; + +public class SimpleSkeleton { + + // Waist + protected final TransformNode hmdNode = new TransformNode("HMD", false); + protected final TransformNode headNode = new TransformNode("Head", false); + protected final TransformNode neckNode = new TransformNode("Neck", false); + protected final TransformNode waistNode = new TransformNode("Waist", false); + protected final TransformNode chestNode = new TransformNode("Chest", false); + + protected float chestDistance = 0.42f; + /** + * Distance from eyes to waist + */ + protected float waistDistance = 0.85f; + /** + * Distance from eyes to the base of the neck + */ + protected float neckLength = HumanSkeletonWithWaist.NECK_LENGTH_DEFAULT; + /** + * Distance from eyes to ear + */ + protected float headShift = HumanSkeletonWithWaist.HEAD_SHIFT_DEFAULT; + + // Legs + protected final TransformNode leftHipNode = new TransformNode("Left-Hip", false); + protected final TransformNode leftKneeNode = new TransformNode("Left-Knee", false); + protected final TransformNode leftAnkleNode = new TransformNode("Left-Ankle", false); + protected final TransformNode rightHipNode = new TransformNode("Right-Hip", false); + protected final TransformNode rightKneeNode = new TransformNode("Right-Knee", false); + protected final TransformNode rightAnkleNode = new TransformNode("Right-Ankle", false); + + /** + * Distance between centers of both hips + */ + protected float hipsWidth = HumanSkeletonWithLegs.HIPS_WIDTH_DEFAULT; + /** + * Length from waist to knees + */ + protected float kneeHeight = 0.42f; + /** + * Distance from waist to ankle + */ + protected float legsLength = 0.84f; + + protected final HashMap nodes = new HashMap(); + + public SimpleSkeleton() { + // Assemble skeleton to waist + hmdNode.attachChild(headNode); + headNode.localTransform.setTranslation(0, 0, headShift); + + headNode.attachChild(neckNode); + neckNode.localTransform.setTranslation(0, -neckLength, 0); + + neckNode.attachChild(chestNode); + chestNode.localTransform.setTranslation(0, -chestDistance, 0); + + chestNode.attachChild(waistNode); + waistNode.localTransform.setTranslation(0, -(waistDistance - chestDistance), 0); + + // Assemble skeleton to feet + waistNode.attachChild(leftHipNode); + leftHipNode.localTransform.setTranslation(-hipsWidth / 2, 0, 0); + + waistNode.attachChild(rightHipNode); + rightHipNode.localTransform.setTranslation(hipsWidth / 2, 0, 0); + + leftHipNode.attachChild(leftKneeNode); + leftKneeNode.localTransform.setTranslation(0, -(legsLength - kneeHeight), 0); + + rightHipNode.attachChild(rightKneeNode); + rightKneeNode.localTransform.setTranslation(0, -(legsLength - kneeHeight), 0); + + leftKneeNode.attachChild(leftAnkleNode); + leftAnkleNode.localTransform.setTranslation(0, -kneeHeight, 0); + + rightKneeNode.attachChild(rightAnkleNode); + rightAnkleNode.localTransform.setTranslation(0, -kneeHeight, 0); + + // Set up a HashMap to get nodes by name easily + hmdNode.depthFirstTraversal(visitor -> { + nodes.put(visitor.getName(), visitor); + }); + } + + public SimpleSkeleton(Iterable> configs, Iterable> altConfigs) { + // Initialize + this(); + + // Set configs + if (altConfigs != null) { + // Set alts first, so if there's any overlap it doesn't affect the values + setSkeletonConfigs(altConfigs); + } + setSkeletonConfigs(configs); + } + + public SimpleSkeleton(Map configs, Map altConfigs) { + this(configs.entrySet(), altConfigs.entrySet()); + } + + public SimpleSkeleton(Iterable> configs) { + this(configs, null); + } + + public SimpleSkeleton(Map configs) { + this(configs.entrySet()); + } + + public void setPoseFromSkeleton(HumanSkeletonWithLegs humanSkeleton) { + TransformNode rootNode = humanSkeleton.getRootNode(); + + // Copy headset position + hmdNode.localTransform.setTranslation(rootNode.localTransform.getTranslation()); + + // Copy all rotations + rootNode.depthFirstTraversal(visitor -> { + TransformNode targetNode = nodes.get(visitor.getName()); + + // Handle unexpected nodes gracefully + if (targetNode != null) { + targetNode.localTransform.setRotation(visitor.localTransform.getRotation()); + } + }); + } + + public void setPoseFromFrame(PoseFrame frame) { + // Copy headset position + hmdNode.localTransform.setTranslation(frame.rootPos); + + if (frame.rotations != null) { + // Copy all rotations + for (Entry rotation : frame.rotations.entrySet()) { + TransformNode targetNode = nodes.get(rotation.getKey()); + + // Handle unexpected nodes gracefully + if (targetNode != null) { + targetNode.localTransform.setRotation(rotation.getValue()); + } + } + } + + updatePose(); + } + + public void setSkeletonConfigs(Iterable> configs) { + for (Entry config : configs) { + setSkeletonConfig(config.getKey(), config.getValue()); + } + } + + public void setSkeletonConfigs(Map configs) { + setSkeletonConfigs(configs.entrySet()); + } + + public void setSkeletonConfig(String joint, float newLength) { + setSkeletonConfig(joint, newLength, false); + } + + public void setSkeletonConfig(String joint, float newLength, boolean updatePose) { + switch(joint) { + case "Head": + headShift = newLength; + headNode.localTransform.setTranslation(0, 0, headShift); + if (updatePose) { + headNode.update(); + } + break; + case "Neck": + neckLength = newLength; + neckNode.localTransform.setTranslation(0, -neckLength, 0); + if (updatePose) { + neckNode.update(); + } + break; + case "Waist": + waistDistance = newLength; + waistNode.localTransform.setTranslation(0, -(waistDistance - chestDistance), 0); + if (updatePose) { + waistNode.update(); + } + break; + case "Chest": + chestDistance = newLength; + chestNode.localTransform.setTranslation(0, -chestDistance, 0); + waistNode.localTransform.setTranslation(0, -(waistDistance - chestDistance), 0); + if (updatePose) { + chestNode.update(); + } + break; + case "Hips width": + hipsWidth = newLength; + leftHipNode.localTransform.setTranslation(-hipsWidth / 2, 0, 0); + rightHipNode.localTransform.setTranslation(hipsWidth / 2, 0, 0); + if (updatePose) { + leftHipNode.update(); + rightHipNode.update(); + } + break; + case "Knee height": + kneeHeight = newLength; + leftAnkleNode.localTransform.setTranslation(0, -kneeHeight, 0); + rightAnkleNode.localTransform.setTranslation(0, -kneeHeight, 0); + leftKneeNode.localTransform.setTranslation(0, -(legsLength - kneeHeight), 0); + rightKneeNode.localTransform.setTranslation(0, -(legsLength - kneeHeight), 0); + if (updatePose) { + leftKneeNode.update(); + rightKneeNode.update(); + } + break; + case "Legs length": + legsLength = newLength; + leftKneeNode.localTransform.setTranslation(0, -(legsLength - kneeHeight), 0); + rightKneeNode.localTransform.setTranslation(0, -(legsLength - kneeHeight), 0); + if (updatePose) { + leftKneeNode.update(); + rightKneeNode.update(); + } + break; + } + } + + public Float getSkeletonConfig(String joint) { + switch(joint) { + case "Head": + return headShift; + case "Neck": + return neckLength; + case "Waist": + return waistDistance; + case "Chest": + return chestDistance; + case "Hips width": + return hipsWidth; + case "Knee height": + return kneeHeight; + case "Legs length": + return legsLength; + } + + return null; + } + + public void updatePose() { + hmdNode.update(); + } + + public Vector3f getNodePosition(String node) { + TransformNode transformNode = nodes.get(node); + return transformNode != null ? transformNode.worldTransform.getTranslation() : null; + } + + public Vector3f getHMDPos() { + return hmdNode.worldTransform.getTranslation(); + } + + public Vector3f getLeftFootPos() { + return leftAnkleNode.worldTransform.getTranslation(); + } + + public Vector3f getRightFootPos() { + return rightAnkleNode.worldTransform.getTranslation(); + } + + public void saveConfigs(YamlFile config) { + // Save waist configs + config.setProperty("body.headShift", headShift); + config.setProperty("body.neckLength", neckLength); + config.setProperty("body.waistDistance", waistDistance); + config.setProperty("body.chestDistance", chestDistance); + + // Save leg configs + config.setProperty("body.hipsWidth", hipsWidth); + config.setProperty("body.kneeHeight", kneeHeight); + config.setProperty("body.legsLength", legsLength); + } +}