From 31375855a0ce70d4808bc611e9424fd27932cb4a Mon Sep 17 00:00:00 2001 From: ButterscotchVanilla Date: Sat, 18 Sep 2021 22:50:41 -0400 Subject: [PATCH 1/9] Fix PoseRecorder frame timing --- .../dev/slimevr/poserecorder/PoseRecorder.java | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/src/main/java/dev/slimevr/poserecorder/PoseRecorder.java b/src/main/java/dev/slimevr/poserecorder/PoseRecorder.java index 9e72e57033..f46f3ff99f 100644 --- a/src/main/java/dev/slimevr/poserecorder/PoseRecorder.java +++ b/src/main/java/dev/slimevr/poserecorder/PoseRecorder.java @@ -37,11 +37,17 @@ public void onTick() { if(numFrames > 0) { PoseFrame poseFrame = this.poseFrame; List> trackers = this.trackers; - if(poseFrame != null && trackers != null) { - if(frameCursor < numFrames) { - if(System.currentTimeMillis() >= nextFrameTimeMs) { - nextFrameTimeMs = System.currentTimeMillis() + frameRecordingInterval; - + if (poseFrame != null && trackers != null) { + if (frameCursor < numFrames) { + long curTime = System.currentTimeMillis(); + if (curTime >= nextFrameTimeMs) { + nextFrameTimeMs += frameRecordingInterval; + + // To prevent duplicate frames, make sure the frame time is always in the future + if (nextFrameTimeMs <= curTime) { + nextFrameTimeMs = curTime + frameRecordingInterval; + } + int cursor = frameCursor++; for(Pair tracker : trackers) { // Add a frame for each tracker From e7f81eb1aaa8cc133704abf3e5942ee069134d66 Mon Sep 17 00:00:00 2001 From: ButterscotchVanilla Date: Sun, 19 Sep 2021 21:59:52 -0400 Subject: [PATCH 2/9] Rename PoseFrame to PoseFrames --- .../java/dev/slimevr/autobone/AutoBone.java | 238 +++++++++--------- .../java/dev/slimevr/gui/AutoBoneWindow.java | 140 +++++------ .../dev/slimevr/poserecorder/PoseFrameIO.java | 54 ++-- .../{PoseFrame.java => PoseFrames.java} | 72 +++--- .../slimevr/poserecorder/PoseRecorder.java | 78 +++--- 5 files changed, 291 insertions(+), 291 deletions(-) rename src/main/java/dev/slimevr/poserecorder/{PoseFrame.java => PoseFrames.java} (89%) diff --git a/src/main/java/dev/slimevr/autobone/AutoBone.java b/src/main/java/dev/slimevr/autobone/AutoBone.java index 5f4afe23f6..214a97593b 100644 --- a/src/main/java/dev/slimevr/autobone/AutoBone.java +++ b/src/main/java/dev/slimevr/autobone/AutoBone.java @@ -7,7 +7,7 @@ import com.jme3.math.Vector3f; -import dev.slimevr.poserecorder.PoseFrame; +import dev.slimevr.poserecorder.PoseFrames; import dev.slimevr.poserecorder.TrackerFrame; import dev.slimevr.poserecorder.TrackerFrameData; import io.eiren.util.ann.ThreadSafe; @@ -21,78 +21,78 @@ import io.eiren.vr.trackers.TrackerUtils; public class AutoBone { - + public class Epoch { - + public final int epoch; public final float epochError; - + public Epoch(int epoch, float epochError) { this.epoch = epoch; this.epochError = epochError; } - + @Override public String toString() { return "Epoch: " + epoch + ", Epoch Error: " + epochError; } } - + public int cursorIncrement = 1; - + public int minDataDistance = 2; public int maxDataDistance = 32; - + public int numEpochs = 5; - + public float initialAdjustRate = 2.5f; public float adjustRateDecay = 1.01f; - + public float slideErrorFactor = 1.0f; public float offsetErrorFactor = 0.0f; public float proportionErrorFactor = 0.2f; public float heightErrorFactor = 0.1f; public float positionErrorFactor = 0.0f; public float positionOffsetErrorFactor = 0.0f; - + // Human average is probably 1.1235 (SD 0.07) public float legBodyRatio = 1.1235f; // SD of 0.07, capture 68% within range public float legBodyRatioRange = 0.07f; - + // Assume these to be approximately half public float kneeLegRatio = 0.5f; public float chestWaistRatio = 0.5f; - + protected final VRServer server; - + protected HumanSkeletonWithLegs skeleton = null; - + // This is filled by reloadConfigValues() public final HashMap configs = new HashMap(); public final HashMap staticConfigs = new HashMap(); - + public final FastList heightConfigs = new FastList(new String[]{"Neck", "Waist", "Legs length" }); - + public AutoBone(VRServer server) { this.server = server; - + reloadConfigValues(); - + server.addSkeletonUpdatedCallback(this::skeletonUpdated); } - + public void reloadConfigValues() { reloadConfigValues(null); } - + public void reloadConfigValues(TrackerFrame[] frame) { // Load waist configs staticConfigs.put("Head", server.config.getFloat("body.headShift", HumanSkeletonWithWaist.HEAD_SHIFT_DEFAULT)); staticConfigs.put("Neck", server.config.getFloat("body.neckLength", HumanSkeletonWithWaist.NECK_LENGTH_DEFAULT)); configs.put("Waist", server.config.getFloat("body.waistDistance", 0.85f)); - + if(server.config.getBoolean("autobone.forceChestTracker", false) || (frame != null && TrackerUtils.findTrackerForBodyPosition(frame, TrackerPosition.CHEST) != null) || TrackerUtils.findTrackerForBodyPosition(server.getAllTrackers(), TrackerPosition.CHEST) != null) { // If force enabled or has a chest tracker configs.put("Chest", server.config.getFloat("body.chestDistance", 0.42f)); @@ -101,13 +101,13 @@ public void reloadConfigValues(TrackerFrame[] frame) { configs.remove("Chest"); staticConfigs.put("Chest", server.config.getFloat("body.chestDistance", 0.42f)); } - + // Load leg configs staticConfigs.put("Hips width", server.config.getFloat("body.hipsWidth", HumanSkeletonWithLegs.HIPS_WIDTH_DEFAULT)); configs.put("Legs length", server.config.getFloat("body.legsLength", 0.84f)); configs.put("Knee height", server.config.getFloat("body.kneeHeight", 0.42f)); } - + @ThreadSafe public void skeletonUpdated(HumanSkeleton newSkeleton) { if(newSkeleton instanceof HumanSkeletonWithLegs) { @@ -116,34 +116,34 @@ public void skeletonUpdated(HumanSkeleton newSkeleton) { LogManager.log.info("[AutoBone] Received updated skeleton"); } } - + public void applyConfig() { if(!applyConfigToSkeleton(skeleton)) { // Unable to apply to skeleton, save directly saveConfigs(); } } - + public boolean applyConfigToSkeleton(HumanSkeleton skeleton) { if(skeleton == null) { return false; } - + configs.forEach(skeleton::setSkeletonConfig); - + server.saveConfig(); - + LogManager.log.info("[AutoBone] Configured skeleton bone lengths"); return true; } - + private void setConfig(String name, String path) { Float value = configs.get(name); if(value != null) { server.config.setProperty(path, value); } } - + // This doesn't require a skeleton, therefore can be used if skeleton is null public void saveConfigs() { setConfig("Head", "body.headShift"); @@ -153,52 +153,52 @@ public void saveConfigs() { setConfig("Hips width", "body.hipsWidth"); setConfig("Legs length", "body.legsLength"); setConfig("Knee height", "body.kneeHeight"); - + server.saveConfig(); } - + public Float getConfig(String config) { Float configVal = configs.get(config); return configVal != null ? configVal : staticConfigs.get(config); } - + public Float getConfig(String config, Map configs, Map configsAlt) { if(configs == null) { throw new NullPointerException("Argument \"configs\" must not be null"); } - + Float configVal = configs.get(config); return configVal != null || configsAlt == null ? configVal : configsAlt.get(config); } - + public float getHeight(Map configs) { return getHeight(configs, null); } - + public float getHeight(Map configs, Map configsAlt) { float height = 0f; - + for(String heightConfig : heightConfigs) { Float length = getConfig(heightConfig, configs, configsAlt); if(length != null) { height += length; } } - + return height; } - + public float getLengthSum(Map configs) { float length = 0f; - + for(float boneLength : configs.values()) { length += boneLength; } - + return length; } - - public float getMaxHmdHeight(PoseFrame frames) { + + public float getMaxHmdHeight(PoseFrames frames) { float maxHeight = 0f; for(TrackerFrame[] frame : frames) { TrackerFrame hmd = TrackerUtils.findTrackerForBodyPosition(frame, TrackerPosition.HMD); @@ -208,39 +208,39 @@ public float getMaxHmdHeight(PoseFrame frames) { } return maxHeight; } - - public void processFrames(PoseFrame frames) { + + public void processFrames(PoseFrames frames) { processFrames(frames, -1f); } - - public void processFrames(PoseFrame frames, Consumer epochCallback) { + + public void processFrames(PoseFrames frames, Consumer epochCallback) { processFrames(frames, -1f, epochCallback); } - - public void processFrames(PoseFrame frames, float targetHeight) { + + public void processFrames(PoseFrames frames, float targetHeight) { processFrames(frames, true, targetHeight); } - - public void processFrames(PoseFrame frames, float targetHeight, Consumer epochCallback) { + + public void processFrames(PoseFrames frames, float targetHeight, Consumer epochCallback) { processFrames(frames, true, targetHeight, epochCallback); } - - public float processFrames(PoseFrame frames, boolean calcInitError, float targetHeight) { + + public float processFrames(PoseFrames frames, boolean calcInitError, float targetHeight) { return processFrames(frames, calcInitError, targetHeight, null); } - - public float processFrames(PoseFrame frames, boolean calcInitError, float targetHeight, Consumer epochCallback) { + + public float processFrames(PoseFrames frames, boolean calcInitError, float targetHeight, Consumer epochCallback) { final int frameCount = frames.getMaxFrameCount(); - + final SimpleSkeleton skeleton1 = new SimpleSkeleton(configs, staticConfigs); final TrackerFrame[] trackerBuffer1 = new TrackerFrame[frames.getTrackerCount()]; - + frames.getFrames(0, trackerBuffer1); reloadConfigValues(trackerBuffer1); // Reload configs and detect chest tracker from the first frame - + final SimpleSkeleton skeleton2 = new SimpleSkeleton(configs, staticConfigs); final TrackerFrame[] trackerBuffer2 = new TrackerFrame[frames.getTrackerCount()]; - + // If target height isn't specified, auto-detect if(targetHeight < 0f) { if(skeleton != null) { @@ -253,62 +253,62 @@ public float processFrames(PoseFrame frames, boolean calcInitError, float target } else { LogManager.log.info("[AutoBone] Max headset height detected: " + hmdHeight); } - + // Estimate target height from HMD height targetHeight = hmdHeight; } } - + for(int epoch = calcInitError ? -1 : 0; epoch < numEpochs; epoch++) { float sumError = 0f; int errorCount = 0; - + float adjustRate = epoch >= 0 ? (float) (initialAdjustRate / Math.pow(adjustRateDecay, epoch)) : 0f; - + for(int cursorOffset = minDataDistance; cursorOffset <= maxDataDistance && cursorOffset < frameCount; cursorOffset++) { for(int frameCursor = 0; frameCursor < frameCount - cursorOffset; frameCursor += cursorIncrement) { frames.getFrames(frameCursor, trackerBuffer1); frames.getFrames(frameCursor + cursorOffset, trackerBuffer2); - + skeleton1.setSkeletonConfigs(configs); skeleton2.setSkeletonConfigs(configs); - + skeleton1.setPoseFromFrame(trackerBuffer1); skeleton2.setPoseFromFrame(trackerBuffer2); - + float totalLength = getLengthSum(configs); float curHeight = getHeight(configs, staticConfigs); float errorDeriv = getErrorDeriv(trackerBuffer1, trackerBuffer2, skeleton1, skeleton2, targetHeight - curHeight); float error = errorFunc(errorDeriv); - + // In case of fire if(Float.isNaN(error) || Float.isInfinite(error)) { // Extinguish LogManager.log.warning("[AutoBone] Error value is invalid, resetting variables to recover"); reloadConfigValues(trackerBuffer1); - + // Reset error sum values sumError = 0f; errorCount = 0; - + // Continue on new data continue; } - + // Store the error count for logging purposes sumError += errorDeriv; errorCount++; - + float adjustVal = error * adjustRate; - + for(Entry entry : configs.entrySet()) { // Skip adjustment if the epoch is before starting (for logging only) if(epoch < 0) { break; } - + float originalLength = entry.getValue(); - + // Try positive and negative adjustments boolean isHeightVar = heightConfigs.contains(entry.getKey()); float minError = errorDeriv; @@ -317,78 +317,78 @@ public float processFrames(PoseFrame frames, boolean calcInitError, float target // Scale by the ratio for smooth adjustment and more stable results float curAdjustVal = ((i == 0 ? adjustVal : -adjustVal) * originalLength) / totalLength; float newLength = originalLength + curAdjustVal; - + // No small or negative numbers!!! Bad algorithm! if(newLength < 0.01f) { continue; } - + updateSkeletonBoneLength(skeleton1, skeleton2, entry.getKey(), newLength); - + float newHeight = isHeightVar ? curHeight + curAdjustVal : curHeight; float newErrorDeriv = getErrorDeriv(trackerBuffer1, trackerBuffer2, skeleton1, skeleton2, targetHeight - newHeight); - + if(newErrorDeriv < minError) { minError = newErrorDeriv; finalNewLength = newLength; } } - + if(finalNewLength > 0f) { entry.setValue(finalNewLength); } - + // Reset the length to minimize bias in other variables, it's applied later updateSkeletonBoneLength(skeleton1, skeleton2, entry.getKey(), originalLength); } } } - + // Calculate average error over the epoch float avgError = errorCount > 0 ? sumError / errorCount : -1f; LogManager.log.info("[AutoBone] Epoch " + (epoch + 1) + " average error: " + avgError); - + if(epochCallback != null) { epochCallback.accept(new Epoch(epoch + 1, avgError)); } } - + float finalHeight = getHeight(configs, staticConfigs); LogManager.log.info("[AutoBone] Target height: " + targetHeight + " New height: " + finalHeight); - + return Math.abs(finalHeight - targetHeight); } - + // The change in position of the ankle over time protected float getSlideErrorDeriv(SimpleSkeleton skeleton1, SimpleSkeleton skeleton2) { float slideLeft = skeleton1.getNodePosition(TrackerPosition.LEFT_ANKLE).distance(skeleton2.getNodePosition(TrackerPosition.LEFT_ANKLE)); float slideRight = skeleton1.getNodePosition(TrackerPosition.RIGHT_ANKLE).distance(skeleton2.getNodePosition(TrackerPosition.RIGHT_ANKLE)); - + // Divide by 4 to halve and average, it's halved because you want to approach a midpoint, not the other point return (slideLeft + slideRight) / 4f; } - + // The offset between both feet at one instant and over time protected float getOffsetErrorDeriv(SimpleSkeleton skeleton1, SimpleSkeleton skeleton2) { float skeleton1Left = skeleton1.getNodePosition(TrackerPosition.LEFT_ANKLE).getY(); float skeleton1Right = skeleton1.getNodePosition(TrackerPosition.RIGHT_ANKLE).getY(); - + float skeleton2Left = skeleton2.getNodePosition(TrackerPosition.LEFT_ANKLE).getY(); float skeleton2Right = skeleton2.getNodePosition(TrackerPosition.RIGHT_ANKLE).getY(); - + float dist1 = Math.abs(skeleton1Left - skeleton1Right); float dist2 = Math.abs(skeleton2Left - skeleton2Right); - + float dist3 = Math.abs(skeleton1Left - skeleton2Right); float dist4 = Math.abs(skeleton2Left - skeleton1Right); - + float dist5 = Math.abs(skeleton1Left - skeleton2Left); float dist6 = Math.abs(skeleton1Right - skeleton2Right); - + // Divide by 12 to halve and average, it's halved because you want to approach a midpoint, not the other point return (dist1 + dist2 + dist3 + dist4 + dist5 + dist6) / 12f; } - + // The distance from average human proportions protected float getProportionErrorDeriv(SimpleSkeleton skeleton) { Float neckLength = skeleton.getSkeletonConfig("Neck"); @@ -396,119 +396,119 @@ protected float getProportionErrorDeriv(SimpleSkeleton skeleton) { Float waistLength = skeleton.getSkeletonConfig("Waist"); Float legsLength = skeleton.getSkeletonConfig("Legs length"); Float kneeHeight = skeleton.getSkeletonConfig("Knee height"); - + float chestWaist = chestLength != null && waistLength != null ? Math.abs((chestLength / waistLength) - chestWaistRatio) : 0f; float legBody = legsLength != null && waistLength != null && neckLength != null ? Math.abs((legsLength / (waistLength + neckLength)) - legBodyRatio) : 0f; float kneeLeg = kneeHeight != null && legsLength != null ? Math.abs((kneeHeight / legsLength) - kneeLegRatio) : 0f; - + if(legBody <= legBodyRatioRange) { legBody = 0f; } else { legBody -= legBodyRatioRange; } - + return (chestWaist + legBody + kneeLeg) / 3f; } - + // The distance of any points to the corresponding absolute position protected float getPositionErrorDeriv(TrackerFrame[] frame, SimpleSkeleton skeleton) { float offset = 0f; int offsetCount = 0; - + for(TrackerFrame trackerFrame : frame) { if(trackerFrame == null || !trackerFrame.hasData(TrackerFrameData.POSITION)) { continue; } - + Vector3f nodePos = skeleton.getNodePosition(trackerFrame.designation.designation); if(nodePos != null) { offset += Math.abs(nodePos.distance(trackerFrame.position)); offsetCount++; } } - + return offsetCount > 0 ? offset / offsetCount : 0f; } - + // The difference between offset of absolute position and the corresponding point over time protected float getPositionOffsetErrorDeriv(TrackerFrame[] frame1, TrackerFrame[] frame2, SimpleSkeleton skeleton1, SimpleSkeleton skeleton2) { float offset = 0f; int offsetCount = 0; - + for(TrackerFrame trackerFrame1 : frame1) { if(trackerFrame1 == null || !trackerFrame1.hasData(TrackerFrameData.POSITION)) { continue; } - + TrackerFrame trackerFrame2 = TrackerUtils.findTrackerForBodyPosition(frame2, trackerFrame1.designation); if(trackerFrame2 == null || !trackerFrame2.hasData(TrackerFrameData.POSITION)) { continue; } - + Vector3f nodePos1 = skeleton1.getNodePosition(trackerFrame1.designation); if(nodePos1 == null) { continue; } - + Vector3f nodePos2 = skeleton2.getNodePosition(trackerFrame2.designation); if(nodePos2 == null) { continue; } - + float dist1 = Math.abs(nodePos1.distance(trackerFrame1.position)); float dist2 = Math.abs(nodePos2.distance(trackerFrame2.position)); - + offset += Math.abs(dist2 - dist1); offsetCount++; } - + return offsetCount > 0 ? offset / offsetCount : 0f; } - + protected float getErrorDeriv(TrackerFrame[] frame1, TrackerFrame[] frame2, SimpleSkeleton skeleton1, SimpleSkeleton skeleton2, float heightChange) { float totalError = 0f; float sumWeight = 0f; - + if(slideErrorFactor > 0f) { totalError += getSlideErrorDeriv(skeleton1, skeleton2) * slideErrorFactor; sumWeight += slideErrorFactor; } - + if(offsetErrorFactor > 0f) { totalError += getOffsetErrorDeriv(skeleton1, skeleton2) * offsetErrorFactor; sumWeight += offsetErrorFactor; } - + if(proportionErrorFactor > 0f) { // Either skeleton will work fine, skeleton1 is used as a default totalError += getProportionErrorDeriv(skeleton1) * proportionErrorFactor; sumWeight += proportionErrorFactor; } - + if(heightErrorFactor > 0f) { totalError += Math.abs(heightChange) * heightErrorFactor; sumWeight += heightErrorFactor; } - + if(positionErrorFactor > 0f) { totalError += (getPositionErrorDeriv(frame1, skeleton1) + getPositionErrorDeriv(frame2, skeleton2) / 2f) * positionErrorFactor; sumWeight += positionErrorFactor; } - + if(positionOffsetErrorFactor > 0f) { totalError += getPositionOffsetErrorDeriv(frame1, frame2, skeleton1, skeleton2) * positionOffsetErrorFactor; sumWeight += positionOffsetErrorFactor; } - + // Minimize sliding, minimize foot height offset, minimize change in total height return sumWeight > 0f ? totalError / sumWeight : 0f; } - + // Mean square error function protected static float errorFunc(float errorDeriv) { return 0.5f * (errorDeriv * errorDeriv); } - + protected void updateSkeletonBoneLength(SimpleSkeleton skeleton1, SimpleSkeleton skeleton2, String joint, float newLength) { skeleton1.setSkeletonConfig(joint, newLength, true); skeleton2.setSkeletonConfig(joint, newLength, true); diff --git a/src/main/java/dev/slimevr/gui/AutoBoneWindow.java b/src/main/java/dev/slimevr/gui/AutoBoneWindow.java index 55d2c5b7ec..3c97e22f77 100644 --- a/src/main/java/dev/slimevr/gui/AutoBoneWindow.java +++ b/src/main/java/dev/slimevr/gui/AutoBoneWindow.java @@ -25,47 +25,47 @@ import dev.slimevr.autobone.AutoBone; import dev.slimevr.gui.swing.EJBox; -import dev.slimevr.poserecorder.PoseFrame; +import dev.slimevr.poserecorder.PoseFrames; import dev.slimevr.poserecorder.PoseFrameIO; import dev.slimevr.poserecorder.PoseRecorder; public class AutoBoneWindow extends JFrame { - + private static File saveDir = new File("Recordings"); private static File loadDir = new File("LoadRecordings"); - + private EJBox pane; - + private final transient VRServer server; private final transient SkeletonConfig skeletonConfig; private final transient PoseRecorder poseRecorder; private final transient AutoBone autoBone; - + private transient Thread recordingThread = null; private transient Thread saveRecordingThread = null; private transient Thread autoBoneThread = null; - + private JButton saveRecordingButton; private JButton adjustButton; private JButton applyButton; - + private JLabel processLabel; private JLabel lengthsLabel; - + public AutoBoneWindow(VRServer server, SkeletonConfig skeletonConfig) { super("Skeleton Auto-Configuration"); - + this.server = server; this.skeletonConfig = skeletonConfig; this.poseRecorder = new PoseRecorder(server); this.autoBone = new AutoBone(server); - + getContentPane().setLayout(new BoxLayout(getContentPane(), BoxLayout.PAGE_AXIS)); add(new JScrollPane(pane = new EJBox(BoxLayout.PAGE_AXIS), ScrollPaneConstants.VERTICAL_SCROLLBAR_AS_NEEDED, ScrollPaneConstants.HORIZONTAL_SCROLLBAR_AS_NEEDED)); - + build(); } - + private String getLengthsString() { boolean first = true; StringBuilder configInfo = new StringBuilder(""); @@ -75,21 +75,21 @@ private String getLengthsString() { } else { first = false; } - + configInfo.append(entry.getKey() + ": " + StringUtils.prettyNumber(entry.getValue() * 100f, 2)); } - + return configInfo.toString(); } - - private void saveRecording(PoseFrame frames) { + + private void saveRecording(PoseFrames frames) { if(saveDir.isDirectory() || saveDir.mkdirs()) { File saveRecording; int recordingIndex = 1; do { saveRecording = new File(saveDir, "ABRecording" + recordingIndex++ + ".pfr"); } while(saveRecording.exists()); - + LogManager.log.info("[AutoBone] Exporting frames to \"" + saveRecording.getPath() + "\"..."); if(PoseFrameIO.writeToFile(saveRecording, frames)) { LogManager.log.info("[AutoBone] Done exporting! Recording can be found at \"" + saveRecording.getPath() + "\"."); @@ -100,17 +100,17 @@ private void saveRecording(PoseFrame frames) { LogManager.log.severe("[AutoBone] Failed to create the recording directory \"" + saveDir.getPath() + "\"."); } } - - private List> loadRecordings() { - List> recordings = new FastList>(); + + private List> loadRecordings() { + List> recordings = new FastList>(); if(loadDir.isDirectory()) { File[] files = loadDir.listFiles(); if(files != null) { for(File file : files) { if(file.isFile() && org.apache.commons.lang3.StringUtils.endsWithIgnoreCase(file.getName(), ".pfr")) { LogManager.log.info("[AutoBone] Detected recording at \"" + file.getPath() + "\", loading frames..."); - PoseFrame frames = PoseFrameIO.readFromFile(file); - + PoseFrames frames = PoseFrameIO.readFromFile(file); + if(frames == null) { LogManager.log.severe("Reading frames from \"" + file.getPath() + "\" failed..."); } else { @@ -120,26 +120,26 @@ private List> loadRecordings() { } } } - + return recordings; } - - private float processFrames(PoseFrame frames) { + + private float processFrames(PoseFrames frames) { autoBone.minDataDistance = server.config.getInt("autobone.minimumDataDistance", autoBone.minDataDistance); autoBone.maxDataDistance = server.config.getInt("autobone.maximumDataDistance", autoBone.maxDataDistance); - + autoBone.numEpochs = server.config.getInt("autobone.epochCount", autoBone.numEpochs); - + autoBone.initialAdjustRate = server.config.getFloat("autobone.adjustRate", autoBone.initialAdjustRate); autoBone.adjustRateDecay = server.config.getFloat("autobone.adjustRateDecay", autoBone.adjustRateDecay); - + autoBone.slideErrorFactor = server.config.getFloat("autobone.slideErrorFactor", autoBone.slideErrorFactor); autoBone.offsetErrorFactor = server.config.getFloat("autobone.offsetErrorFactor", autoBone.offsetErrorFactor); autoBone.proportionErrorFactor = server.config.getFloat("autobone.proportionErrorFactor", autoBone.proportionErrorFactor); autoBone.heightErrorFactor = server.config.getFloat("autobone.heightErrorFactor", autoBone.heightErrorFactor); autoBone.positionErrorFactor = server.config.getFloat("autobone.positionErrorFactor", autoBone.positionErrorFactor); autoBone.positionOffsetErrorFactor = server.config.getFloat("autobone.positionOffsetErrorFactor", autoBone.positionOffsetErrorFactor); - + boolean calcInitError = server.config.getBoolean("autobone.calculateInitialError", true); float targetHeight = server.config.getFloat("autobone.manualTargetHeight", -1f); return autoBone.processFrames(frames, calcInitError, targetHeight, (epoch) -> { @@ -147,7 +147,7 @@ private float processFrames(PoseFrame frames) { lengthsLabel.setText(getLengthsString()); }); } - + @AWTThread private void build() { pane.add(new EJBox(BoxLayout.LINE_AXIS) { @@ -162,7 +162,7 @@ public void mouseClicked(MouseEvent e) { if(!isEnabled() || recordingThread != null) { return; } - + Thread thread = new Thread() { @Override public void run() { @@ -172,13 +172,13 @@ public void run() { // 1000 samples at 20 ms per sample is 20 seconds int sampleCount = server.config.getInt("autobone.sampleCount", 1000); long sampleRate = server.config.getLong("autobone.sampleRateMs", 20L); - Future framesFuture = poseRecorder.startFrameRecording(sampleCount, sampleRate); - PoseFrame frames = framesFuture.get(); + Future framesFuture = poseRecorder.startFrameRecording(sampleCount, sampleRate); + PoseFrames frames = framesFuture.get(); LogManager.log.info("[AutoBone] Done recording!"); - + saveRecordingButton.setEnabled(true); adjustButton.setEnabled(true); - + if(server.config.getBoolean("autobone.saveRecordings", false)) { setText("Saving..."); saveRecording(frames); @@ -203,14 +203,14 @@ public void run() { } } }; - + recordingThread = thread; thread.start(); } }); } }); - + add(saveRecordingButton = new JButton("Save Recording") { { setEnabled(poseRecorder.hasRecording()); @@ -221,27 +221,27 @@ public void mouseClicked(MouseEvent e) { if(!isEnabled() || saveRecordingThread != null) { return; } - + Thread thread = new Thread() { @Override public void run() { try { - Future framesFuture = poseRecorder.getFramesAsync(); + Future framesFuture = poseRecorder.getFramesAsync(); if(framesFuture != null) { setText("Waiting for Recording..."); - PoseFrame frames = framesFuture.get(); - + PoseFrames frames = framesFuture.get(); + if(frames.getTrackerCount() <= 0) { throw new IllegalStateException("Recording has no trackers"); } - + if(frames.getMaxFrameCount() <= 0) { throw new IllegalStateException("Recording has no frames"); } - + setText("Saving..."); saveRecording(frames); - + setText("Recording Saved!"); try { Thread.sleep(3000); // Wait for 3 seconds @@ -272,14 +272,14 @@ public void run() { } } }; - + saveRecordingThread = thread; thread.start(); } }); } }); - + add(adjustButton = new JButton("Auto-Adjust") { { // If there are files to load, enable the button @@ -291,30 +291,30 @@ public void mouseClicked(MouseEvent e) { if(!isEnabled() || autoBoneThread != null) { return; } - + Thread thread = new Thread() { @Override public void run() { try { setText("Load..."); - List> frameRecordings = loadRecordings(); - + List> frameRecordings = loadRecordings(); + if(!frameRecordings.isEmpty()) { LogManager.log.info("[AutoBone] Done loading frames!"); } else { - Future framesFuture = poseRecorder.getFramesAsync(); + Future framesFuture = poseRecorder.getFramesAsync(); if(framesFuture != null) { setText("Waiting for Recording..."); - PoseFrame frames = framesFuture.get(); - + PoseFrames frames = framesFuture.get(); + if(frames.getTrackerCount() <= 0) { throw new IllegalStateException("Recording has no trackers"); } - + if(frames.getMaxFrameCount() <= 0) { throw new IllegalStateException("Recording has no frames"); } - + frameRecordings.add(Pair.of("", frames)); } else { setText("No Recordings..."); @@ -327,17 +327,17 @@ public void run() { return; } } - + setText("Processing..."); LogManager.log.info("[AutoBone] Processing frames..."); FastList heightPercentError = new FastList(frameRecordings.size()); - for(Pair recording : frameRecordings) { + for(Pair recording : frameRecordings) { LogManager.log.info("[AutoBone] Processing frames from \"" + recording.getKey() + "\"..."); - + heightPercentError.add(processFrames(recording.getValue())); LogManager.log.info("[AutoBone] Done processing!"); applyButton.setEnabled(true); - + //#region Stats/Values Float neckLength = autoBone.getConfig("Neck"); Float chestLength = autoBone.getConfig("Chest"); @@ -345,35 +345,35 @@ public void run() { Float hipWidth = autoBone.getConfig("Hips width"); Float legsLength = autoBone.getConfig("Legs length"); Float kneeHeight = autoBone.getConfig("Knee height"); - + float neckWaist = neckLength != null && waistLength != null ? neckLength / waistLength : 0f; float chestWaist = chestLength != null && waistLength != null ? chestLength / waistLength : 0f; float hipWaist = hipWidth != null && waistLength != null ? hipWidth / waistLength : 0f; float legWaist = legsLength != null && waistLength != null ? legsLength / waistLength : 0f; float legBody = legsLength != null && waistLength != null && neckLength != null ? legsLength / (waistLength + neckLength) : 0f; float kneeLeg = kneeHeight != null && legsLength != null ? kneeHeight / legsLength : 0f; - + LogManager.log.info("[AutoBone] Ratios: [{Neck-Waist: " + StringUtils.prettyNumber(neckWaist) + "}, {Chest-Waist: " + StringUtils.prettyNumber(chestWaist) + "}, {Hip-Waist: " + StringUtils.prettyNumber(hipWaist) + "}, {Leg-Waist: " + StringUtils.prettyNumber(legWaist) + "}, {Leg-Body: " + StringUtils.prettyNumber(legBody) + "}, {Knee-Leg: " + StringUtils.prettyNumber(kneeLeg) + "}]"); - + String lengthsString = getLengthsString(); LogManager.log.info("[AutoBone] Length values: " + lengthsString); lengthsLabel.setText(lengthsString); } - + if(!heightPercentError.isEmpty()) { float mean = 0f; for(float val : heightPercentError) { mean += val; } mean /= heightPercentError.size(); - + float std = 0f; for(float val : heightPercentError) { float stdVal = val - mean; std += stdVal * stdVal; } std = (float) Math.sqrt(std / heightPercentError.size()); - + LogManager.log.info("[AutoBone] Average height error: " + StringUtils.prettyNumber(mean, 6) + " (SD " + StringUtils.prettyNumber(std, 6) + ")"); } //#endregion @@ -391,14 +391,14 @@ public void run() { } } }; - + autoBoneThread = thread; thread.start(); } }); } }); - + add(applyButton = new JButton("Apply Values") { { setEnabled(false); @@ -408,7 +408,7 @@ public void mouseClicked(MouseEvent e) { if(!isEnabled()) { return; } - + autoBone.applyConfig(); // Update GUI values after applying skeletonConfig.refreshAll(); @@ -418,21 +418,21 @@ public void mouseClicked(MouseEvent e) { }); } }); - + pane.add(new EJBox(BoxLayout.LINE_AXIS) { { setBorder(new EmptyBorder(i(5))); add(processLabel = new JLabel("Processing has not been started...")); } }); - + pane.add(new EJBox(BoxLayout.LINE_AXIS) { { setBorder(new EmptyBorder(i(5))); add(lengthsLabel = new JLabel(getLengthsString())); } }); - + // Pack and display pack(); setLocationRelativeTo(null); diff --git a/src/main/java/dev/slimevr/poserecorder/PoseFrameIO.java b/src/main/java/dev/slimevr/poserecorder/PoseFrameIO.java index 9b8e3bad0a..d7cd52449c 100644 --- a/src/main/java/dev/slimevr/poserecorder/PoseFrameIO.java +++ b/src/main/java/dev/slimevr/poserecorder/PoseFrameIO.java @@ -16,17 +16,17 @@ import io.eiren.vr.trackers.TrackerPosition; public final class PoseFrameIO { - + private PoseFrameIO() { // Do not allow instantiating } - - public static boolean writeFrames(DataOutputStream outputStream, PoseFrame frames) { + + public static boolean writeFrames(DataOutputStream outputStream, PoseFrames frames) { try { if(frames != null) { outputStream.writeInt(frames.getTrackerCount()); for(PoseFrameTracker tracker : frames.getTrackers()) { - + outputStream.writeUTF(tracker.name); outputStream.writeInt(tracker.getFrameCount()); for(int i = 0; i < tracker.getFrameCount(); i++) { @@ -35,20 +35,20 @@ public static boolean writeFrames(DataOutputStream outputStream, PoseFrame frame outputStream.writeInt(0); continue; } - + outputStream.writeInt(trackerFrame.getDataFlags()); - + if(trackerFrame.hasData(TrackerFrameData.DESIGNATION)) { outputStream.writeUTF(trackerFrame.designation.designation); } - + if(trackerFrame.hasData(TrackerFrameData.ROTATION)) { outputStream.writeFloat(trackerFrame.rotation.getX()); outputStream.writeFloat(trackerFrame.rotation.getY()); outputStream.writeFloat(trackerFrame.rotation.getZ()); outputStream.writeFloat(trackerFrame.rotation.getW()); } - + if(trackerFrame.hasData(TrackerFrameData.POSITION)) { outputStream.writeFloat(trackerFrame.position.getX()); outputStream.writeFloat(trackerFrame.position.getY()); @@ -63,39 +63,39 @@ public static boolean writeFrames(DataOutputStream outputStream, PoseFrame frame LogManager.log.severe("Error writing frame to stream", e); return false; } - + return true; } - - public static boolean writeToFile(File file, PoseFrame frames) { + + public static boolean writeToFile(File file, PoseFrames frames) { try(DataOutputStream outputStream = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(file)))) { writeFrames(outputStream, frames); } catch(Exception e) { LogManager.log.severe("Error writing frames to file", e); return false; } - + return true; } - - public static PoseFrame readFrames(DataInputStream inputStream) { + + public static PoseFrames readFrames(DataInputStream inputStream) { try { - + int trackerCount = inputStream.readInt(); FastList trackers = new FastList(trackerCount); for(int i = 0; i < trackerCount; i++) { - + String name = inputStream.readUTF(); int trackerFrameCount = inputStream.readInt(); FastList trackerFrames = new FastList(trackerFrameCount); for(int j = 0; j < trackerFrameCount; j++) { int dataFlags = inputStream.readInt(); - + TrackerPosition designation = null; if(TrackerFrameData.DESIGNATION.check(dataFlags)) { designation = TrackerPosition.getByDesignation(inputStream.readUTF()); } - + Quaternion rotation = null; if(TrackerFrameData.ROTATION.check(dataFlags)) { float quatX = inputStream.readFloat(); @@ -104,7 +104,7 @@ public static PoseFrame readFrames(DataInputStream inputStream) { float quatW = inputStream.readFloat(); rotation = new Quaternion(quatX, quatY, quatZ, quatW); } - + Vector3f position = null; if(TrackerFrameData.POSITION.check(dataFlags)) { float posX = inputStream.readFloat(); @@ -112,28 +112,28 @@ public static PoseFrame readFrames(DataInputStream inputStream) { float posZ = inputStream.readFloat(); position = new Vector3f(posX, posY, posZ); } - + trackerFrames.add(new TrackerFrame(designation, rotation, position)); } - + trackers.add(new PoseFrameTracker(name, trackerFrames)); } - - return new PoseFrame(trackers); + + return new PoseFrames(trackers); } catch(Exception e) { LogManager.log.severe("Error reading frame from stream", e); } - + return null; } - - public static PoseFrame readFromFile(File file) { + + public static PoseFrames readFromFile(File file) { try { return readFrames(new DataInputStream(new BufferedInputStream(new FileInputStream(file)))); } catch(Exception e) { LogManager.log.severe("Error reading frame from file", e); } - + return null; } } diff --git a/src/main/java/dev/slimevr/poserecorder/PoseFrame.java b/src/main/java/dev/slimevr/poserecorder/PoseFrames.java similarity index 89% rename from src/main/java/dev/slimevr/poserecorder/PoseFrame.java rename to src/main/java/dev/slimevr/poserecorder/PoseFrames.java index 0f8cba4ea3..2779dcb556 100644 --- a/src/main/java/dev/slimevr/poserecorder/PoseFrame.java +++ b/src/main/java/dev/slimevr/poserecorder/PoseFrames.java @@ -7,73 +7,73 @@ import io.eiren.util.collections.FastList; import io.eiren.vr.trackers.Tracker; -public final class PoseFrame implements Iterable { - +public final class PoseFrames implements Iterable { + private final FastList trackers; - - public PoseFrame(FastList trackers) { + + public PoseFrames(FastList trackers) { this.trackers = trackers; } - - public PoseFrame(int initialCapacity) { + + public PoseFrames(int initialCapacity) { this.trackers = new FastList(initialCapacity); } - - public PoseFrame() { + + public PoseFrames() { this(5); } - + public PoseFrameTracker addTracker(PoseFrameTracker tracker) { trackers.add(tracker); return tracker; } - + public PoseFrameTracker addTracker(Tracker tracker, int initialCapacity) { return addTracker(new PoseFrameTracker(tracker.getName(), initialCapacity)); } - + public PoseFrameTracker addTracker(Tracker tracker) { return addTracker(tracker, 5); } - + public PoseFrameTracker removeTracker(int index) { return trackers.remove(index); } - + public PoseFrameTracker removeTracker(PoseFrameTracker tracker) { trackers.remove(tracker); return tracker; } - + public void clearTrackers() { trackers.clear(); } - + public void fakeClearTrackers() { trackers.fakeClear(); } - + public int getTrackerCount() { return trackers.size(); } - + public List getTrackers() { return trackers; } - + public int getMaxFrameCount() { int maxFrames = 0; - + for(int i = 0; i < trackers.size(); i++) { PoseFrameTracker tracker = trackers.get(i); if(tracker != null && tracker.getFrameCount() > maxFrames) { maxFrames = tracker.getFrameCount(); } } - + return maxFrames; } - + public int getFrames(int frameIndex, TrackerFrame[] buffer) { for(int i = 0; i < trackers.size(); i++) { PoseFrameTracker tracker = trackers.get(i); @@ -81,7 +81,7 @@ public int getFrames(int frameIndex, TrackerFrame[] buffer) { } return trackers.size(); } - + public int getFrames(int frameIndex, List buffer) { for(int i = 0; i < trackers.size(); i++) { PoseFrameTracker tracker = trackers.get(i); @@ -89,54 +89,54 @@ public int getFrames(int frameIndex, List buffer) { } return trackers.size(); } - + public TrackerFrame[] getFrames(int frameIndex) { TrackerFrame[] trackerFrames = new TrackerFrame[trackers.size()]; getFrames(frameIndex, trackerFrames); return trackerFrames; } - + @Override public Iterator iterator() { return new PoseFrameIterator(this); } - + public class PoseFrameIterator implements Iterator { - - private final PoseFrame poseFrame; + + private final PoseFrames poseFrame; private final TrackerFrame[] trackerFrameBuffer; - + private int cursor = 0; - - public PoseFrameIterator(PoseFrame poseFrame) { + + public PoseFrameIterator(PoseFrames poseFrame) { this.poseFrame = poseFrame; trackerFrameBuffer = new TrackerFrame[poseFrame.getTrackerCount()]; } - + @Override public boolean hasNext() { if(trackers.isEmpty()) { return false; } - + for(int i = 0; i < trackers.size(); i++) { PoseFrameTracker tracker = trackers.get(i); if(tracker != null && cursor < tracker.getFrameCount()) { return true; } } - + return false; } - + @Override public TrackerFrame[] next() { if(!hasNext()) { throw new NoSuchElementException(); } - + poseFrame.getFrames(cursor++, trackerFrameBuffer); - + return trackerFrameBuffer; } } diff --git a/src/main/java/dev/slimevr/poserecorder/PoseRecorder.java b/src/main/java/dev/slimevr/poserecorder/PoseRecorder.java index f46f3ff99f..12041ee358 100644 --- a/src/main/java/dev/slimevr/poserecorder/PoseRecorder.java +++ b/src/main/java/dev/slimevr/poserecorder/PoseRecorder.java @@ -14,28 +14,28 @@ import io.eiren.vr.trackers.Tracker; public class PoseRecorder { - - protected PoseFrame poseFrame = null; - + + protected PoseFrames poseFrame = null; + protected int numFrames = -1; protected int frameCursor = 0; protected long frameRecordingInterval = 60L; protected long nextFrameTimeMs = -1L; - - protected CompletableFuture currentRecording; - + + protected CompletableFuture currentRecording; + protected final VRServer server; FastList> trackers = new FastList>(); - + public PoseRecorder(VRServer server) { this.server = server; server.addOnTick(this::onTick); } - + @VRServerThread public void onTick() { if(numFrames > 0) { - PoseFrame poseFrame = this.poseFrame; + PoseFrames poseFrame = this.poseFrame; List> trackers = this.trackers; if (poseFrame != null && trackers != null) { if (frameCursor < numFrames) { @@ -53,7 +53,7 @@ public void onTick() { // Add a frame for each tracker tracker.getRight().addFrame(cursor, tracker.getLeft()); } - + // If done, send finished recording if(frameCursor >= numFrames) { internalStopRecording(); @@ -66,12 +66,12 @@ public void onTick() { } } } - - public synchronized Future startFrameRecording(int numFrames, long interval) { + + public synchronized Future startFrameRecording(int numFrames, long interval) { return startFrameRecording(numFrames, interval, server.getAllTrackers()); } - - public synchronized Future startFrameRecording(int numFrames, long interval, List trackers) { + + public synchronized Future startFrameRecording(int numFrames, long interval, List trackers) { if(numFrames < 1) { throw new IllegalArgumentException("numFrames must at least have a value of 1"); } @@ -87,11 +87,11 @@ public synchronized Future startFrameRecording(int numFrames, long in if(!isReadyToRecord()) { throw new IllegalStateException("PoseRecorder isn't ready to record!"); } - + cancelFrameRecording(); - - poseFrame = new PoseFrame(trackers.size()); - + + poseFrame = new PoseFrames(trackers.size()); + // Update tracker list this.trackers.ensureCapacity(trackers.size()); for(Tracker tracker : trackers) { @@ -99,71 +99,71 @@ public synchronized Future startFrameRecording(int numFrames, long in if(tracker == null || tracker.isComputed()) { continue; } - + // Pair tracker with recording this.trackers.add(Pair.of(tracker, poseFrame.addTracker(tracker, numFrames))); } - + this.frameCursor = 0; this.numFrames = numFrames; - + frameRecordingInterval = interval; nextFrameTimeMs = -1L; - + LogManager.log.info("[PoseRecorder] Recording " + numFrames + " samples at a " + interval + " ms frame interval"); - - currentRecording = new CompletableFuture(); + + currentRecording = new CompletableFuture(); return currentRecording; } - + private void internalStopRecording() { - CompletableFuture currentRecording = this.currentRecording; + CompletableFuture currentRecording = this.currentRecording; if(currentRecording != null && !currentRecording.isDone()) { // Stop the recording, returning the frames recorded currentRecording.complete(poseFrame); } - + numFrames = -1; frameCursor = 0; trackers.clear(); poseFrame = null; } - + public synchronized void stopFrameRecording() { internalStopRecording(); } - + public synchronized void cancelFrameRecording() { - CompletableFuture currentRecording = this.currentRecording; + CompletableFuture currentRecording = this.currentRecording; if(currentRecording != null && !currentRecording.isDone()) { // Cancel the current recording and return nothing currentRecording.cancel(true); } - + numFrames = -1; frameCursor = 0; trackers.clear(); poseFrame = null; } - + public boolean isReadyToRecord() { return server.getTrackersCount() > 0; } - + public boolean isRecording() { return numFrames > frameCursor; } - + public boolean hasRecording() { return currentRecording != null; } - - public Future getFramesAsync() { + + public Future getFramesAsync() { return currentRecording; } - - public PoseFrame getFrames() throws ExecutionException, InterruptedException { - CompletableFuture currentRecording = this.currentRecording; + + public PoseFrames getFrames() throws ExecutionException, InterruptedException { + CompletableFuture currentRecording = this.currentRecording; return currentRecording != null ? currentRecording.get() : null; } } From 472fcab82155460ad3e7a656a178014cf92db1c7 Mon Sep 17 00:00:00 2001 From: ButterscotchVanilla Date: Wed, 22 Sep 2021 00:49:46 -0400 Subject: [PATCH 3/9] Add a basic PoseStreamer implementation for streaming mocap data --- .../slimevr/poserecorder/PoseFileStream.java | 39 ++++++++ .../slimevr/poserecorder/PoseRecorder.java | 14 +-- .../slimevr/poserecorder/PoseStreamer.java | 94 +++++++++++++++++++ 3 files changed, 140 insertions(+), 7 deletions(-) create mode 100644 src/main/java/dev/slimevr/poserecorder/PoseFileStream.java create mode 100644 src/main/java/dev/slimevr/poserecorder/PoseStreamer.java diff --git a/src/main/java/dev/slimevr/poserecorder/PoseFileStream.java b/src/main/java/dev/slimevr/poserecorder/PoseFileStream.java new file mode 100644 index 0000000000..02c066f67a --- /dev/null +++ b/src/main/java/dev/slimevr/poserecorder/PoseFileStream.java @@ -0,0 +1,39 @@ +package dev.slimevr.poserecorder; + +import java.io.BufferedOutputStream; +import java.io.DataOutputStream; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; + +import io.eiren.vr.processor.HumanSkeleton; + +public abstract class PoseFileStream implements AutoCloseable { + + protected DataOutputStream dataStream; + + protected PoseFileStream(OutputStream outputStream) { + this.dataStream = new DataOutputStream(new BufferedOutputStream(outputStream)); + } + + protected PoseFileStream(File file) throws FileNotFoundException { + this(new FileOutputStream(file)); + } + + protected PoseFileStream(String file) throws FileNotFoundException { + this(new FileOutputStream(file)); + } + + abstract boolean writeHeader(HumanSkeleton skeleton) throws IOException; + + abstract boolean writeFrame(HumanSkeleton skeleton) throws IOException; + + abstract boolean writeFooter(HumanSkeleton skeleton) throws IOException; + + @Override + public void close() throws IOException { + dataStream.close(); + } +} diff --git a/src/main/java/dev/slimevr/poserecorder/PoseRecorder.java b/src/main/java/dev/slimevr/poserecorder/PoseRecorder.java index 12041ee358..2aa5955f6e 100644 --- a/src/main/java/dev/slimevr/poserecorder/PoseRecorder.java +++ b/src/main/java/dev/slimevr/poserecorder/PoseRecorder.java @@ -67,16 +67,16 @@ public void onTick() { } } - public synchronized Future startFrameRecording(int numFrames, long interval) { - return startFrameRecording(numFrames, interval, server.getAllTrackers()); + public synchronized Future startFrameRecording(int numFrames, long intervalMs) { + return startFrameRecording(numFrames, intervalMs, server.getAllTrackers()); } - public synchronized Future startFrameRecording(int numFrames, long interval, List trackers) { + public synchronized Future startFrameRecording(int numFrames, long intervalMs, List trackers) { if(numFrames < 1) { throw new IllegalArgumentException("numFrames must at least have a value of 1"); } - if(interval < 1) { - throw new IllegalArgumentException("interval must at least have a value of 1"); + if(intervalMs < 1) { + throw new IllegalArgumentException("intervalMs must at least have a value of 1"); } if(trackers == null) { throw new IllegalArgumentException("trackers must not be null"); @@ -107,10 +107,10 @@ public synchronized Future startFrameRecording(int numFrames, long i this.frameCursor = 0; this.numFrames = numFrames; - frameRecordingInterval = interval; + frameRecordingInterval = intervalMs; nextFrameTimeMs = -1L; - LogManager.log.info("[PoseRecorder] Recording " + numFrames + " samples at a " + interval + " ms frame interval"); + LogManager.log.info("[PoseRecorder] Recording " + numFrames + " samples at a " + intervalMs + " ms frame interval"); currentRecording = new CompletableFuture(); return currentRecording; diff --git a/src/main/java/dev/slimevr/poserecorder/PoseStreamer.java b/src/main/java/dev/slimevr/poserecorder/PoseStreamer.java new file mode 100644 index 0000000000..2f2fcca47a --- /dev/null +++ b/src/main/java/dev/slimevr/poserecorder/PoseStreamer.java @@ -0,0 +1,94 @@ +package dev.slimevr.poserecorder; + +import java.io.IOException; + +import io.eiren.util.ann.VRServerThread; +import io.eiren.util.logging.LogManager; +import io.eiren.vr.VRServer; +import io.eiren.vr.processor.HumanSkeleton; + +public class PoseStreamer { + + protected long frameRecordingInterval = 60L; + protected long nextFrameTimeMs = -1L; + + private HumanSkeleton skeleton; + private PoseFileStream poseFileStream; + + protected final VRServer server; + + public PoseStreamer(VRServer server) { + this.server = server; + + // Register callbacks/events + server.addSkeletonUpdatedCallback(this::onSkeletonUpdated); + server.addOnTick(this::onTick); + } + + @VRServerThread + public void onSkeletonUpdated(HumanSkeleton skeleton) { + this.skeleton = skeleton; + } + + @VRServerThread + public void onTick() { + PoseFileStream poseFileStream = this.poseFileStream; + if (poseFileStream != null) { + + long curTime = System.currentTimeMillis(); + if (curTime >= nextFrameTimeMs) { + nextFrameTimeMs += frameRecordingInterval; + + // To prevent duplicate frames, make sure the frame time is always in the future + if (nextFrameTimeMs <= curTime) { + nextFrameTimeMs = curTime + frameRecordingInterval; + } + + try { + poseFileStream.writeFrame(skeleton); + } catch (Exception e) { + // Handle any exceptions without crashing the program + LogManager.log.severe("[PoseStreamer] Exception while saving frame", e); + } + } + } + } + + public void setFrameInterval(long intervalMs) { + if(intervalMs < 1) { + throw new IllegalArgumentException("intervalMs must at least have a value of 1"); + } + + this.frameRecordingInterval = intervalMs; + } + + public void setOutput(PoseFileStream poseFileStream) throws IOException { + poseFileStream.writeHeader(skeleton); + this.poseFileStream = poseFileStream; + nextFrameTimeMs = -1L; // Reset the frame timing + } + + public PoseFileStream getOutput() { + return poseFileStream; + } + + public void closeOutput() { + PoseFileStream poseFileStream = this.poseFileStream; + + if (poseFileStream != null) { + try { + poseFileStream.writeFooter(skeleton); + } catch (Exception e) { + // Ignore + LogManager.log.severe("[PoseStreamer] Exception while writing file footer", e); + } + + try { + poseFileStream.close(); + } catch (Exception e) { + // Ignore + LogManager.log.severe("[PoseStreamer] Exception while closing file stream", e); + } + } + } +} From a326d76f6a1e1388acb8b4059c0c9f458cbd45ec Mon Sep 17 00:00:00 2001 From: ButterscotchVanilla Date: Fri, 24 Sep 2021 05:10:08 -0400 Subject: [PATCH 4/9] Add basic BVH file streamer --- .editorconfig | 4 +- .../slimevr/poserecorder/BVHFileStream.java | 184 ++++++++++++++++++ .../slimevr/poserecorder/PoseFileStream.java | 19 +- .../slimevr/poserecorder/PoseStreamer.java | 27 ++- 4 files changed, 208 insertions(+), 26 deletions(-) create mode 100644 src/main/java/dev/slimevr/poserecorder/BVHFileStream.java diff --git a/.editorconfig b/.editorconfig index 1a00f79d3e..379165a0aa 100644 --- a/.editorconfig +++ b/.editorconfig @@ -7,5 +7,5 @@ root = true indent_style = tab end_of_line = lf charset = utf-8 -trim_trailing_whitespace = true -insert_final_newline = true \ No newline at end of file +#trim_trailing_whitespace = true +insert_final_newline = true diff --git a/src/main/java/dev/slimevr/poserecorder/BVHFileStream.java b/src/main/java/dev/slimevr/poserecorder/BVHFileStream.java new file mode 100644 index 0000000000..00d8a832ed --- /dev/null +++ b/src/main/java/dev/slimevr/poserecorder/BVHFileStream.java @@ -0,0 +1,184 @@ +package dev.slimevr.poserecorder; + +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.io.OutputStreamWriter; + +import com.jme3.math.Quaternion; +import com.jme3.math.Vector3f; + +import org.apache.commons.lang3.StringUtils; + +import io.eiren.vr.processor.HumanSkeleton; +import io.eiren.vr.processor.TransformNode; + +public class BVHFileStream extends PoseFileStream { + + private static final int LONG_MAX_VALUE_DIGITS = Long.toString(Long.MAX_VALUE).length(); + private static final float POS_SCALE = 10f; + + private long frameCount = 0; + private final BufferedWriter writer; + + private long frameCountOffset; + + private float[] angleBuf = new float[3]; + private Quaternion rotBuf = new Quaternion(); + + public BVHFileStream(OutputStream outputStream) { + super(outputStream); + writer = new BufferedWriter(new OutputStreamWriter(outputStream), 128); + } + + public BVHFileStream(File file) throws FileNotFoundException { + super(file); + writer = new BufferedWriter(new OutputStreamWriter(outputStream), 128); + } + + public BVHFileStream(String file) throws FileNotFoundException { + super(file); + writer = new BufferedWriter(new OutputStreamWriter(outputStream), 128); + } + + private String getBufferedFrameCount(long frameCount) { + String frameString = Long.toString(frameCount); + int bufferCount = LONG_MAX_VALUE_DIGITS - frameString.length(); + + return bufferCount > 0 ? frameString + StringUtils.repeat(' ', bufferCount) : frameString; + } + + private void writeTransformNodeHierarchy(TransformNode node) throws IOException { + writeTransformNodeHierarchy(node, 0); + } + + private void writeTransformNodeHierarchy(TransformNode node, int level) throws IOException { + String indentLevel = StringUtils.repeat("\t", level); + String nextIndentLevel = indentLevel + "\t"; + + // Handle ends + if (node.children.isEmpty()) { + writer.write(indentLevel + "End Site\n"); + } else { + writer.write((level > 0 ? indentLevel + "JOINT " : "ROOT ") + node.getName() + "\n"); + } + writer.write(indentLevel + "{\n"); + + if (level > 0) { + Vector3f offset = node.localTransform.getTranslation(); + writer.write(nextIndentLevel + "OFFSET " + Float.toString(offset.getX() * POS_SCALE) + " " + Float.toString(offset.getY() * POS_SCALE) + " " + Float.toString(offset.getZ() * POS_SCALE) + "\n"); + } else { + writer.write(nextIndentLevel + "OFFSET 0.0 0.0 0.0\n"); + } + + // Handle ends + if (!node.children.isEmpty()) { + // Only give position for root + if (level > 0) { + writer.write(nextIndentLevel + "CHANNELS 3 Zrotation Xrotation Yrotation\n"); + } else { + writer.write(nextIndentLevel + "CHANNELS 6 Xposition Yposition Zposition Zrotation Xrotation Yrotation\n"); + } + + for (TransformNode childNode : node.children) { + writeTransformNodeHierarchy(childNode, level + 1); + } + } + + writer.write(indentLevel + "}\n"); + } + + @Override + public void writeHeader(HumanSkeleton skeleton, PoseStreamer streamer) throws IOException { + if (skeleton == null) { + throw new NullPointerException("skeleton must not be null"); + } + if (streamer == null) { + throw new NullPointerException("streamer must not be null"); + } + + writer.write("HIERARCHY\n"); + writeTransformNodeHierarchy(skeleton.getRootNode()); + + writer.write("MOTION\n"); + writer.write("Frames: "); + + // Get frame offset for finishing writing the file + if (parentStream instanceof FileOutputStream) { + FileOutputStream fileOutputStream = (FileOutputStream)parentStream; + // Flush buffer to get proper offset + writer.flush(); + frameCountOffset = fileOutputStream.getChannel().position(); + } + + writer.write(getBufferedFrameCount(frameCount) + "\n"); + + // Frame time in seconds + writer.write("Frame Time: " + (streamer.frameRecordingInterval / 1000d) + "\n"); + } + + private void writeTransformHierarchyRotation(TransformNode node, Quaternion inverseRootRot) throws IOException { + rotBuf = node.localTransform.getRotation(rotBuf); + + // Adjust to local rotation + if (inverseRootRot != null) { + rotBuf = inverseRootRot.mult(rotBuf, rotBuf); + } + + angleBuf = rotBuf.toAngles(angleBuf); + writer.write(Float.toString((float)Math.toDegrees(angleBuf[2])) + " " + Float.toString((float)Math.toDegrees(angleBuf[0])) + " " + Float.toString((float)Math.toDegrees(angleBuf[1]))); + + // Get inverse rotation for child local rotations + Quaternion inverseRot = node.localTransform.getRotation().inverse(); + for (TransformNode childNode : node.children) { + if (childNode.children.isEmpty()) { + // If it's an end node, skip + continue; + } + + // Add spacing + writer.write(" "); + writeTransformHierarchyRotation(childNode, inverseRot); + } + } + + @Override + public void writeFrame(HumanSkeleton skeleton) throws IOException { + if (skeleton == null) { + throw new NullPointerException("skeleton must not be null"); + } + + TransformNode root = skeleton.getRootNode(); + Vector3f rootPos = root.localTransform.getTranslation(); + + // Write root position + writer.write(Float.toString(rootPos.getX() * POS_SCALE) + " " + Float.toString(rootPos.getY() * POS_SCALE) + " " + Float.toString(rootPos.getZ() * POS_SCALE) + " "); + writeTransformHierarchyRotation(root, null); + + writer.newLine(); + + frameCount++; + } + + @Override + public void writeFooter(HumanSkeleton skeleton) throws IOException { + // Write the final frame count for files + if (parentStream instanceof FileOutputStream) { + FileOutputStream fileOutputStream = (FileOutputStream)parentStream; + // Flush before anything else + writer.flush(); + // Seek to the count offset + fileOutputStream.getChannel().position(frameCountOffset); + // Overwrite the count with a new value + writer.write(Long.toString(frameCount)); + } + } + + @Override + public void close() throws IOException { + writer.close(); + } +} diff --git a/src/main/java/dev/slimevr/poserecorder/PoseFileStream.java b/src/main/java/dev/slimevr/poserecorder/PoseFileStream.java index 02c066f67a..a1927f00b3 100644 --- a/src/main/java/dev/slimevr/poserecorder/PoseFileStream.java +++ b/src/main/java/dev/slimevr/poserecorder/PoseFileStream.java @@ -1,7 +1,6 @@ package dev.slimevr.poserecorder; import java.io.BufferedOutputStream; -import java.io.DataOutputStream; import java.io.File; import java.io.FileNotFoundException; import java.io.FileOutputStream; @@ -11,11 +10,13 @@ import io.eiren.vr.processor.HumanSkeleton; public abstract class PoseFileStream implements AutoCloseable { - - protected DataOutputStream dataStream; + + protected final OutputStream parentStream; + protected final BufferedOutputStream outputStream; protected PoseFileStream(OutputStream outputStream) { - this.dataStream = new DataOutputStream(new BufferedOutputStream(outputStream)); + this.parentStream = outputStream; + this.outputStream = new BufferedOutputStream(outputStream); } protected PoseFileStream(File file) throws FileNotFoundException { @@ -26,14 +27,16 @@ protected PoseFileStream(String file) throws FileNotFoundException { this(new FileOutputStream(file)); } - abstract boolean writeHeader(HumanSkeleton skeleton) throws IOException; + public void writeHeader(HumanSkeleton skeleton, PoseStreamer streamer) throws IOException { + } - abstract boolean writeFrame(HumanSkeleton skeleton) throws IOException; + abstract void writeFrame(HumanSkeleton skeleton) throws IOException; - abstract boolean writeFooter(HumanSkeleton skeleton) throws IOException; + public void writeFooter(HumanSkeleton skeleton) throws IOException { + } @Override public void close() throws IOException { - dataStream.close(); + outputStream.close(); } } diff --git a/src/main/java/dev/slimevr/poserecorder/PoseStreamer.java b/src/main/java/dev/slimevr/poserecorder/PoseStreamer.java index 2f2fcca47a..72d952e521 100644 --- a/src/main/java/dev/slimevr/poserecorder/PoseStreamer.java +++ b/src/main/java/dev/slimevr/poserecorder/PoseStreamer.java @@ -63,32 +63,27 @@ public void setFrameInterval(long intervalMs) { } public void setOutput(PoseFileStream poseFileStream) throws IOException { - poseFileStream.writeHeader(skeleton); + poseFileStream.writeHeader(skeleton, this); this.poseFileStream = poseFileStream; nextFrameTimeMs = -1L; // Reset the frame timing } + public void setOutput(PoseFileStream poseFileStream, long intervalMs) throws IOException { + setFrameInterval(intervalMs); + setOutput(poseFileStream); + } + public PoseFileStream getOutput() { return poseFileStream; } - public void closeOutput() { + public void closeOutput() throws IOException { PoseFileStream poseFileStream = this.poseFileStream; - - if (poseFileStream != null) { - try { - poseFileStream.writeFooter(skeleton); - } catch (Exception e) { - // Ignore - LogManager.log.severe("[PoseStreamer] Exception while writing file footer", e); - } - try { - poseFileStream.close(); - } catch (Exception e) { - // Ignore - LogManager.log.severe("[PoseStreamer] Exception while closing file stream", e); - } + if (poseFileStream != null) { + poseFileStream.writeFooter(skeleton); + poseFileStream.close(); + this.poseFileStream = null; } } } From c3fc5607ba1e5ab36a3b9e53401cae8ae0837039 Mon Sep 17 00:00:00 2001 From: ButterscotchVanilla Date: Wed, 13 Oct 2021 01:35:43 -0400 Subject: [PATCH 5/9] Explain the editorconfig change in a comment --- .editorconfig | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.editorconfig b/.editorconfig index 379165a0aa..e9a2391aee 100644 --- a/.editorconfig +++ b/.editorconfig @@ -7,5 +7,7 @@ root = true indent_style = tab end_of_line = lf charset = utf-8 +# This line causes problems with VSCode and potentially with other editors where all purely +# whitespace lines are trimmed to be empty when saved, causing excessive worthless changes with Git #trim_trailing_whitespace = true insert_final_newline = true From e1d3af073481e052360df6136288cc1874b7f850 Mon Sep 17 00:00:00 2001 From: ButterscotchVanilla Date: Wed, 13 Oct 2021 02:38:30 -0400 Subject: [PATCH 6/9] Fix formatting --- .../java/dev/slimevr/autobone/AutoBone.java | 222 +++++++++--------- .../java/dev/slimevr/gui/AutoBoneWindow.java | 112 ++++----- .../dev/slimevr/poserecorder/PoseFrameIO.java | 44 ++-- .../dev/slimevr/poserecorder/PoseFrames.java | 60 ++--- 4 files changed, 219 insertions(+), 219 deletions(-) diff --git a/src/main/java/dev/slimevr/autobone/AutoBone.java b/src/main/java/dev/slimevr/autobone/AutoBone.java index 214a97593b..dfed39e9ec 100644 --- a/src/main/java/dev/slimevr/autobone/AutoBone.java +++ b/src/main/java/dev/slimevr/autobone/AutoBone.java @@ -21,78 +21,78 @@ import io.eiren.vr.trackers.TrackerUtils; public class AutoBone { - + public class Epoch { - + public final int epoch; public final float epochError; - + public Epoch(int epoch, float epochError) { this.epoch = epoch; this.epochError = epochError; } - + @Override public String toString() { return "Epoch: " + epoch + ", Epoch Error: " + epochError; } } - + public int cursorIncrement = 1; - + public int minDataDistance = 2; public int maxDataDistance = 32; - + public int numEpochs = 5; - + public float initialAdjustRate = 2.5f; public float adjustRateDecay = 1.01f; - + public float slideErrorFactor = 1.0f; public float offsetErrorFactor = 0.0f; public float proportionErrorFactor = 0.2f; public float heightErrorFactor = 0.1f; public float positionErrorFactor = 0.0f; public float positionOffsetErrorFactor = 0.0f; - + // Human average is probably 1.1235 (SD 0.07) public float legBodyRatio = 1.1235f; // SD of 0.07, capture 68% within range public float legBodyRatioRange = 0.07f; - + // Assume these to be approximately half public float kneeLegRatio = 0.5f; public float chestWaistRatio = 0.5f; - + protected final VRServer server; - + protected HumanSkeletonWithLegs skeleton = null; - + // This is filled by reloadConfigValues() public final HashMap configs = new HashMap(); public final HashMap staticConfigs = new HashMap(); - + public final FastList heightConfigs = new FastList(new String[]{"Neck", "Waist", "Legs length" }); - + public AutoBone(VRServer server) { this.server = server; - + reloadConfigValues(); - + server.addSkeletonUpdatedCallback(this::skeletonUpdated); } - + public void reloadConfigValues() { reloadConfigValues(null); } - + public void reloadConfigValues(TrackerFrame[] frame) { // Load waist configs staticConfigs.put("Head", server.config.getFloat("body.headShift", HumanSkeletonWithWaist.HEAD_SHIFT_DEFAULT)); staticConfigs.put("Neck", server.config.getFloat("body.neckLength", HumanSkeletonWithWaist.NECK_LENGTH_DEFAULT)); configs.put("Waist", server.config.getFloat("body.waistDistance", 0.85f)); - + if(server.config.getBoolean("autobone.forceChestTracker", false) || (frame != null && TrackerUtils.findTrackerForBodyPosition(frame, TrackerPosition.CHEST) != null) || TrackerUtils.findTrackerForBodyPosition(server.getAllTrackers(), TrackerPosition.CHEST) != null) { // If force enabled or has a chest tracker configs.put("Chest", server.config.getFloat("body.chestDistance", 0.42f)); @@ -101,13 +101,13 @@ public void reloadConfigValues(TrackerFrame[] frame) { configs.remove("Chest"); staticConfigs.put("Chest", server.config.getFloat("body.chestDistance", 0.42f)); } - + // Load leg configs staticConfigs.put("Hips width", server.config.getFloat("body.hipsWidth", HumanSkeletonWithLegs.HIPS_WIDTH_DEFAULT)); configs.put("Legs length", server.config.getFloat("body.legsLength", 0.84f)); configs.put("Knee height", server.config.getFloat("body.kneeHeight", 0.42f)); } - + @ThreadSafe public void skeletonUpdated(HumanSkeleton newSkeleton) { if(newSkeleton instanceof HumanSkeletonWithLegs) { @@ -116,34 +116,34 @@ public void skeletonUpdated(HumanSkeleton newSkeleton) { LogManager.log.info("[AutoBone] Received updated skeleton"); } } - + public void applyConfig() { if(!applyConfigToSkeleton(skeleton)) { // Unable to apply to skeleton, save directly saveConfigs(); } } - + public boolean applyConfigToSkeleton(HumanSkeleton skeleton) { if(skeleton == null) { return false; } - + configs.forEach(skeleton::setSkeletonConfig); - + server.saveConfig(); - + LogManager.log.info("[AutoBone] Configured skeleton bone lengths"); return true; } - + private void setConfig(String name, String path) { Float value = configs.get(name); if(value != null) { server.config.setProperty(path, value); } } - + // This doesn't require a skeleton, therefore can be used if skeleton is null public void saveConfigs() { setConfig("Head", "body.headShift"); @@ -153,51 +153,51 @@ public void saveConfigs() { setConfig("Hips width", "body.hipsWidth"); setConfig("Legs length", "body.legsLength"); setConfig("Knee height", "body.kneeHeight"); - + server.saveConfig(); } - + public Float getConfig(String config) { Float configVal = configs.get(config); return configVal != null ? configVal : staticConfigs.get(config); } - + public Float getConfig(String config, Map configs, Map configsAlt) { if(configs == null) { throw new NullPointerException("Argument \"configs\" must not be null"); } - + Float configVal = configs.get(config); return configVal != null || configsAlt == null ? configVal : configsAlt.get(config); } - + public float getHeight(Map configs) { return getHeight(configs, null); } - + public float getHeight(Map configs, Map configsAlt) { float height = 0f; - + for(String heightConfig : heightConfigs) { Float length = getConfig(heightConfig, configs, configsAlt); if(length != null) { height += length; } } - + return height; } - + public float getLengthSum(Map configs) { float length = 0f; - + for(float boneLength : configs.values()) { length += boneLength; } - + return length; } - + public float getMaxHmdHeight(PoseFrames frames) { float maxHeight = 0f; for(TrackerFrame[] frame : frames) { @@ -208,39 +208,39 @@ public float getMaxHmdHeight(PoseFrames frames) { } return maxHeight; } - + public void processFrames(PoseFrames frames) { processFrames(frames, -1f); } - + public void processFrames(PoseFrames frames, Consumer epochCallback) { processFrames(frames, -1f, epochCallback); } - + public void processFrames(PoseFrames frames, float targetHeight) { processFrames(frames, true, targetHeight); } - + public void processFrames(PoseFrames frames, float targetHeight, Consumer epochCallback) { processFrames(frames, true, targetHeight, epochCallback); } - + public float processFrames(PoseFrames frames, boolean calcInitError, float targetHeight) { return processFrames(frames, calcInitError, targetHeight, null); } - + public float processFrames(PoseFrames frames, boolean calcInitError, float targetHeight, Consumer epochCallback) { final int frameCount = frames.getMaxFrameCount(); - + final SimpleSkeleton skeleton1 = new SimpleSkeleton(configs, staticConfigs); final TrackerFrame[] trackerBuffer1 = new TrackerFrame[frames.getTrackerCount()]; - + frames.getFrames(0, trackerBuffer1); reloadConfigValues(trackerBuffer1); // Reload configs and detect chest tracker from the first frame - + final SimpleSkeleton skeleton2 = new SimpleSkeleton(configs, staticConfigs); final TrackerFrame[] trackerBuffer2 = new TrackerFrame[frames.getTrackerCount()]; - + // If target height isn't specified, auto-detect if(targetHeight < 0f) { if(skeleton != null) { @@ -253,62 +253,62 @@ public float processFrames(PoseFrames frames, boolean calcInitError, float targe } else { LogManager.log.info("[AutoBone] Max headset height detected: " + hmdHeight); } - + // Estimate target height from HMD height targetHeight = hmdHeight; } } - + for(int epoch = calcInitError ? -1 : 0; epoch < numEpochs; epoch++) { float sumError = 0f; int errorCount = 0; - + float adjustRate = epoch >= 0 ? (float) (initialAdjustRate / Math.pow(adjustRateDecay, epoch)) : 0f; - + for(int cursorOffset = minDataDistance; cursorOffset <= maxDataDistance && cursorOffset < frameCount; cursorOffset++) { for(int frameCursor = 0; frameCursor < frameCount - cursorOffset; frameCursor += cursorIncrement) { frames.getFrames(frameCursor, trackerBuffer1); frames.getFrames(frameCursor + cursorOffset, trackerBuffer2); - + skeleton1.setSkeletonConfigs(configs); skeleton2.setSkeletonConfigs(configs); - + skeleton1.setPoseFromFrame(trackerBuffer1); skeleton2.setPoseFromFrame(trackerBuffer2); - + float totalLength = getLengthSum(configs); float curHeight = getHeight(configs, staticConfigs); float errorDeriv = getErrorDeriv(trackerBuffer1, trackerBuffer2, skeleton1, skeleton2, targetHeight - curHeight); float error = errorFunc(errorDeriv); - + // In case of fire if(Float.isNaN(error) || Float.isInfinite(error)) { // Extinguish LogManager.log.warning("[AutoBone] Error value is invalid, resetting variables to recover"); reloadConfigValues(trackerBuffer1); - + // Reset error sum values sumError = 0f; errorCount = 0; - + // Continue on new data continue; } - + // Store the error count for logging purposes sumError += errorDeriv; errorCount++; - + float adjustVal = error * adjustRate; - + for(Entry entry : configs.entrySet()) { // Skip adjustment if the epoch is before starting (for logging only) if(epoch < 0) { break; } - + float originalLength = entry.getValue(); - + // Try positive and negative adjustments boolean isHeightVar = heightConfigs.contains(entry.getKey()); float minError = errorDeriv; @@ -317,78 +317,78 @@ public float processFrames(PoseFrames frames, boolean calcInitError, float targe // Scale by the ratio for smooth adjustment and more stable results float curAdjustVal = ((i == 0 ? adjustVal : -adjustVal) * originalLength) / totalLength; float newLength = originalLength + curAdjustVal; - + // No small or negative numbers!!! Bad algorithm! if(newLength < 0.01f) { continue; } - + updateSkeletonBoneLength(skeleton1, skeleton2, entry.getKey(), newLength); - + float newHeight = isHeightVar ? curHeight + curAdjustVal : curHeight; float newErrorDeriv = getErrorDeriv(trackerBuffer1, trackerBuffer2, skeleton1, skeleton2, targetHeight - newHeight); - + if(newErrorDeriv < minError) { minError = newErrorDeriv; finalNewLength = newLength; } } - + if(finalNewLength > 0f) { entry.setValue(finalNewLength); } - + // Reset the length to minimize bias in other variables, it's applied later updateSkeletonBoneLength(skeleton1, skeleton2, entry.getKey(), originalLength); } } } - + // Calculate average error over the epoch float avgError = errorCount > 0 ? sumError / errorCount : -1f; LogManager.log.info("[AutoBone] Epoch " + (epoch + 1) + " average error: " + avgError); - + if(epochCallback != null) { epochCallback.accept(new Epoch(epoch + 1, avgError)); } } - + float finalHeight = getHeight(configs, staticConfigs); LogManager.log.info("[AutoBone] Target height: " + targetHeight + " New height: " + finalHeight); - + return Math.abs(finalHeight - targetHeight); } - + // The change in position of the ankle over time protected float getSlideErrorDeriv(SimpleSkeleton skeleton1, SimpleSkeleton skeleton2) { float slideLeft = skeleton1.getNodePosition(TrackerPosition.LEFT_ANKLE).distance(skeleton2.getNodePosition(TrackerPosition.LEFT_ANKLE)); float slideRight = skeleton1.getNodePosition(TrackerPosition.RIGHT_ANKLE).distance(skeleton2.getNodePosition(TrackerPosition.RIGHT_ANKLE)); - + // Divide by 4 to halve and average, it's halved because you want to approach a midpoint, not the other point return (slideLeft + slideRight) / 4f; } - + // The offset between both feet at one instant and over time protected float getOffsetErrorDeriv(SimpleSkeleton skeleton1, SimpleSkeleton skeleton2) { float skeleton1Left = skeleton1.getNodePosition(TrackerPosition.LEFT_ANKLE).getY(); float skeleton1Right = skeleton1.getNodePosition(TrackerPosition.RIGHT_ANKLE).getY(); - + float skeleton2Left = skeleton2.getNodePosition(TrackerPosition.LEFT_ANKLE).getY(); float skeleton2Right = skeleton2.getNodePosition(TrackerPosition.RIGHT_ANKLE).getY(); - + float dist1 = Math.abs(skeleton1Left - skeleton1Right); float dist2 = Math.abs(skeleton2Left - skeleton2Right); - + float dist3 = Math.abs(skeleton1Left - skeleton2Right); float dist4 = Math.abs(skeleton2Left - skeleton1Right); - + float dist5 = Math.abs(skeleton1Left - skeleton2Left); float dist6 = Math.abs(skeleton1Right - skeleton2Right); - + // Divide by 12 to halve and average, it's halved because you want to approach a midpoint, not the other point return (dist1 + dist2 + dist3 + dist4 + dist5 + dist6) / 12f; } - + // The distance from average human proportions protected float getProportionErrorDeriv(SimpleSkeleton skeleton) { Float neckLength = skeleton.getSkeletonConfig("Neck"); @@ -396,119 +396,119 @@ protected float getProportionErrorDeriv(SimpleSkeleton skeleton) { Float waistLength = skeleton.getSkeletonConfig("Waist"); Float legsLength = skeleton.getSkeletonConfig("Legs length"); Float kneeHeight = skeleton.getSkeletonConfig("Knee height"); - + float chestWaist = chestLength != null && waistLength != null ? Math.abs((chestLength / waistLength) - chestWaistRatio) : 0f; float legBody = legsLength != null && waistLength != null && neckLength != null ? Math.abs((legsLength / (waistLength + neckLength)) - legBodyRatio) : 0f; float kneeLeg = kneeHeight != null && legsLength != null ? Math.abs((kneeHeight / legsLength) - kneeLegRatio) : 0f; - + if(legBody <= legBodyRatioRange) { legBody = 0f; } else { legBody -= legBodyRatioRange; } - + return (chestWaist + legBody + kneeLeg) / 3f; } - + // The distance of any points to the corresponding absolute position protected float getPositionErrorDeriv(TrackerFrame[] frame, SimpleSkeleton skeleton) { float offset = 0f; int offsetCount = 0; - + for(TrackerFrame trackerFrame : frame) { if(trackerFrame == null || !trackerFrame.hasData(TrackerFrameData.POSITION)) { continue; } - + Vector3f nodePos = skeleton.getNodePosition(trackerFrame.designation.designation); if(nodePos != null) { offset += Math.abs(nodePos.distance(trackerFrame.position)); offsetCount++; } } - + return offsetCount > 0 ? offset / offsetCount : 0f; } - + // The difference between offset of absolute position and the corresponding point over time protected float getPositionOffsetErrorDeriv(TrackerFrame[] frame1, TrackerFrame[] frame2, SimpleSkeleton skeleton1, SimpleSkeleton skeleton2) { float offset = 0f; int offsetCount = 0; - + for(TrackerFrame trackerFrame1 : frame1) { if(trackerFrame1 == null || !trackerFrame1.hasData(TrackerFrameData.POSITION)) { continue; } - + TrackerFrame trackerFrame2 = TrackerUtils.findTrackerForBodyPosition(frame2, trackerFrame1.designation); if(trackerFrame2 == null || !trackerFrame2.hasData(TrackerFrameData.POSITION)) { continue; } - + Vector3f nodePos1 = skeleton1.getNodePosition(trackerFrame1.designation); if(nodePos1 == null) { continue; } - + Vector3f nodePos2 = skeleton2.getNodePosition(trackerFrame2.designation); if(nodePos2 == null) { continue; } - + float dist1 = Math.abs(nodePos1.distance(trackerFrame1.position)); float dist2 = Math.abs(nodePos2.distance(trackerFrame2.position)); - + offset += Math.abs(dist2 - dist1); offsetCount++; } - + return offsetCount > 0 ? offset / offsetCount : 0f; } - + protected float getErrorDeriv(TrackerFrame[] frame1, TrackerFrame[] frame2, SimpleSkeleton skeleton1, SimpleSkeleton skeleton2, float heightChange) { float totalError = 0f; float sumWeight = 0f; - + if(slideErrorFactor > 0f) { totalError += getSlideErrorDeriv(skeleton1, skeleton2) * slideErrorFactor; sumWeight += slideErrorFactor; } - + if(offsetErrorFactor > 0f) { totalError += getOffsetErrorDeriv(skeleton1, skeleton2) * offsetErrorFactor; sumWeight += offsetErrorFactor; } - + if(proportionErrorFactor > 0f) { // Either skeleton will work fine, skeleton1 is used as a default totalError += getProportionErrorDeriv(skeleton1) * proportionErrorFactor; sumWeight += proportionErrorFactor; } - + if(heightErrorFactor > 0f) { totalError += Math.abs(heightChange) * heightErrorFactor; sumWeight += heightErrorFactor; } - + if(positionErrorFactor > 0f) { totalError += (getPositionErrorDeriv(frame1, skeleton1) + getPositionErrorDeriv(frame2, skeleton2) / 2f) * positionErrorFactor; sumWeight += positionErrorFactor; } - + if(positionOffsetErrorFactor > 0f) { totalError += getPositionOffsetErrorDeriv(frame1, frame2, skeleton1, skeleton2) * positionOffsetErrorFactor; sumWeight += positionOffsetErrorFactor; } - + // Minimize sliding, minimize foot height offset, minimize change in total height return sumWeight > 0f ? totalError / sumWeight : 0f; } - + // Mean square error function protected static float errorFunc(float errorDeriv) { return 0.5f * (errorDeriv * errorDeriv); } - + protected void updateSkeletonBoneLength(SimpleSkeleton skeleton1, SimpleSkeleton skeleton2, String joint, float newLength) { skeleton1.setSkeletonConfig(joint, newLength, true); skeleton2.setSkeletonConfig(joint, newLength, true); diff --git a/src/main/java/dev/slimevr/gui/AutoBoneWindow.java b/src/main/java/dev/slimevr/gui/AutoBoneWindow.java index 3c97e22f77..41aea4752c 100644 --- a/src/main/java/dev/slimevr/gui/AutoBoneWindow.java +++ b/src/main/java/dev/slimevr/gui/AutoBoneWindow.java @@ -30,42 +30,42 @@ import dev.slimevr.poserecorder.PoseRecorder; public class AutoBoneWindow extends JFrame { - + private static File saveDir = new File("Recordings"); private static File loadDir = new File("LoadRecordings"); - + private EJBox pane; - + private final transient VRServer server; private final transient SkeletonConfig skeletonConfig; private final transient PoseRecorder poseRecorder; private final transient AutoBone autoBone; - + private transient Thread recordingThread = null; private transient Thread saveRecordingThread = null; private transient Thread autoBoneThread = null; - + private JButton saveRecordingButton; private JButton adjustButton; private JButton applyButton; - + private JLabel processLabel; private JLabel lengthsLabel; - + public AutoBoneWindow(VRServer server, SkeletonConfig skeletonConfig) { super("Skeleton Auto-Configuration"); - + this.server = server; this.skeletonConfig = skeletonConfig; this.poseRecorder = new PoseRecorder(server); this.autoBone = new AutoBone(server); - + getContentPane().setLayout(new BoxLayout(getContentPane(), BoxLayout.PAGE_AXIS)); add(new JScrollPane(pane = new EJBox(BoxLayout.PAGE_AXIS), ScrollPaneConstants.VERTICAL_SCROLLBAR_AS_NEEDED, ScrollPaneConstants.HORIZONTAL_SCROLLBAR_AS_NEEDED)); - + build(); } - + private String getLengthsString() { boolean first = true; StringBuilder configInfo = new StringBuilder(""); @@ -75,13 +75,13 @@ private String getLengthsString() { } else { first = false; } - + configInfo.append(entry.getKey() + ": " + StringUtils.prettyNumber(entry.getValue() * 100f, 2)); } - + return configInfo.toString(); } - + private void saveRecording(PoseFrames frames) { if(saveDir.isDirectory() || saveDir.mkdirs()) { File saveRecording; @@ -89,7 +89,7 @@ private void saveRecording(PoseFrames frames) { do { saveRecording = new File(saveDir, "ABRecording" + recordingIndex++ + ".pfr"); } while(saveRecording.exists()); - + LogManager.log.info("[AutoBone] Exporting frames to \"" + saveRecording.getPath() + "\"..."); if(PoseFrameIO.writeToFile(saveRecording, frames)) { LogManager.log.info("[AutoBone] Done exporting! Recording can be found at \"" + saveRecording.getPath() + "\"."); @@ -100,7 +100,7 @@ private void saveRecording(PoseFrames frames) { LogManager.log.severe("[AutoBone] Failed to create the recording directory \"" + saveDir.getPath() + "\"."); } } - + private List> loadRecordings() { List> recordings = new FastList>(); if(loadDir.isDirectory()) { @@ -110,7 +110,7 @@ private List> loadRecordings() { if(file.isFile() && org.apache.commons.lang3.StringUtils.endsWithIgnoreCase(file.getName(), ".pfr")) { LogManager.log.info("[AutoBone] Detected recording at \"" + file.getPath() + "\", loading frames..."); PoseFrames frames = PoseFrameIO.readFromFile(file); - + if(frames == null) { LogManager.log.severe("Reading frames from \"" + file.getPath() + "\" failed..."); } else { @@ -120,26 +120,26 @@ private List> loadRecordings() { } } } - + return recordings; } - + private float processFrames(PoseFrames frames) { autoBone.minDataDistance = server.config.getInt("autobone.minimumDataDistance", autoBone.minDataDistance); autoBone.maxDataDistance = server.config.getInt("autobone.maximumDataDistance", autoBone.maxDataDistance); - + autoBone.numEpochs = server.config.getInt("autobone.epochCount", autoBone.numEpochs); - + autoBone.initialAdjustRate = server.config.getFloat("autobone.adjustRate", autoBone.initialAdjustRate); autoBone.adjustRateDecay = server.config.getFloat("autobone.adjustRateDecay", autoBone.adjustRateDecay); - + autoBone.slideErrorFactor = server.config.getFloat("autobone.slideErrorFactor", autoBone.slideErrorFactor); autoBone.offsetErrorFactor = server.config.getFloat("autobone.offsetErrorFactor", autoBone.offsetErrorFactor); autoBone.proportionErrorFactor = server.config.getFloat("autobone.proportionErrorFactor", autoBone.proportionErrorFactor); autoBone.heightErrorFactor = server.config.getFloat("autobone.heightErrorFactor", autoBone.heightErrorFactor); autoBone.positionErrorFactor = server.config.getFloat("autobone.positionErrorFactor", autoBone.positionErrorFactor); autoBone.positionOffsetErrorFactor = server.config.getFloat("autobone.positionOffsetErrorFactor", autoBone.positionOffsetErrorFactor); - + boolean calcInitError = server.config.getBoolean("autobone.calculateInitialError", true); float targetHeight = server.config.getFloat("autobone.manualTargetHeight", -1f); return autoBone.processFrames(frames, calcInitError, targetHeight, (epoch) -> { @@ -147,7 +147,7 @@ private float processFrames(PoseFrames frames) { lengthsLabel.setText(getLengthsString()); }); } - + @AWTThread private void build() { pane.add(new EJBox(BoxLayout.LINE_AXIS) { @@ -162,7 +162,7 @@ public void mouseClicked(MouseEvent e) { if(!isEnabled() || recordingThread != null) { return; } - + Thread thread = new Thread() { @Override public void run() { @@ -175,10 +175,10 @@ public void run() { Future framesFuture = poseRecorder.startFrameRecording(sampleCount, sampleRate); PoseFrames frames = framesFuture.get(); LogManager.log.info("[AutoBone] Done recording!"); - + saveRecordingButton.setEnabled(true); adjustButton.setEnabled(true); - + if(server.config.getBoolean("autobone.saveRecordings", false)) { setText("Saving..."); saveRecording(frames); @@ -203,14 +203,14 @@ public void run() { } } }; - + recordingThread = thread; thread.start(); } }); } }); - + add(saveRecordingButton = new JButton("Save Recording") { { setEnabled(poseRecorder.hasRecording()); @@ -221,7 +221,7 @@ public void mouseClicked(MouseEvent e) { if(!isEnabled() || saveRecordingThread != null) { return; } - + Thread thread = new Thread() { @Override public void run() { @@ -230,18 +230,18 @@ public void run() { if(framesFuture != null) { setText("Waiting for Recording..."); PoseFrames frames = framesFuture.get(); - + if(frames.getTrackerCount() <= 0) { throw new IllegalStateException("Recording has no trackers"); } - + if(frames.getMaxFrameCount() <= 0) { throw new IllegalStateException("Recording has no frames"); } - + setText("Saving..."); saveRecording(frames); - + setText("Recording Saved!"); try { Thread.sleep(3000); // Wait for 3 seconds @@ -272,14 +272,14 @@ public void run() { } } }; - + saveRecordingThread = thread; thread.start(); } }); } }); - + add(adjustButton = new JButton("Auto-Adjust") { { // If there are files to load, enable the button @@ -291,14 +291,14 @@ public void mouseClicked(MouseEvent e) { if(!isEnabled() || autoBoneThread != null) { return; } - + Thread thread = new Thread() { @Override public void run() { try { setText("Load..."); List> frameRecordings = loadRecordings(); - + if(!frameRecordings.isEmpty()) { LogManager.log.info("[AutoBone] Done loading frames!"); } else { @@ -306,15 +306,15 @@ public void run() { if(framesFuture != null) { setText("Waiting for Recording..."); PoseFrames frames = framesFuture.get(); - + if(frames.getTrackerCount() <= 0) { throw new IllegalStateException("Recording has no trackers"); } - + if(frames.getMaxFrameCount() <= 0) { throw new IllegalStateException("Recording has no frames"); } - + frameRecordings.add(Pair.of("", frames)); } else { setText("No Recordings..."); @@ -327,17 +327,17 @@ public void run() { return; } } - + setText("Processing..."); LogManager.log.info("[AutoBone] Processing frames..."); FastList heightPercentError = new FastList(frameRecordings.size()); for(Pair recording : frameRecordings) { LogManager.log.info("[AutoBone] Processing frames from \"" + recording.getKey() + "\"..."); - + heightPercentError.add(processFrames(recording.getValue())); LogManager.log.info("[AutoBone] Done processing!"); applyButton.setEnabled(true); - + //#region Stats/Values Float neckLength = autoBone.getConfig("Neck"); Float chestLength = autoBone.getConfig("Chest"); @@ -345,35 +345,35 @@ public void run() { Float hipWidth = autoBone.getConfig("Hips width"); Float legsLength = autoBone.getConfig("Legs length"); Float kneeHeight = autoBone.getConfig("Knee height"); - + float neckWaist = neckLength != null && waistLength != null ? neckLength / waistLength : 0f; float chestWaist = chestLength != null && waistLength != null ? chestLength / waistLength : 0f; float hipWaist = hipWidth != null && waistLength != null ? hipWidth / waistLength : 0f; float legWaist = legsLength != null && waistLength != null ? legsLength / waistLength : 0f; float legBody = legsLength != null && waistLength != null && neckLength != null ? legsLength / (waistLength + neckLength) : 0f; float kneeLeg = kneeHeight != null && legsLength != null ? kneeHeight / legsLength : 0f; - + LogManager.log.info("[AutoBone] Ratios: [{Neck-Waist: " + StringUtils.prettyNumber(neckWaist) + "}, {Chest-Waist: " + StringUtils.prettyNumber(chestWaist) + "}, {Hip-Waist: " + StringUtils.prettyNumber(hipWaist) + "}, {Leg-Waist: " + StringUtils.prettyNumber(legWaist) + "}, {Leg-Body: " + StringUtils.prettyNumber(legBody) + "}, {Knee-Leg: " + StringUtils.prettyNumber(kneeLeg) + "}]"); - + String lengthsString = getLengthsString(); LogManager.log.info("[AutoBone] Length values: " + lengthsString); lengthsLabel.setText(lengthsString); } - + if(!heightPercentError.isEmpty()) { float mean = 0f; for(float val : heightPercentError) { mean += val; } mean /= heightPercentError.size(); - + float std = 0f; for(float val : heightPercentError) { float stdVal = val - mean; std += stdVal * stdVal; } std = (float) Math.sqrt(std / heightPercentError.size()); - + LogManager.log.info("[AutoBone] Average height error: " + StringUtils.prettyNumber(mean, 6) + " (SD " + StringUtils.prettyNumber(std, 6) + ")"); } //#endregion @@ -391,14 +391,14 @@ public void run() { } } }; - + autoBoneThread = thread; thread.start(); } }); } }); - + add(applyButton = new JButton("Apply Values") { { setEnabled(false); @@ -408,7 +408,7 @@ public void mouseClicked(MouseEvent e) { if(!isEnabled()) { return; } - + autoBone.applyConfig(); // Update GUI values after applying skeletonConfig.refreshAll(); @@ -418,21 +418,21 @@ public void mouseClicked(MouseEvent e) { }); } }); - + pane.add(new EJBox(BoxLayout.LINE_AXIS) { { setBorder(new EmptyBorder(i(5))); add(processLabel = new JLabel("Processing has not been started...")); } }); - + pane.add(new EJBox(BoxLayout.LINE_AXIS) { { setBorder(new EmptyBorder(i(5))); add(lengthsLabel = new JLabel(getLengthsString())); } }); - + // Pack and display pack(); setLocationRelativeTo(null); diff --git a/src/main/java/dev/slimevr/poserecorder/PoseFrameIO.java b/src/main/java/dev/slimevr/poserecorder/PoseFrameIO.java index d7cd52449c..55a3a316e1 100644 --- a/src/main/java/dev/slimevr/poserecorder/PoseFrameIO.java +++ b/src/main/java/dev/slimevr/poserecorder/PoseFrameIO.java @@ -16,17 +16,17 @@ import io.eiren.vr.trackers.TrackerPosition; public final class PoseFrameIO { - + private PoseFrameIO() { // Do not allow instantiating } - + public static boolean writeFrames(DataOutputStream outputStream, PoseFrames frames) { try { if(frames != null) { outputStream.writeInt(frames.getTrackerCount()); for(PoseFrameTracker tracker : frames.getTrackers()) { - + outputStream.writeUTF(tracker.name); outputStream.writeInt(tracker.getFrameCount()); for(int i = 0; i < tracker.getFrameCount(); i++) { @@ -35,20 +35,20 @@ public static boolean writeFrames(DataOutputStream outputStream, PoseFrames fram outputStream.writeInt(0); continue; } - + outputStream.writeInt(trackerFrame.getDataFlags()); - + if(trackerFrame.hasData(TrackerFrameData.DESIGNATION)) { outputStream.writeUTF(trackerFrame.designation.designation); } - + if(trackerFrame.hasData(TrackerFrameData.ROTATION)) { outputStream.writeFloat(trackerFrame.rotation.getX()); outputStream.writeFloat(trackerFrame.rotation.getY()); outputStream.writeFloat(trackerFrame.rotation.getZ()); outputStream.writeFloat(trackerFrame.rotation.getW()); } - + if(trackerFrame.hasData(TrackerFrameData.POSITION)) { outputStream.writeFloat(trackerFrame.position.getX()); outputStream.writeFloat(trackerFrame.position.getY()); @@ -63,10 +63,10 @@ public static boolean writeFrames(DataOutputStream outputStream, PoseFrames fram LogManager.log.severe("Error writing frame to stream", e); return false; } - + return true; } - + public static boolean writeToFile(File file, PoseFrames frames) { try(DataOutputStream outputStream = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(file)))) { writeFrames(outputStream, frames); @@ -74,28 +74,28 @@ public static boolean writeToFile(File file, PoseFrames frames) { LogManager.log.severe("Error writing frames to file", e); return false; } - + return true; } - + public static PoseFrames readFrames(DataInputStream inputStream) { try { - + int trackerCount = inputStream.readInt(); FastList trackers = new FastList(trackerCount); for(int i = 0; i < trackerCount; i++) { - + String name = inputStream.readUTF(); int trackerFrameCount = inputStream.readInt(); FastList trackerFrames = new FastList(trackerFrameCount); for(int j = 0; j < trackerFrameCount; j++) { int dataFlags = inputStream.readInt(); - + TrackerPosition designation = null; if(TrackerFrameData.DESIGNATION.check(dataFlags)) { designation = TrackerPosition.getByDesignation(inputStream.readUTF()); } - + Quaternion rotation = null; if(TrackerFrameData.ROTATION.check(dataFlags)) { float quatX = inputStream.readFloat(); @@ -104,7 +104,7 @@ public static PoseFrames readFrames(DataInputStream inputStream) { float quatW = inputStream.readFloat(); rotation = new Quaternion(quatX, quatY, quatZ, quatW); } - + Vector3f position = null; if(TrackerFrameData.POSITION.check(dataFlags)) { float posX = inputStream.readFloat(); @@ -112,28 +112,28 @@ public static PoseFrames readFrames(DataInputStream inputStream) { float posZ = inputStream.readFloat(); position = new Vector3f(posX, posY, posZ); } - + trackerFrames.add(new TrackerFrame(designation, rotation, position)); } - + trackers.add(new PoseFrameTracker(name, trackerFrames)); } - + return new PoseFrames(trackers); } catch(Exception e) { LogManager.log.severe("Error reading frame from stream", e); } - + return null; } - + public static PoseFrames readFromFile(File file) { try { return readFrames(new DataInputStream(new BufferedInputStream(new FileInputStream(file)))); } catch(Exception e) { LogManager.log.severe("Error reading frame from file", e); } - + return null; } } diff --git a/src/main/java/dev/slimevr/poserecorder/PoseFrames.java b/src/main/java/dev/slimevr/poserecorder/PoseFrames.java index 2779dcb556..cf4a39643f 100644 --- a/src/main/java/dev/slimevr/poserecorder/PoseFrames.java +++ b/src/main/java/dev/slimevr/poserecorder/PoseFrames.java @@ -8,72 +8,72 @@ import io.eiren.vr.trackers.Tracker; public final class PoseFrames implements Iterable { - + private final FastList trackers; - + public PoseFrames(FastList trackers) { this.trackers = trackers; } - + public PoseFrames(int initialCapacity) { this.trackers = new FastList(initialCapacity); } - + public PoseFrames() { this(5); } - + public PoseFrameTracker addTracker(PoseFrameTracker tracker) { trackers.add(tracker); return tracker; } - + public PoseFrameTracker addTracker(Tracker tracker, int initialCapacity) { return addTracker(new PoseFrameTracker(tracker.getName(), initialCapacity)); } - + public PoseFrameTracker addTracker(Tracker tracker) { return addTracker(tracker, 5); } - + public PoseFrameTracker removeTracker(int index) { return trackers.remove(index); } - + public PoseFrameTracker removeTracker(PoseFrameTracker tracker) { trackers.remove(tracker); return tracker; } - + public void clearTrackers() { trackers.clear(); } - + public void fakeClearTrackers() { trackers.fakeClear(); } - + public int getTrackerCount() { return trackers.size(); } - + public List getTrackers() { return trackers; } - + public int getMaxFrameCount() { int maxFrames = 0; - + for(int i = 0; i < trackers.size(); i++) { PoseFrameTracker tracker = trackers.get(i); if(tracker != null && tracker.getFrameCount() > maxFrames) { maxFrames = tracker.getFrameCount(); } } - + return maxFrames; } - + public int getFrames(int frameIndex, TrackerFrame[] buffer) { for(int i = 0; i < trackers.size(); i++) { PoseFrameTracker tracker = trackers.get(i); @@ -81,7 +81,7 @@ public int getFrames(int frameIndex, TrackerFrame[] buffer) { } return trackers.size(); } - + public int getFrames(int frameIndex, List buffer) { for(int i = 0; i < trackers.size(); i++) { PoseFrameTracker tracker = trackers.get(i); @@ -89,54 +89,54 @@ public int getFrames(int frameIndex, List buffer) { } return trackers.size(); } - + public TrackerFrame[] getFrames(int frameIndex) { TrackerFrame[] trackerFrames = new TrackerFrame[trackers.size()]; getFrames(frameIndex, trackerFrames); return trackerFrames; } - + @Override public Iterator iterator() { return new PoseFrameIterator(this); } - + public class PoseFrameIterator implements Iterator { - + private final PoseFrames poseFrame; private final TrackerFrame[] trackerFrameBuffer; - + private int cursor = 0; - + public PoseFrameIterator(PoseFrames poseFrame) { this.poseFrame = poseFrame; trackerFrameBuffer = new TrackerFrame[poseFrame.getTrackerCount()]; } - + @Override public boolean hasNext() { if(trackers.isEmpty()) { return false; } - + for(int i = 0; i < trackers.size(); i++) { PoseFrameTracker tracker = trackers.get(i); if(tracker != null && cursor < tracker.getFrameCount()) { return true; } } - + return false; } - + @Override public TrackerFrame[] next() { if(!hasNext()) { throw new NoSuchElementException(); } - + poseFrame.getFrames(cursor++, trackerFrameBuffer); - + return trackerFrameBuffer; } } From 8f052847928e064e39a53d26e15d4cd42d4be093 Mon Sep 17 00:00:00 2001 From: ButterscotchVanilla Date: Wed, 13 Oct 2021 03:05:39 -0400 Subject: [PATCH 7/9] Rename PoseFileStream and remove unexpected buffer --- .../dev/slimevr/poserecorder/BVHFileStream.java | 16 ++++++++-------- .../{PoseFileStream.java => PoseDataStream.java} | 15 ++++++--------- .../dev/slimevr/poserecorder/PoseStreamer.java | 12 ++++++------ 3 files changed, 20 insertions(+), 23 deletions(-) rename src/main/java/dev/slimevr/poserecorder/{PoseFileStream.java => PoseDataStream.java} (57%) diff --git a/src/main/java/dev/slimevr/poserecorder/BVHFileStream.java b/src/main/java/dev/slimevr/poserecorder/BVHFileStream.java index 00d8a832ed..01814a0a46 100644 --- a/src/main/java/dev/slimevr/poserecorder/BVHFileStream.java +++ b/src/main/java/dev/slimevr/poserecorder/BVHFileStream.java @@ -16,7 +16,7 @@ import io.eiren.vr.processor.HumanSkeleton; import io.eiren.vr.processor.TransformNode; -public class BVHFileStream extends PoseFileStream { +public class BVHFileStream extends PoseDataStream { private static final int LONG_MAX_VALUE_DIGITS = Long.toString(Long.MAX_VALUE).length(); private static final float POS_SCALE = 10f; @@ -31,17 +31,17 @@ public class BVHFileStream extends PoseFileStream { public BVHFileStream(OutputStream outputStream) { super(outputStream); - writer = new BufferedWriter(new OutputStreamWriter(outputStream), 128); + writer = new BufferedWriter(new OutputStreamWriter(outputStream), 4096); } public BVHFileStream(File file) throws FileNotFoundException { super(file); - writer = new BufferedWriter(new OutputStreamWriter(outputStream), 128); + writer = new BufferedWriter(new OutputStreamWriter(outputStream), 4096); } public BVHFileStream(String file) throws FileNotFoundException { super(file); - writer = new BufferedWriter(new OutputStreamWriter(outputStream), 128); + writer = new BufferedWriter(new OutputStreamWriter(outputStream), 4096); } private String getBufferedFrameCount(long frameCount) { @@ -107,8 +107,8 @@ public void writeHeader(HumanSkeleton skeleton, PoseStreamer streamer) throws IO writer.write("Frames: "); // Get frame offset for finishing writing the file - if (parentStream instanceof FileOutputStream) { - FileOutputStream fileOutputStream = (FileOutputStream)parentStream; + if (outputStream instanceof FileOutputStream) { + FileOutputStream fileOutputStream = (FileOutputStream)outputStream; // Flush buffer to get proper offset writer.flush(); frameCountOffset = fileOutputStream.getChannel().position(); @@ -166,8 +166,8 @@ public void writeFrame(HumanSkeleton skeleton) throws IOException { @Override public void writeFooter(HumanSkeleton skeleton) throws IOException { // Write the final frame count for files - if (parentStream instanceof FileOutputStream) { - FileOutputStream fileOutputStream = (FileOutputStream)parentStream; + if (outputStream instanceof FileOutputStream) { + FileOutputStream fileOutputStream = (FileOutputStream)outputStream; // Flush before anything else writer.flush(); // Seek to the count offset diff --git a/src/main/java/dev/slimevr/poserecorder/PoseFileStream.java b/src/main/java/dev/slimevr/poserecorder/PoseDataStream.java similarity index 57% rename from src/main/java/dev/slimevr/poserecorder/PoseFileStream.java rename to src/main/java/dev/slimevr/poserecorder/PoseDataStream.java index a1927f00b3..42da437db5 100644 --- a/src/main/java/dev/slimevr/poserecorder/PoseFileStream.java +++ b/src/main/java/dev/slimevr/poserecorder/PoseDataStream.java @@ -1,6 +1,5 @@ package dev.slimevr.poserecorder; -import java.io.BufferedOutputStream; import java.io.File; import java.io.FileNotFoundException; import java.io.FileOutputStream; @@ -9,21 +8,19 @@ import io.eiren.vr.processor.HumanSkeleton; -public abstract class PoseFileStream implements AutoCloseable { +public abstract class PoseDataStream implements AutoCloseable { - protected final OutputStream parentStream; - protected final BufferedOutputStream outputStream; + protected final OutputStream outputStream; - protected PoseFileStream(OutputStream outputStream) { - this.parentStream = outputStream; - this.outputStream = new BufferedOutputStream(outputStream); + protected PoseDataStream(OutputStream outputStream) { + this.outputStream = outputStream; } - protected PoseFileStream(File file) throws FileNotFoundException { + protected PoseDataStream(File file) throws FileNotFoundException { this(new FileOutputStream(file)); } - protected PoseFileStream(String file) throws FileNotFoundException { + protected PoseDataStream(String file) throws FileNotFoundException { this(new FileOutputStream(file)); } diff --git a/src/main/java/dev/slimevr/poserecorder/PoseStreamer.java b/src/main/java/dev/slimevr/poserecorder/PoseStreamer.java index 72d952e521..3d16869eb5 100644 --- a/src/main/java/dev/slimevr/poserecorder/PoseStreamer.java +++ b/src/main/java/dev/slimevr/poserecorder/PoseStreamer.java @@ -13,7 +13,7 @@ public class PoseStreamer { protected long nextFrameTimeMs = -1L; private HumanSkeleton skeleton; - private PoseFileStream poseFileStream; + private PoseDataStream poseFileStream; protected final VRServer server; @@ -32,7 +32,7 @@ public void onSkeletonUpdated(HumanSkeleton skeleton) { @VRServerThread public void onTick() { - PoseFileStream poseFileStream = this.poseFileStream; + PoseDataStream poseFileStream = this.poseFileStream; if (poseFileStream != null) { long curTime = System.currentTimeMillis(); @@ -62,23 +62,23 @@ public void setFrameInterval(long intervalMs) { this.frameRecordingInterval = intervalMs; } - public void setOutput(PoseFileStream poseFileStream) throws IOException { + public void setOutput(PoseDataStream poseFileStream) throws IOException { poseFileStream.writeHeader(skeleton, this); this.poseFileStream = poseFileStream; nextFrameTimeMs = -1L; // Reset the frame timing } - public void setOutput(PoseFileStream poseFileStream, long intervalMs) throws IOException { + public void setOutput(PoseDataStream poseFileStream, long intervalMs) throws IOException { setFrameInterval(intervalMs); setOutput(poseFileStream); } - public PoseFileStream getOutput() { + public PoseDataStream getOutput() { return poseFileStream; } public void closeOutput() throws IOException { - PoseFileStream poseFileStream = this.poseFileStream; + PoseDataStream poseFileStream = this.poseFileStream; if (poseFileStream != null) { poseFileStream.writeFooter(skeleton); From 35c26bec0fcf74c4de158bfb8a3351702ab64462 Mon Sep 17 00:00:00 2001 From: ButterscotchVanilla Date: Wed, 13 Oct 2021 07:05:29 -0400 Subject: [PATCH 8/9] Fix Pose Recorder and Streamer synchronization and reduce if nesting --- .../slimevr/poserecorder/BVHFileStream.java | 1 + .../slimevr/poserecorder/PoseDataStream.java | 6 ++ .../slimevr/poserecorder/PoseRecorder.java | 86 ++++++++++--------- .../slimevr/poserecorder/PoseStreamer.java | 63 +++++++++----- 4 files changed, 96 insertions(+), 60 deletions(-) diff --git a/src/main/java/dev/slimevr/poserecorder/BVHFileStream.java b/src/main/java/dev/slimevr/poserecorder/BVHFileStream.java index 01814a0a46..ab856a5f91 100644 --- a/src/main/java/dev/slimevr/poserecorder/BVHFileStream.java +++ b/src/main/java/dev/slimevr/poserecorder/BVHFileStream.java @@ -180,5 +180,6 @@ public void writeFooter(HumanSkeleton skeleton) throws IOException { @Override public void close() throws IOException { writer.close(); + super.close(); } } diff --git a/src/main/java/dev/slimevr/poserecorder/PoseDataStream.java b/src/main/java/dev/slimevr/poserecorder/PoseDataStream.java index 42da437db5..414e30f0cf 100644 --- a/src/main/java/dev/slimevr/poserecorder/PoseDataStream.java +++ b/src/main/java/dev/slimevr/poserecorder/PoseDataStream.java @@ -10,6 +10,7 @@ public abstract class PoseDataStream implements AutoCloseable { + protected boolean closed = false; protected final OutputStream outputStream; protected PoseDataStream(OutputStream outputStream) { @@ -32,8 +33,13 @@ public void writeHeader(HumanSkeleton skeleton, PoseStreamer streamer) throws IO public void writeFooter(HumanSkeleton skeleton) throws IOException { } + public boolean isClosed() { + return closed; + } + @Override public void close() throws IOException { outputStream.close(); + closed = true; } } diff --git a/src/main/java/dev/slimevr/poserecorder/PoseRecorder.java b/src/main/java/dev/slimevr/poserecorder/PoseRecorder.java index 2aa5955f6e..1be97dcad7 100644 --- a/src/main/java/dev/slimevr/poserecorder/PoseRecorder.java +++ b/src/main/java/dev/slimevr/poserecorder/PoseRecorder.java @@ -34,35 +34,47 @@ public PoseRecorder(VRServer server) { @VRServerThread public void onTick() { - if(numFrames > 0) { - PoseFrames poseFrame = this.poseFrame; - List> trackers = this.trackers; - if (poseFrame != null && trackers != null) { - if (frameCursor < numFrames) { - long curTime = System.currentTimeMillis(); - if (curTime >= nextFrameTimeMs) { - nextFrameTimeMs += frameRecordingInterval; - - // To prevent duplicate frames, make sure the frame time is always in the future - if (nextFrameTimeMs <= curTime) { - nextFrameTimeMs = curTime + frameRecordingInterval; - } - - int cursor = frameCursor++; - for(Pair tracker : trackers) { - // Add a frame for each tracker - tracker.getRight().addFrame(cursor, tracker.getLeft()); - } - - // If done, send finished recording - if(frameCursor >= numFrames) { - internalStopRecording(); - } - } - } else { - // If done and hasn't yet, send finished recording - internalStopRecording(); - } + if (numFrames <= 0) { + return; + } + + PoseFrames poseFrame = this.poseFrame; + List> trackers = this.trackers; + if (poseFrame == null || trackers == null) { + return; + } + + if (frameCursor >= numFrames) { + // If done and hasn't yet, send finished recording + stopFrameRecording(); + return; + } + + long curTime = System.currentTimeMillis(); + if (curTime < nextFrameTimeMs) { + return; + } + + nextFrameTimeMs += frameRecordingInterval; + + // To prevent duplicate frames, make sure the frame time is always in the future + if (nextFrameTimeMs <= curTime) { + nextFrameTimeMs = curTime + frameRecordingInterval; + } + + // Make sure it's synchronized since this is the server thread interacting with + // an unknown outside thread controlling this class + synchronized (this) { + // A stopped recording will be accounted for by an empty "trackers" list + int cursor = frameCursor++; + for(Pair tracker : trackers) { + // Add a frame for each tracker + tracker.getRight().addFrame(cursor, tracker.getLeft()); + } + + // If done, send finished recording + if(frameCursor >= numFrames) { + stopFrameRecording(); } } } @@ -116,7 +128,7 @@ public synchronized Future startFrameRecording(int numFrames, long i return currentRecording; } - private void internalStopRecording() { + public synchronized void stopFrameRecording() { CompletableFuture currentRecording = this.currentRecording; if(currentRecording != null && !currentRecording.isDone()) { // Stop the recording, returning the frames recorded @@ -129,10 +141,6 @@ private void internalStopRecording() { poseFrame = null; } - public synchronized void stopFrameRecording() { - internalStopRecording(); - } - public synchronized void cancelFrameRecording() { CompletableFuture currentRecording = this.currentRecording; if(currentRecording != null && !currentRecording.isDone()) { @@ -146,23 +154,23 @@ public synchronized void cancelFrameRecording() { poseFrame = null; } - public boolean isReadyToRecord() { + public synchronized boolean isReadyToRecord() { return server.getTrackersCount() > 0; } - public boolean isRecording() { + public synchronized boolean isRecording() { return numFrames > frameCursor; } - public boolean hasRecording() { + public synchronized boolean hasRecording() { return currentRecording != null; } - public Future getFramesAsync() { + public synchronized Future getFramesAsync() { return currentRecording; } - public PoseFrames getFrames() throws ExecutionException, InterruptedException { + public synchronized PoseFrames getFrames() throws ExecutionException, InterruptedException { CompletableFuture currentRecording = this.currentRecording; return currentRecording != null ? currentRecording.get() : null; } diff --git a/src/main/java/dev/slimevr/poserecorder/PoseStreamer.java b/src/main/java/dev/slimevr/poserecorder/PoseStreamer.java index 3d16869eb5..88570cbfbb 100644 --- a/src/main/java/dev/slimevr/poserecorder/PoseStreamer.java +++ b/src/main/java/dev/slimevr/poserecorder/PoseStreamer.java @@ -33,28 +33,45 @@ public void onSkeletonUpdated(HumanSkeleton skeleton) { @VRServerThread public void onTick() { PoseDataStream poseFileStream = this.poseFileStream; - if (poseFileStream != null) { + if (poseFileStream == null) { + return; + } + + HumanSkeleton skeleton = this.skeleton; + if (skeleton == null) { + return; + } + + long curTime = System.currentTimeMillis(); + if (curTime < nextFrameTimeMs) { + return; + } + + nextFrameTimeMs += frameRecordingInterval; - long curTime = System.currentTimeMillis(); - if (curTime >= nextFrameTimeMs) { - nextFrameTimeMs += frameRecordingInterval; - - // To prevent duplicate frames, make sure the frame time is always in the future - if (nextFrameTimeMs <= curTime) { - nextFrameTimeMs = curTime + frameRecordingInterval; - } - - try { - poseFileStream.writeFrame(skeleton); - } catch (Exception e) { - // Handle any exceptions without crashing the program - LogManager.log.severe("[PoseStreamer] Exception while saving frame", e); - } + // To prevent duplicate frames, make sure the frame time is always in the future + if (nextFrameTimeMs <= curTime) { + nextFrameTimeMs = curTime + frameRecordingInterval; + } + + // Make sure it's synchronized since this is the server thread interacting with + // an unknown outside thread controlling this class + synchronized (this) { + // Make sure the stream is open before trying to write + if (poseFileStream.isClosed()) { + return; + } + + try { + poseFileStream.writeFrame(skeleton); + } catch (Exception e) { + // Handle any exceptions without crashing the program + LogManager.log.severe("[PoseStreamer] Exception while saving frame", e); } } } - public void setFrameInterval(long intervalMs) { + public synchronized void setFrameInterval(long intervalMs) { if(intervalMs < 1) { throw new IllegalArgumentException("intervalMs must at least have a value of 1"); } @@ -62,22 +79,26 @@ public void setFrameInterval(long intervalMs) { this.frameRecordingInterval = intervalMs; } - public void setOutput(PoseDataStream poseFileStream) throws IOException { + public synchronized long getFrameInterval() { + return frameRecordingInterval; + } + + public synchronized void setOutput(PoseDataStream poseFileStream) throws IOException { poseFileStream.writeHeader(skeleton, this); this.poseFileStream = poseFileStream; nextFrameTimeMs = -1L; // Reset the frame timing } - public void setOutput(PoseDataStream poseFileStream, long intervalMs) throws IOException { + public synchronized void setOutput(PoseDataStream poseFileStream, long intervalMs) throws IOException { setFrameInterval(intervalMs); setOutput(poseFileStream); } - public PoseDataStream getOutput() { + public synchronized PoseDataStream getOutput() { return poseFileStream; } - public void closeOutput() throws IOException { + public synchronized void closeOutput() throws IOException { PoseDataStream poseFileStream = this.poseFileStream; if (poseFileStream != null) { From 820d06f0084e69eecc840316b9f1b846279b7226 Mon Sep 17 00:00:00 2001 From: ButterscotchVanilla Date: Thu, 21 Oct 2021 22:48:38 -0400 Subject: [PATCH 9/9] Move PoseStreamer files --- .../slimevr/{poserecorder => posestreamer}/BVHFileStream.java | 2 +- .../slimevr/{poserecorder => posestreamer}/PoseDataStream.java | 2 +- .../slimevr/{poserecorder => posestreamer}/PoseStreamer.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) rename src/main/java/dev/slimevr/{poserecorder => posestreamer}/BVHFileStream.java (99%) rename src/main/java/dev/slimevr/{poserecorder => posestreamer}/PoseDataStream.java (96%) rename src/main/java/dev/slimevr/{poserecorder => posestreamer}/PoseStreamer.java (98%) diff --git a/src/main/java/dev/slimevr/poserecorder/BVHFileStream.java b/src/main/java/dev/slimevr/posestreamer/BVHFileStream.java similarity index 99% rename from src/main/java/dev/slimevr/poserecorder/BVHFileStream.java rename to src/main/java/dev/slimevr/posestreamer/BVHFileStream.java index ab856a5f91..61b4502d8e 100644 --- a/src/main/java/dev/slimevr/poserecorder/BVHFileStream.java +++ b/src/main/java/dev/slimevr/posestreamer/BVHFileStream.java @@ -1,4 +1,4 @@ -package dev.slimevr.poserecorder; +package dev.slimevr.posestreamer; import java.io.BufferedWriter; import java.io.File; diff --git a/src/main/java/dev/slimevr/poserecorder/PoseDataStream.java b/src/main/java/dev/slimevr/posestreamer/PoseDataStream.java similarity index 96% rename from src/main/java/dev/slimevr/poserecorder/PoseDataStream.java rename to src/main/java/dev/slimevr/posestreamer/PoseDataStream.java index 414e30f0cf..dc23038b69 100644 --- a/src/main/java/dev/slimevr/poserecorder/PoseDataStream.java +++ b/src/main/java/dev/slimevr/posestreamer/PoseDataStream.java @@ -1,4 +1,4 @@ -package dev.slimevr.poserecorder; +package dev.slimevr.posestreamer; import java.io.File; import java.io.FileNotFoundException; diff --git a/src/main/java/dev/slimevr/poserecorder/PoseStreamer.java b/src/main/java/dev/slimevr/posestreamer/PoseStreamer.java similarity index 98% rename from src/main/java/dev/slimevr/poserecorder/PoseStreamer.java rename to src/main/java/dev/slimevr/posestreamer/PoseStreamer.java index 88570cbfbb..e339be4e95 100644 --- a/src/main/java/dev/slimevr/poserecorder/PoseStreamer.java +++ b/src/main/java/dev/slimevr/posestreamer/PoseStreamer.java @@ -1,4 +1,4 @@ -package dev.slimevr.poserecorder; +package dev.slimevr.posestreamer; import java.io.IOException;