From 8fa535d8b8c4276f0680af532ffeb60cf8e1aa47 Mon Sep 17 00:00:00 2001 From: Butterscotch! Date: Sun, 19 Nov 2023 10:50:27 -0500 Subject: [PATCH] Separate scale from proportions in AutoBone (#846) --- .../src/main/java/dev/slimevr/VRServer.kt | 3 +- .../java/dev/slimevr/autobone/AutoBone.kt | 367 +++++++++++------- .../dev/slimevr/autobone/AutoBoneHandler.kt | 211 +++++----- .../java/dev/slimevr/autobone/AutoBoneStep.kt | 2 +- .../autobone/errors/BodyProportionError.kt | 36 +- .../proportions/HardProportionLimiter.kt | 9 +- .../errors/proportions/ProportionLimiter.kt | 2 + .../proportions/RangeProportionLimiter.kt | 18 +- .../java/dev/slimevr/config/AutoBoneConfig.kt | 5 +- .../config/CurrentVRConfigConverter.java | 14 + .../java/dev/slimevr/config/VRConfig.java | 2 +- .../poseframeformat/player/PlayerTracker.kt | 13 +- .../player/TrackerFramesPlayer.kt | 6 + .../config/SkeletonConfigManager.java | 25 +- 14 files changed, 420 insertions(+), 293 deletions(-) diff --git a/server/core/src/main/java/dev/slimevr/VRServer.kt b/server/core/src/main/java/dev/slimevr/VRServer.kt index 2f4e9767d9..c4a94e3fb4 100644 --- a/server/core/src/main/java/dev/slimevr/VRServer.kt +++ b/server/core/src/main/java/dev/slimevr/VRServer.kt @@ -106,9 +106,10 @@ class VRServer @JvmOverloads constructor( provisioningHandler = ProvisioningHandler(this) resetHandler = ResetHandler() tapSetupHandler = TapSetupHandler() + humanPoseManager = HumanPoseManager(this) + // AutoBone requires HumanPoseManager first autoBoneHandler = AutoBoneHandler(this) protocolAPI = ProtocolAPI(this) - humanPoseManager = HumanPoseManager(this) val computedTrackers = humanPoseManager.computedTrackers // Start server for SlimeVR trackers diff --git a/server/core/src/main/java/dev/slimevr/autobone/AutoBone.kt b/server/core/src/main/java/dev/slimevr/autobone/AutoBone.kt index e027103c9a..78055d7d27 100644 --- a/server/core/src/main/java/dev/slimevr/autobone/AutoBone.kt +++ b/server/core/src/main/java/dev/slimevr/autobone/AutoBone.kt @@ -42,22 +42,11 @@ class AutoBone(server: VRServer) { SkeletonConfigOffsets.LOWER_LEG ) ) - val heightOffsetDefaults = EnumMap( - SkeletonConfigOffsets::class.java - ) - // This is filled by loadConfigValues() - val heightOffsets = FastList( - arrayOf( - SkeletonConfigOffsets.NECK, - SkeletonConfigOffsets.UPPER_CHEST, - SkeletonConfigOffsets.CHEST, - SkeletonConfigOffsets.WAIST, - SkeletonConfigOffsets.HIP, - SkeletonConfigOffsets.UPPER_LEG, - SkeletonConfigOffsets.LOWER_LEG - ) - ) + var estimatedHeight: Float = 1f + + // The total height of the normalized adjusted offsets + var adjustedHeightNormalized: Float = 1f private val server: VRServer @@ -87,6 +76,7 @@ class AutoBone(server: VRServer) { // Get current or default skeleton configs val skeleton = server.humanPoseManager + // Still compensate for a null skeleton, as it may not be initialized yet val getOffset: Function = if (skeleton != null) { Function { key: SkeletonConfigOffsets -> skeleton.getOffset(key) } @@ -102,12 +92,6 @@ class AutoBone(server: VRServer) { offsets[bone] = offset } } - for (bone in heightOffsets) { - val offset = getOffset.apply(bone) - if (offset > 0f) { - heightOffsetDefaults[bone] = offset - } - } } fun getBoneDirection( @@ -162,48 +146,6 @@ class AutoBone(server: VRServer) { return true } - fun sumSelectConfigs( - selection: List, - configs: Map, - configsAlt: Map? = null, - ): Float { - var sum = 0f - for (config in selection) { - val length = configs[config] ?: configsAlt?.get(config) - if (length != null) { - sum += length - } - } - return sum - } - - fun calcHeight(): Float { - return sumSelectConfigs(heightOffsets, offsets, heightOffsetDefaults) - } - - fun getLengthSum(configs: Map): Float { - return getLengthSum(configs, null) - } - - fun getLengthSum( - configs: Map, - configsAlt: Map?, - ): Float { - var length = 0f - if (configsAlt != null) { - for ((key, value) in configsAlt) { - // If there isn't a duplicate config - if (!configs.containsKey(key)) { - length += value - } - } - } - for (boneLength in configs.values) { - length += boneLength - } - return length - } - fun calcTargetHmdHeight( frames: PoseFrames, config: AutoBoneConfig = globalConfig, @@ -211,6 +153,7 @@ class AutoBone(server: VRServer) { val targetHeight: Float // Get the current skeleton from the server val humanPoseManager = server.humanPoseManager + // Still compensate for a null skeleton, as it may not be initialized yet if (config.useSkeletonHeight && humanPoseManager != null) { // If there is a skeleton available, calculate the target height // from its configs @@ -238,6 +181,13 @@ class AutoBone(server: VRServer) { return targetHeight } + private fun updateRecordingScale(trainingStep: AutoBoneStep, scale: Float) { + trainingStep.framePlayer1.setScales(scale) + trainingStep.framePlayer2.setScales(scale) + trainingStep.skeleton1.update() + trainingStep.skeleton2.update() + } + @Throws(AutoBoneException::class) fun processFrames( frames: PoseFrames, @@ -256,7 +206,7 @@ class AutoBone(server: VRServer) { val targetFullHeight = if (config.targetFullHeight > 0f) { config.targetFullHeight } else { - targetHmdHeight * BodyProportionError.eyeHeightToHeightRatio + targetHmdHeight / BodyProportionError.eyeHeightToHeightRatio } // Set up the current state, making all required players and setting up the @@ -273,6 +223,80 @@ class AutoBone(server: VRServer) { // Initialize the frame order randomizer with a repeatable seed rand.setSeed(config.randSeed) + // Normalize the skeletons and get the normalized height for adjusted offsets + scaleSkeleton(trainingStep.skeleton1) + scaleSkeleton(trainingStep.skeleton2) + adjustedHeightNormalized = sumAdjustedHeightOffsets(trainingStep.skeleton1) + + // Normalize offsets based on the initial normalized skeleton + scaleOffsets() + + // Apply the initial normalized config values + applyConfig(trainingStep.skeleton1) + applyConfig(trainingStep.skeleton2) + + // Initialize normalization to the set target height (also updates skeleton) + estimatedHeight = targetHmdHeight + updateRecordingScale(trainingStep, 1f / targetHmdHeight) + + if (config.useFrameFiltering) { + // Calculate the initial frame errors and recording stats + val frameErrors = FloatArray(frames.maxFrameCount) + val frameStats = StatsCalculator() + val recordingStats = StatsCalculator() + for (i in 0 until frames.maxFrameCount) { + frameStats.reset() + for (j in 0 until frames.maxFrameCount) { + if (i == j) continue + + trainingStep.setCursors( + i, + j, + updatePlayerCursors = true + ) + + frameStats.addValue(getErrorDeriv(trainingStep)) + } + frameErrors[i] = frameStats.mean + recordingStats.addValue(frameStats.mean) + // LogManager.info("[AutoBone] Frame: ${i + 1}, Mean error: ${frameStats.mean} (SD ${frameStats.standardDeviation})") + } + LogManager.info("[AutoBone] Full recording mean error: ${frameStats.mean} (SD ${frameStats.standardDeviation})") + + // Remove outlier frames + val sdMult = 1.4f + val mean = recordingStats.mean + val sd = recordingStats.standardDeviation * sdMult + for (i in frameErrors.size - 1 downTo 0) { + val err = frameErrors[i] + if (err < mean - sd || err > mean + sd) { + for (frameHolder in frames.frameHolders) { + frameHolder.frames.removeAt(i) + } + } + } + trainingStep.maxFrameCount = frames.maxFrameCount + + // Calculate and print the resulting recording stats + recordingStats.reset() + for (i in 0 until frames.maxFrameCount) { + frameStats.reset() + for (j in 0 until frames.maxFrameCount) { + if (i == j) continue + + trainingStep.setCursors( + i, + j, + updatePlayerCursors = true + ) + + frameStats.addValue(getErrorDeriv(trainingStep)) + } + recordingStats.addValue(frameStats.mean) + } + LogManager.info("[AutoBone] Full recording after mean error: ${frameStats.mean} (SD ${frameStats.standardDeviation})") + } + // Epoch loop, each epoch is one full iteration over the full dataset for (epoch in (if (config.calcInitError) -1 else 0) until config.numEpochs) { // Set the current epoch to process @@ -282,14 +306,18 @@ class AutoBone(server: VRServer) { internalEpoch(trainingStep) } - val finalHeight = calcHeight() + // Scale the normalized offsets to the estimated height for the final result + for (entry in offsets.entries) { + entry.setValue(entry.value * estimatedHeight) + } + LogManager .info( - "[AutoBone] Target height: ${trainingStep.targetHmdHeight}, New height: $finalHeight" + "[AutoBone] Target height: ${trainingStep.targetHmdHeight}, Final height: $estimatedHeight" ) return AutoBoneResults( - finalHeight, + estimatedHeight, trainingStep.targetHmdHeight, trainingStep.errorStats, offsets @@ -333,10 +361,6 @@ class AutoBone(server: VRServer) { while (frameCursor < frameCount - cursorOffset) { val frameCursor2 = frameCursor + cursorOffset - // Apply the current adjusted config to both skeletons - applyConfig(trainingStep.skeleton1) - applyConfig(trainingStep.skeleton2) - // Then set the frame cursors and apply them to both skeletons if (config.randomizeFrameOrder && randomFrameIndices != null) { trainingStep @@ -368,9 +392,20 @@ class AutoBone(server: VRServer) { .info( "[AutoBone] Epoch: ${epoch + 1}, Mean error: ${errorStats.mean} (SD ${errorStats.standardDeviation}), Adjust rate: ${trainingStep.curAdjustRate}" ) + LogManager + .info( + "[AutoBone] Target height: ${trainingStep.targetHmdHeight}, Estimated height: $estimatedHeight" + ) } - trainingStep.epochCallback?.accept(Epoch(epoch + 1, config.numEpochs, errorStats, offsets)) + if (trainingStep.epochCallback != null) { + // Scale the normalized offsets to the estimated height for the callback + val scaledOffsets = EnumMap(offsets) + for (entry in scaledOffsets.entries) { + entry.setValue(entry.value * estimatedHeight) + } + trainingStep.epochCallback.accept(Epoch(epoch + 1, config.numEpochs, errorStats, scaledOffsets)) + } } private fun internalIter(trainingStep: AutoBoneStep) { @@ -378,9 +413,43 @@ class AutoBone(server: VRServer) { val skeleton1 = trainingStep.skeleton1 val skeleton2 = trainingStep.skeleton2 - val totalLength = getLengthSum(offsets) - val curHeight = calcHeight() - trainingStep.currentHmdHeight = curHeight + // Scaling each step used to mean enforcing the target height, so keep that + // behaviour to retain predictability + if (!trainingStep.config.scaleEachStep) { + // Try to estimate a new height by calculating the height with the lowest + // error between adding or subtracting from the height + val maxHeight = trainingStep.targetHmdHeight + 0.2f + val minHeight = trainingStep.targetHmdHeight - 0.2f + + trainingStep.currentHmdHeight = estimatedHeight + val heightErrorDeriv = getErrorDeriv(trainingStep) + val heightAdjust = errorFunc(heightErrorDeriv) * trainingStep.curAdjustRate + + val negHeight = (estimatedHeight - heightAdjust).coerceIn(minHeight, maxHeight) + updateRecordingScale(trainingStep, 1f / negHeight) + trainingStep.currentHmdHeight = negHeight + val negHeightErrorDeriv = getErrorDeriv(trainingStep) + + val posHeight = (estimatedHeight + heightAdjust).coerceIn(minHeight, maxHeight) + updateRecordingScale(trainingStep, 1f / posHeight) + trainingStep.currentHmdHeight = posHeight + val posHeightErrorDeriv = getErrorDeriv(trainingStep) + + if (negHeightErrorDeriv < heightErrorDeriv && negHeightErrorDeriv < posHeightErrorDeriv) { + estimatedHeight = negHeight + // Apply the negative height scale + updateRecordingScale(trainingStep, 1f / negHeight) + } else if (posHeightErrorDeriv < heightErrorDeriv) { + estimatedHeight = posHeight + // The last estimated height set was the positive adjustment, so no need to apply it again + } else { + // Reset to the initial scale + updateRecordingScale(trainingStep, 1f / estimatedHeight) + } + } + + // Update the heights used for error calculations + trainingStep.currentHmdHeight = estimatedHeight val errorDeriv = getErrorDeriv(trainingStep) val error = errorFunc(errorDeriv) @@ -419,9 +488,10 @@ class AutoBone(server: VRServer) { .getComputedTracker(TrackerRole.RIGHT_FOOT).position - skeleton1.getComputedTracker(TrackerRole.RIGHT_FOOT).position - for (entry in offsets.entries) { - // Skip adjustment if the epoch is before starting (for - // logging only) or if there are no BoneTypes for this value + val intermediateOffsets = EnumMap(offsets) + for (entry in intermediateOffsets.entries) { + // Skip adjustment if the epoch is before starting (for logging only) or + // if there are no BoneTypes for this value if (trainingStep.curEpoch < 0 || entry.key.affectedOffsets.isEmpty()) { break } @@ -443,14 +513,10 @@ class AutoBone(server: VRServer) { ) // Calculate the total effect of the bone based on change in rotation - val dotLength = ( - originalLength - * ((leftDotProduct + rightDotProduct) / 2f) - ) + val dotLength = originalLength * ((leftDotProduct + rightDotProduct) / 2f) - // Scale by the ratio for smooth adjustment and more - // stable results - val curAdjustVal = adjustVal * -dotLength / totalLength + // Scale by the total effect of the bone + val curAdjustVal = adjustVal * -dotLength val newLength = originalLength + curAdjustVal // No small or negative numbers!!! Bad algorithm! @@ -461,6 +527,8 @@ class AutoBone(server: VRServer) { // Apply new offset length skeleton1.setOffset(entry.key, newLength) skeleton2.setOffset(entry.key, newLength) + scaleSkeleton(skeleton1, onlyAdjustedHeight = true) + scaleSkeleton(skeleton2, onlyAdjustedHeight = true) // Update the skeleton poses for the new offset length skeleton1.update() @@ -472,45 +540,73 @@ class AutoBone(server: VRServer) { entry.setValue(newLength) } - // Reset the length to minimize bias in other variables, - // it's applied later - skeleton1.setOffset(entry.key, originalLength) - skeleton2.setOffset(entry.key, originalLength) + // Reset the skeleton values to minimize bias in other variables, it's applied later + applyConfig(skeleton1) + applyConfig(skeleton2) + } + + // Update the offsets from the adjusted ones + offsets.putAll(intermediateOffsets) + + // Normalize the scale, it will be upscaled to the target height later + // We only need to scale height offsets, as other offsets are not affected by height + scaleOffsets(onlyHeightOffsets = true) + + // Apply the normalized offsets to the skeleton + applyConfig(skeleton1) + applyConfig(skeleton2) + } + + /** + * Sums only the adjusted height offsets of the provided HumanPoseManager + */ + private fun sumAdjustedHeightOffsets(humanPoseManager: HumanPoseManager): Float { + var sum = 0f + SkeletonConfigManager.HEIGHT_OFFSETS.forEach { + if (!adjustOffsets.contains(it)) return@forEach + sum += humanPoseManager.getOffset(it) } + return sum + } - if (trainingStep.config.scaleEachStep) { - // Scale to the target height if requested by the config - scaleToTargetHeight(trainingStep) + /** + * Sums only the height offsets of the provided offset map + */ + private fun sumHeightOffsets(offsets: EnumMap = this.offsets): Float { + var sum = 0f + SkeletonConfigManager.HEIGHT_OFFSETS.forEach { + sum += offsets[it] ?: return@forEach } + return sum } - private fun scaleToTargetHeight(trainingStep: AutoBoneStep) { - // Recalculate the height and update it in the AutoBoneStep - val stepHeight = calcHeight() - trainingStep.currentHmdHeight = stepHeight - - if (stepHeight > 0f) { - val stepHeightDiff = trainingStep.targetHmdHeight - stepHeight - for (entry in offsets.entries) { - // Only height variables - if (entry.key == SkeletonConfigOffsets.NECK || - !heightOffsets.contains(entry.key) - ) { - continue - } - val length = entry.value + private fun scaleSkeleton(humanPoseManager: HumanPoseManager, targetHeight: Float = 1f, onlyAdjustedHeight: Boolean = false) { + // Get the scale to apply for the appropriate offsets + val scale = if (onlyAdjustedHeight) { + // Only adjusted height offsets + val adjHeight = sumAdjustedHeightOffsets(humanPoseManager) + // Remove the constant from the target, leaving only the target for adjusted height offsets + val adjTarget = targetHeight - (humanPoseManager.userHeightFromConfig - adjHeight) + // Return only the scale for adjusted offsets + adjTarget / adjHeight + } else { + targetHeight / humanPoseManager.userHeightFromConfig + } + + val offsets = if (onlyAdjustedHeight) SkeletonConfigManager.HEIGHT_OFFSETS else SkeletonConfigOffsets.values + for (offset in offsets) { + if (onlyAdjustedHeight && !adjustOffsets.contains(offset)) continue + humanPoseManager.setOffset(offset, humanPoseManager.getOffset(offset) * scale) + } + } - // Multiply the diff by the length to height - // ratio - val adjVal = stepHeightDiff * (length / stepHeight) + private fun scaleOffsets(offsets: EnumMap = this.offsets, targetHeight: Float = adjustedHeightNormalized, onlyHeightOffsets: Boolean = false) { + // Get the scale to apply for the appropriate offsets + val scale = targetHeight / sumHeightOffsets(offsets) - // Scale the length to fit the target height - entry.setValue( - (length + adjVal / 2f).coerceAtLeast( - 0.01f - ) - ) - } + for (entry in offsets.entries) { + if (onlyHeightOffsets && !SkeletonConfigManager.HEIGHT_OFFSETS.contains(entry.key)) continue + entry.setValue(entry.value * scale) } } @@ -560,7 +656,7 @@ class AutoBone(server: VRServer) { val lengthsString: String get() { val configInfo = StringBuilder() - offsets.forEach { (key: SkeletonConfigOffsets, value: Float) -> + offsets.forEach { (key, value) -> if (configInfo.isNotEmpty()) { configInfo.append(", ") } @@ -610,27 +706,20 @@ class AutoBone(server: VRServer) { fun loadRecordings(): FastList> { val recordings = FastList>() - if (loadDir.isDirectory) { - val files = loadDir.listFiles() - if (files != null) { - for (file in files) { - if (file.isFile && - org.apache.commons.lang3.StringUtils - .endsWithIgnoreCase(file.name, ".pfr") - ) { - LogManager - .info( - "[AutoBone] Detected recording at \"${file.path}\", loading frames..." - ) - val frames = PoseFrameIO.tryReadFromFile(file) - if (frames == null) { - LogManager - .severe("Reading frames from \"${file.path}\" failed...") - } else { - recordings.add(Pair.of(file.name, frames)) - } - } - } + if (!loadDir.isDirectory) return recordings + val files = loadDir.listFiles() ?: return recordings + for (file in files) { + if (!file.isFile || !file.name.endsWith(".pfr", ignoreCase = true)) continue + + LogManager + .info( + "[AutoBone] Detected recording at \"${file.path}\", loading frames..." + ) + val frames = PoseFrameIO.tryReadFromFile(file) + if (frames == null) { + LogManager.severe("Reading frames from \"${file.path}\" failed...") + } else { + recordings.add(Pair.of(file.name, frames)) } } return recordings diff --git a/server/core/src/main/java/dev/slimevr/autobone/AutoBoneHandler.kt b/server/core/src/main/java/dev/slimevr/autobone/AutoBoneHandler.kt index 0b21c91a93..eca2334984 100644 --- a/server/core/src/main/java/dev/slimevr/autobone/AutoBoneHandler.kt +++ b/server/core/src/main/java/dev/slimevr/autobone/AutoBoneHandler.kt @@ -8,11 +8,14 @@ import dev.slimevr.poseframeformat.PoseFrames import dev.slimevr.poseframeformat.PoseRecorder import dev.slimevr.poseframeformat.PoseRecorder.RecordingProgress import dev.slimevr.poseframeformat.trackerdata.TrackerFrameData +import dev.slimevr.poseframeformat.trackerdata.TrackerFrames import dev.slimevr.tracking.processor.config.SkeletonConfigManager import dev.slimevr.tracking.processor.config.SkeletonConfigOffsets import io.eiren.util.StringUtils +import io.eiren.util.collections.FastList import io.eiren.util.logging.LogManager import org.apache.commons.lang3.tuple.Pair +import java.util.* import java.util.concurrent.CopyOnWriteArrayList import java.util.concurrent.locks.ReentrantLock import kotlin.concurrent.thread @@ -61,9 +64,6 @@ class AutoBoneHandler(private val server: VRServer) { } } - val lengthsString: String - get() = autoBone.lengthsString - @Throws(AutoBoneException::class) private fun processFrames(frames: PoseFrames): AutoBoneResults { return autoBone @@ -254,9 +254,7 @@ class AutoBoneHandler(private val server: VRServer) { ) LogManager .severe( - "[AutoBone] No recordings found in \"" + - loadDir.path + - "\" and no recording was done..." + "[AutoBone] No recordings found in \"${loadDir.path}\" and no recording was done..." ) return } @@ -264,110 +262,60 @@ class AutoBoneHandler(private val server: VRServer) { announceProcessStatus(AutoBoneProcessType.PROCESS, "Processing recording(s)...") LogManager.info("[AutoBone] Processing frames...") val errorStats = StatsCalculator() + val offsetStats = EnumMap( + SkeletonConfigOffsets::class.java + ) val skeletonConfigManagerBuffer = SkeletonConfigManager(false) for ((key, value) in frameRecordings) { - LogManager - .info("[AutoBone] Processing frames from \"$key\"...") - val trackers = value.frameHolders - val trackerInfo = StringBuilder() - for (tracker in trackers) { - if (tracker == null) continue - val frame = tracker.tryGetFrame(0) - if (frame?.trackerPosition == null) continue - - // Add a comma if this is not the first item listed - if (trackerInfo.isNotEmpty()) { - trackerInfo.append(", ") - } - trackerInfo.append(frame.trackerPosition.designation) - - // Represent the data flags - val trackerFlags = StringBuilder() - if (frame.hasData(TrackerFrameData.ROTATION)) { - trackerFlags.append("R") - } - if (frame.hasData(TrackerFrameData.POSITION)) { - trackerFlags.append("P") - } - if (frame.hasData(TrackerFrameData.ACCELERATION)) { - trackerFlags.append("A") - } - if (frame.hasData(TrackerFrameData.RAW_ROTATION)) { - trackerFlags.append("r") - } + LogManager.info("[AutoBone] Processing frames from \"$key\"...") + // Output tracker info for the recording + printTrackerInfo(value.frameHolders) - // If there are data flags, print them in brackets after the - // designation - if (trackerFlags.isNotEmpty()) { - trackerInfo.append(" (").append(trackerFlags).append(")") - } - } - LogManager - .info( - "[AutoBone] (" + - trackers.size + - " trackers) [" + - trackerInfo + - "]" - ) + // Actually process the recording val autoBoneResults = processFrames(value) - errorStats.addValue(autoBoneResults.heightDifference) LogManager.info("[AutoBone] Done processing!") // #region Stats/Values + // Accumulate height error + errorStats.addValue(autoBoneResults.heightDifference) + + // Accumulate length values + for (offset in autoBoneResults.configValues) { + val statCalc = offsetStats.getOrPut(offset.key) { + StatsCalculator() + } + // Multiply by 100 to get cm + statCalc.addValue(offset.value * 100f) + } + + // Calculate and output skeleton ratios skeletonConfigManagerBuffer.setOffsets(autoBoneResults.configValues) - val neckLength = skeletonConfigManagerBuffer - .getOffset(SkeletonConfigOffsets.NECK) - val upperChestLength = skeletonConfigManagerBuffer - .getOffset(SkeletonConfigOffsets.UPPER_CHEST) - val chestLength = skeletonConfigManagerBuffer - .getOffset(SkeletonConfigOffsets.CHEST) - val waistLength = skeletonConfigManagerBuffer - .getOffset(SkeletonConfigOffsets.WAIST) - val hipLength = skeletonConfigManagerBuffer - .getOffset(SkeletonConfigOffsets.HIP) - val torsoLength = upperChestLength + chestLength + waistLength + hipLength - val hipWidth = skeletonConfigManagerBuffer - .getOffset(SkeletonConfigOffsets.HIPS_WIDTH) - val legLength = ( - skeletonConfigManagerBuffer - .getOffset(SkeletonConfigOffsets.UPPER_LEG) + - skeletonConfigManagerBuffer - .getOffset(SkeletonConfigOffsets.LOWER_LEG) - ) - val lowerLegLength = skeletonConfigManagerBuffer - .getOffset(SkeletonConfigOffsets.LOWER_LEG) - val neckTorso = neckLength / torsoLength - val chestTorso = (upperChestLength + chestLength) / torsoLength - val torsoWaist = hipWidth / torsoLength - val legTorso = legLength / torsoLength - val legBody = legLength / (torsoLength + neckLength) - val kneeLeg = lowerLegLength / legLength - LogManager - .info( - "[AutoBone] Ratios: [{Neck-Torso: " + - StringUtils.prettyNumber(neckTorso) + - "}, {Chest-Torso: " + - StringUtils.prettyNumber(chestTorso) + - "}, {Torso-Waist: " + - StringUtils.prettyNumber(torsoWaist) + - "}, {Leg-Torso: " + - StringUtils.prettyNumber(legTorso) + - "}, {Leg-Body: " + - StringUtils.prettyNumber(legBody) + - "}, {Knee-Leg: " + - StringUtils.prettyNumber(kneeLeg) + - "}]" - ) - LogManager.info("[AutoBone] Length values: " + autoBone.lengthsString) + printSkeletonRatios(skeletonConfigManagerBuffer) + + LogManager.info("[AutoBone] Length values: ${autoBone.lengthsString}") } + // Length value stats + val averageLengthVals = StringBuilder() + offsetStats.forEach { (key, value) -> + if (averageLengthVals.isNotEmpty()) { + averageLengthVals.append(", ") + } + averageLengthVals + .append(key.configKey) + .append(": ") + .append(StringUtils.prettyNumber(value.mean, 2)) + .append(" (SD ") + .append(StringUtils.prettyNumber(value.standardDeviation, 2)) + .append(")") + } + LogManager.info("[AutoBone] Average length values: $averageLengthVals") + + // Height error stats LogManager .info( - "[AutoBone] Average height error: " + - StringUtils.prettyNumber(errorStats.mean, 6) + - " (SD " + - StringUtils.prettyNumber(errorStats.standardDeviation, 6) + - ")" + "[AutoBone] Average height error: ${ + StringUtils.prettyNumber(errorStats.mean, 6) + } (SD ${StringUtils.prettyNumber(errorStats.standardDeviation, 6)})" ) // #endregion listeners.forEach { listener: AutoBoneListener -> listener.onAutoBoneEnd(autoBone.offsets) } @@ -390,6 +338,71 @@ class AutoBoneHandler(private val server: VRServer) { } } + private fun printTrackerInfo(trackers: FastList) { + val trackerInfo = StringBuilder() + for (tracker in trackers) { + val frame = tracker?.tryGetFrame(0) ?: continue + + // Add a comma if this is not the first item listed + if (trackerInfo.isNotEmpty()) { + trackerInfo.append(", ") + } + + trackerInfo.append(frame.tryGetTrackerPosition()?.designation ?: "unassigned") + + // Represent the data flags + val trackerFlags = StringBuilder() + if (frame.hasData(TrackerFrameData.ROTATION)) { + trackerFlags.append("R") + } + if (frame.hasData(TrackerFrameData.POSITION)) { + trackerFlags.append("P") + } + if (frame.hasData(TrackerFrameData.ACCELERATION)) { + trackerFlags.append("A") + } + if (frame.hasData(TrackerFrameData.RAW_ROTATION)) { + trackerFlags.append("r") + } + + // If there are data flags, print them in brackets after the designation + if (trackerFlags.isNotEmpty()) { + trackerInfo.append(" (").append(trackerFlags).append(")") + } + } + LogManager.info("[AutoBone] (${trackers.size} trackers) [$trackerInfo]") + } + + private fun printSkeletonRatios(skeleton: SkeletonConfigManager) { + val neckLength = skeleton.getOffset(SkeletonConfigOffsets.NECK) + val upperChestLength = skeleton.getOffset(SkeletonConfigOffsets.UPPER_CHEST) + val chestLength = skeleton.getOffset(SkeletonConfigOffsets.CHEST) + val waistLength = skeleton.getOffset(SkeletonConfigOffsets.WAIST) + val hipLength = skeleton.getOffset(SkeletonConfigOffsets.HIP) + val torsoLength = upperChestLength + chestLength + waistLength + hipLength + val hipWidth = skeleton.getOffset(SkeletonConfigOffsets.HIPS_WIDTH) + val legLength = skeleton.getOffset(SkeletonConfigOffsets.UPPER_LEG) + + skeleton.getOffset(SkeletonConfigOffsets.LOWER_LEG) + val lowerLegLength = skeleton.getOffset(SkeletonConfigOffsets.LOWER_LEG) + + val neckTorso = neckLength / torsoLength + val chestTorso = (upperChestLength + chestLength) / torsoLength + val torsoWaist = hipWidth / torsoLength + val legTorso = legLength / torsoLength + val legBody = legLength / (torsoLength + neckLength) + val kneeLeg = lowerLegLength / legLength + + LogManager.info( + "[AutoBone] Ratios: [{Neck-Torso: ${ + StringUtils.prettyNumber(neckTorso)}}, {Chest-Torso: ${ + StringUtils.prettyNumber(chestTorso)}}, {Torso-Waist: ${ + StringUtils.prettyNumber(torsoWaist)}}, {Leg-Torso: ${ + StringUtils.prettyNumber(legTorso)}}, {Leg-Body: ${ + StringUtils.prettyNumber(legBody)}}, {Knee-Leg: ${ + StringUtils.prettyNumber(kneeLeg)}}]" + ) + } + fun applyValues() { autoBone.applyAndSaveConfig() } diff --git a/server/core/src/main/java/dev/slimevr/autobone/AutoBoneStep.kt b/server/core/src/main/java/dev/slimevr/autobone/AutoBoneStep.kt index 1ab792d9ae..a0b802f679 100644 --- a/server/core/src/main/java/dev/slimevr/autobone/AutoBoneStep.kt +++ b/server/core/src/main/java/dev/slimevr/autobone/AutoBoneStep.kt @@ -23,7 +23,7 @@ class AutoBoneStep( val eyeHeightToHeightRatio: Float = targetHmdHeight / targetFullHeight - val maxFrameCount = frames.maxFrameCount + var maxFrameCount = frames.maxFrameCount val framePlayer1 = TrackerFramesPlayer(frames) val framePlayer2 = TrackerFramesPlayer(frames) diff --git a/server/core/src/main/java/dev/slimevr/autobone/errors/BodyProportionError.kt b/server/core/src/main/java/dev/slimevr/autobone/errors/BodyProportionError.kt index 678f363037..f0a71765c8 100644 --- a/server/core/src/main/java/dev/slimevr/autobone/errors/BodyProportionError.kt +++ b/server/core/src/main/java/dev/slimevr/autobone/errors/BodyProportionError.kt @@ -13,7 +13,8 @@ class BodyProportionError : IAutoBoneError { override fun getStepError(trainingStep: AutoBoneStep): Float { return getBodyProportionError( trainingStep.skeleton1, - trainingStep.currentHmdHeight / trainingStep.eyeHeightToHeightRatio + // Skeletons are now normalized to reduce bias, so height is always 1 + 1f ) } @@ -52,7 +53,7 @@ class BodyProportionError : IAutoBoneError { // Experimental: 0.059 RangeProportionLimiter( 0.059f, - { config: HumanPoseManager -> config.getOffset(SkeletonConfigOffsets.HEAD) }, + SkeletonConfigOffsets.HEAD, 0.01f ), // Neck @@ -60,35 +61,35 @@ class BodyProportionError : IAutoBoneError { // Experimental: 0.059 RangeProportionLimiter( 0.054f, - { config: HumanPoseManager -> config.getOffset(SkeletonConfigOffsets.NECK) }, + SkeletonConfigOffsets.NECK, 0.0015f ), // Upper Chest // Experimental: 0.0945 RangeProportionLimiter( 0.0945f, - { config: HumanPoseManager -> config.getOffset(SkeletonConfigOffsets.UPPER_CHEST) }, + SkeletonConfigOffsets.UPPER_CHEST, 0.01f ), // Chest // Experimental: 0.0945 RangeProportionLimiter( 0.0945f, - { config: HumanPoseManager -> config.getOffset(SkeletonConfigOffsets.CHEST) }, + SkeletonConfigOffsets.CHEST, 0.01f ), // Waist // Experimental: 0.118 RangeProportionLimiter( 0.118f, - { config: HumanPoseManager -> config.getOffset(SkeletonConfigOffsets.WAIST) }, + SkeletonConfigOffsets.WAIST, 0.05f ), // Hip // Experimental: 0.0237 RangeProportionLimiter( 0.0237f, - { config: HumanPoseManager -> config.getOffset(SkeletonConfigOffsets.HIP) }, + SkeletonConfigOffsets.HIP, 0.01f ), // Hip Width @@ -96,14 +97,14 @@ class BodyProportionError : IAutoBoneError { // Experimental: 0.154 RangeProportionLimiter( 0.184f, - { config: HumanPoseManager -> config.getOffset(SkeletonConfigOffsets.HIPS_WIDTH) }, + SkeletonConfigOffsets.HIPS_WIDTH, 0.04f ), // Upper Leg // Expected: 0.245 RangeProportionLimiter( 0.245f, - { config: HumanPoseManager -> config.getOffset(SkeletonConfigOffsets.UPPER_LEG) }, + SkeletonConfigOffsets.UPPER_LEG, 0.015f ), // Lower Leg @@ -111,25 +112,12 @@ class BodyProportionError : IAutoBoneError { // offset?) RangeProportionLimiter( 0.285f, - { config: HumanPoseManager -> config.getOffset(SkeletonConfigOffsets.LOWER_LEG) }, + SkeletonConfigOffsets.LOWER_LEG, 0.02f ) ) @JvmStatic - fun getProportionLimitForOffset(offset: SkeletonConfigOffsets): ProportionLimiter? { - return when (offset) { - SkeletonConfigOffsets.HEAD -> proportionLimits[0] - SkeletonConfigOffsets.NECK -> proportionLimits[1] - SkeletonConfigOffsets.UPPER_CHEST -> proportionLimits[2] - SkeletonConfigOffsets.CHEST -> proportionLimits[3] - SkeletonConfigOffsets.WAIST -> proportionLimits[4] - SkeletonConfigOffsets.HIP -> proportionLimits[5] - SkeletonConfigOffsets.HIPS_WIDTH -> proportionLimits[6] - SkeletonConfigOffsets.UPPER_LEG -> proportionLimits[7] - SkeletonConfigOffsets.LOWER_LEG -> proportionLimits[8] - else -> null - } - } + val proportionLimitMap = proportionLimits.associateBy { it.skeletonConfigOffset } } } diff --git a/server/core/src/main/java/dev/slimevr/autobone/errors/proportions/HardProportionLimiter.kt b/server/core/src/main/java/dev/slimevr/autobone/errors/proportions/HardProportionLimiter.kt index 33e427273c..81fca431eb 100644 --- a/server/core/src/main/java/dev/slimevr/autobone/errors/proportions/HardProportionLimiter.kt +++ b/server/core/src/main/java/dev/slimevr/autobone/errors/proportions/HardProportionLimiter.kt @@ -1,19 +1,18 @@ package dev.slimevr.autobone.errors.proportions import dev.slimevr.tracking.processor.HumanPoseManager -import java.util.function.Function +import dev.slimevr.tracking.processor.config.SkeletonConfigOffsets /** * @param targetRatio The bone to height ratio to target - * @param boneLengthFunction A function that takes a SkeletonConfig object - * and returns the bone length + * @param skeletonConfigOffset The SkeletonConfigOffset to use for the length */ open class HardProportionLimiter( override val targetRatio: Float = 0f, - protected val boneLengthFunction: Function, + override val skeletonConfigOffset: SkeletonConfigOffsets, ) : ProportionLimiter { override fun getProportionError(humanPoseManager: HumanPoseManager, height: Float): Float { - val boneLength = boneLengthFunction.apply(humanPoseManager) + val boneLength = humanPoseManager.getOffset(skeletonConfigOffset) return targetRatio - boneLength / height } } diff --git a/server/core/src/main/java/dev/slimevr/autobone/errors/proportions/ProportionLimiter.kt b/server/core/src/main/java/dev/slimevr/autobone/errors/proportions/ProportionLimiter.kt index 1a963ddbf9..f3fbbdf02e 100644 --- a/server/core/src/main/java/dev/slimevr/autobone/errors/proportions/ProportionLimiter.kt +++ b/server/core/src/main/java/dev/slimevr/autobone/errors/proportions/ProportionLimiter.kt @@ -1,8 +1,10 @@ package dev.slimevr.autobone.errors.proportions import dev.slimevr.tracking.processor.HumanPoseManager +import dev.slimevr.tracking.processor.config.SkeletonConfigOffsets interface ProportionLimiter { fun getProportionError(humanPoseManager: HumanPoseManager, height: Float): Float val targetRatio: Float + val skeletonConfigOffset: SkeletonConfigOffsets } diff --git a/server/core/src/main/java/dev/slimevr/autobone/errors/proportions/RangeProportionLimiter.kt b/server/core/src/main/java/dev/slimevr/autobone/errors/proportions/RangeProportionLimiter.kt index 7a3bdafd2f..c4f4875a61 100644 --- a/server/core/src/main/java/dev/slimevr/autobone/errors/proportions/RangeProportionLimiter.kt +++ b/server/core/src/main/java/dev/slimevr/autobone/errors/proportions/RangeProportionLimiter.kt @@ -2,7 +2,7 @@ package dev.slimevr.autobone.errors.proportions import com.jme3.math.FastMath import dev.slimevr.tracking.processor.HumanPoseManager -import java.util.function.Function +import dev.slimevr.tracking.processor.config.SkeletonConfigOffsets class RangeProportionLimiter : HardProportionLimiter { private val targetPositiveRange: Float @@ -10,15 +10,14 @@ class RangeProportionLimiter : HardProportionLimiter { /** * @param targetRatio The bone to height ratio to target - * @param boneLengthFunction A function that takes a SkeletonConfig object - * and returns the bone length + * @param skeletonConfigOffset The SkeletonConfigOffset to use for the length * @param range The range from the target ratio to accept (ex. 0.1) */ constructor( targetRatio: Float, - boneLengthFunction: Function, + skeletonConfigOffset: SkeletonConfigOffsets, range: Float, - ) : super(targetRatio, boneLengthFunction) { + ) : super(targetRatio, skeletonConfigOffset) { val absRange = FastMath.abs(range) // Handle if someone puts in a negative value @@ -28,8 +27,7 @@ class RangeProportionLimiter : HardProportionLimiter { /** * @param targetRatio The bone to height ratio to target - * @param boneLengthFunction A function that takes a SkeletonConfig object - * and returns the bone length + * @param skeletonConfigOffset The SkeletonConfigOffset to use for the length * @param positiveRange The positive range from the target ratio to accept * (ex. 0.1) * @param negativeRange The negative range from the target ratio to accept @@ -37,10 +35,10 @@ class RangeProportionLimiter : HardProportionLimiter { */ constructor( targetRatio: Float, - boneLengthFunction: Function, + skeletonConfigOffset: SkeletonConfigOffsets, positiveRange: Float, negativeRange: Float, - ) : super(targetRatio, boneLengthFunction) { + ) : super(targetRatio, skeletonConfigOffset) { // If the positive range is less than the negative range, something is // wrong @@ -50,7 +48,7 @@ class RangeProportionLimiter : HardProportionLimiter { } override fun getProportionError(humanPoseManager: HumanPoseManager, height: Float): Float { - val boneLength = boneLengthFunction.apply(humanPoseManager) + val boneLength = humanPoseManager.getOffset(skeletonConfigOffset) val ratioOffset = targetRatio - boneLength / height // If the range is exceeded, return the offset from the range limit diff --git a/server/core/src/main/java/dev/slimevr/config/AutoBoneConfig.kt b/server/core/src/main/java/dev/slimevr/config/AutoBoneConfig.kt index d931f568ec..6529f64f95 100644 --- a/server/core/src/main/java/dev/slimevr/config/AutoBoneConfig.kt +++ b/server/core/src/main/java/dev/slimevr/config/AutoBoneConfig.kt @@ -9,9 +9,9 @@ class AutoBoneConfig { var initialAdjustRate = 10.0f var adjustRateDecay = 1.0f var slideErrorFactor = 0.0f - var offsetSlideErrorFactor = 2.0f + var offsetSlideErrorFactor = 1.0f var footHeightOffsetErrorFactor = 0.0f - var bodyProportionErrorFactor = 0.825f + var bodyProportionErrorFactor = 0.25f var heightErrorFactor = 0.0f var positionErrorFactor = 0.0f var positionOffsetErrorFactor = 0.0f @@ -25,4 +25,5 @@ class AutoBoneConfig { var saveRecordings = false var useSkeletonHeight = false var randSeed = 4L + var useFrameFiltering = false } diff --git a/server/core/src/main/java/dev/slimevr/config/CurrentVRConfigConverter.java b/server/core/src/main/java/dev/slimevr/config/CurrentVRConfigConverter.java index 9042f2c5ea..ac1c739db1 100644 --- a/server/core/src/main/java/dev/slimevr/config/CurrentVRConfigConverter.java +++ b/server/core/src/main/java/dev/slimevr/config/CurrentVRConfigConverter.java @@ -271,6 +271,20 @@ public ObjectNode convert( } } } + if (version < 12) { + // Update AutoBone defaults + ObjectNode autoBoneNode = (ObjectNode) modelData.get("autoBone"); + if (autoBoneNode != null) { + JsonNode offsetSlideNode = autoBoneNode.get("offsetSlideErrorFactor"); + if (offsetSlideNode != null && offsetSlideNode.floatValue() == 2.0f) { + autoBoneNode.set("offsetSlideErrorFactor", new FloatNode(1.0f)); + } + JsonNode bodyProportionsNode = autoBoneNode.get("bodyProportionErrorFactor"); + if (bodyProportionsNode != null && bodyProportionsNode.floatValue() == 0.825f) { + autoBoneNode.set("bodyProportionErrorFactor", new FloatNode(0.25f)); + } + } + } } catch (Exception e) { LogManager.severe("Error during config migration: " + e); } diff --git a/server/core/src/main/java/dev/slimevr/config/VRConfig.java b/server/core/src/main/java/dev/slimevr/config/VRConfig.java index 9210be8887..233d316707 100644 --- a/server/core/src/main/java/dev/slimevr/config/VRConfig.java +++ b/server/core/src/main/java/dev/slimevr/config/VRConfig.java @@ -14,7 +14,7 @@ @JsonVersionedModel( - currentVersion = "11", defaultDeserializeToVersion = "11", toCurrentConverterClass = CurrentVRConfigConverter.class + currentVersion = "12", defaultDeserializeToVersion = "12", toCurrentConverterClass = CurrentVRConfigConverter.class ) public class VRConfig { diff --git a/server/core/src/main/java/dev/slimevr/poseframeformat/player/PlayerTracker.kt b/server/core/src/main/java/dev/slimevr/poseframeformat/player/PlayerTracker.kt index 7d685fd0c2..0421613820 100644 --- a/server/core/src/main/java/dev/slimevr/poseframeformat/player/PlayerTracker.kt +++ b/server/core/src/main/java/dev/slimevr/poseframeformat/player/PlayerTracker.kt @@ -3,7 +3,7 @@ package dev.slimevr.poseframeformat.player import dev.slimevr.poseframeformat.trackerdata.TrackerFrames import dev.slimevr.tracking.trackers.Tracker -class PlayerTracker(val trackerFrames: TrackerFrames, val tracker: Tracker, private var internalCursor: Int = 0) { +class PlayerTracker(val trackerFrames: TrackerFrames, val tracker: Tracker, private var internalCursor: Int = 0, private var internalScale: Float = 1f) { var cursor: Int get() = internalCursor @@ -13,6 +13,13 @@ class PlayerTracker(val trackerFrames: TrackerFrames, val tracker: Tracker, priv setTrackerStateFromIndex(limitedCursor) } + var scale: Float + get() = internalScale + set(value) { + internalScale = value + setTrackerStateFromIndex() + } + init { setTrackerStateFromIndex(limitCursor()) } @@ -54,12 +61,12 @@ class PlayerTracker(val trackerFrames: TrackerFrames, val tracker: Tracker, priv val position = frame.tryGetPosition() if (position != null) { - tracker.position = position + tracker.position = position * internalScale } val acceleration = frame.tryGetAcceleration() if (acceleration != null) { - tracker.setAcceleration(acceleration) + tracker.setAcceleration(acceleration * internalScale) } } } diff --git a/server/core/src/main/java/dev/slimevr/poseframeformat/player/TrackerFramesPlayer.kt b/server/core/src/main/java/dev/slimevr/poseframeformat/player/TrackerFramesPlayer.kt index 060636bba0..601d6faf68 100644 --- a/server/core/src/main/java/dev/slimevr/poseframeformat/player/TrackerFramesPlayer.kt +++ b/server/core/src/main/java/dev/slimevr/poseframeformat/player/TrackerFramesPlayer.kt @@ -35,4 +35,10 @@ class TrackerFramesPlayer(vararg val frameHolders: TrackerFrames) { playerTracker.cursor = index } } + + fun setScales(scale: Float) { + for (playerTracker in playerTrackers) { + playerTracker.scale = scale + } + } } diff --git a/server/core/src/main/java/dev/slimevr/tracking/processor/config/SkeletonConfigManager.java b/server/core/src/main/java/dev/slimevr/tracking/processor/config/SkeletonConfigManager.java index d027c3cfc7..bbbf60df4d 100644 --- a/server/core/src/main/java/dev/slimevr/tracking/processor/config/SkeletonConfigManager.java +++ b/server/core/src/main/java/dev/slimevr/tracking/processor/config/SkeletonConfigManager.java @@ -16,6 +16,16 @@ public class SkeletonConfigManager { + public static final SkeletonConfigOffsets[] HEIGHT_OFFSETS = new SkeletonConfigOffsets[] { + SkeletonConfigOffsets.NECK, + SkeletonConfigOffsets.UPPER_CHEST, + SkeletonConfigOffsets.CHEST, + SkeletonConfigOffsets.WAIST, + SkeletonConfigOffsets.HIP, + SkeletonConfigOffsets.UPPER_LEG, + SkeletonConfigOffsets.LOWER_LEG + }; + protected final EnumMap configOffsets = new EnumMap<>( SkeletonConfigOffsets.class ); @@ -124,13 +134,11 @@ public float getOffset(SkeletonConfigOffsets config) { } private float calculateUserHeight() { - return getOffset(SkeletonConfigOffsets.NECK) - + getOffset(SkeletonConfigOffsets.UPPER_CHEST) - + getOffset(SkeletonConfigOffsets.CHEST) - + getOffset(SkeletonConfigOffsets.WAIST) - + getOffset(SkeletonConfigOffsets.HIP) - + getOffset(SkeletonConfigOffsets.UPPER_LEG) - + getOffset(SkeletonConfigOffsets.LOWER_LEG); + float height = 0f; + for (SkeletonConfigOffsets offset : HEIGHT_OFFSETS) { + height += getOffset(offset); + } + return height; } public float getUserHeightFromOffsets() { @@ -431,7 +439,8 @@ public void resetOffset(SkeletonConfigOffsets config) { / BodyProportionError.eyeHeightToHeightRatio; if (height > 0.5f) { // Reset only if floor level seems right, ProportionLimiter proportionLimiter = BodyProportionError - .getProportionLimitForOffset(config); + .getProportionLimitMap() + .get(config); if (proportionLimiter != null) { setOffset( config,