feat: xr physical movement

Previously, XR movement was best-effort and had some restrictions and odd behavior.

This change improves XR so that the player physics capsule moves with you when you physically move in vr, and when snap turning it pivots correctly at the players position. Now, if you physically walk of a high ledge the player will actually fall, and if you try to walk through a wall, physics will push back.

Author: ashconnell

src/core/entities/PlayerLocal.js

@@ -1,6 +1,7 @@
 import { Entity } from './Entity'
 import { clamp } from '../utils'
 import * as THREE from '../extras/three'
+import { XRControllerModelFactory } from 'three/addons'
 import { Layers } from '../extras/Layers'
 import { DEG2RAD, RAD2DEG } from '../extras/general'
 import { createNode } from '../extras/createNode'
@@ -105,6 +106,12 @@ export class PlayerLocal extends Entity {
       prevTransform: new THREE.Matrix4(),
     }
 
+    this.xrControllerModelFactory = null
+    this.xrControllerModel1 = null
+    this.xrControllerModel2 = null
+    this.xrRig = new THREE.Object3D()
+    this.xrRig.rotation.reorder('YXZ')
+
     this.mode = Modes.IDLE
     this.axis = new THREE.Vector3()
     this.gaze = new THREE.Vector3()
@@ -117,6 +124,9 @@ export class PlayerLocal extends Entity {
     this.base.position.fromArray(this.data.position)
     this.base.quaternion.fromArray(this.data.quaternion)
 
+    this.hmdDelta = new THREE.Vector3()
+    this.hmdLast = new THREE.Vector3()
+
     this.aura = createNode('group')
 
     this.nametag = createNode('nametag', { label: '', health: this.data.health, active: false })
@@ -175,6 +185,7 @@ export class PlayerLocal extends Entity {
     this.initControl()
 
     this.world.setHot(this, true)
+    this.world.on('xrSession', this.onXRSession)
     this.world.emit('ready', true)
   }
 
@@ -310,6 +321,62 @@ export class PlayerLocal extends Entity {
     // this.control.setActions([{ type: 'escape', label: 'Menu' }])
   }
 
+  onXRSession = session => {
+    if (session) {
+      if (!this.xrControllerModel1) {
+        this.xrControllerModelFactory = new XRControllerModelFactory()
+        this.xrControllerModel1 = this.world.graphics.renderer.xr.getControllerGrip(0)
+        this.xrControllerModel1.add(this.xrControllerModelFactory.createControllerModel(this.xrControllerModel1))
+        this.xrRig.add(this.xrControllerModel1)
+        this.xrControllerModel2 = this.world.graphics.renderer.xr.getControllerGrip(1)
+        this.xrControllerModel2.add(this.xrControllerModelFactory.createControllerModel(this.xrControllerModel2))
+        this.xrRig.add(this.xrControllerModel2)
+      }
+      this.world.stage.scene.add(this.xrRig)
+      this.xrRig.add(this.world.camera)
+      this.cam.zoom = 0
+      this.control.camera.write = false
+      this.isXR = true
+    } else {
+      this.world.stage.scene.remove(this.xrRig)
+      this.world.rig.add(this.world.camera)
+      this.world.camera.position.set(0, 0, 0)
+      this.world.camera.rotation.set(0, 0, 0)
+      this.cam.zoom = 1
+      this.control.camera.write = true
+      this.isXR = false
+    }
+  }
+
+  setXRPlayerPosition(position) {
+    const parent = this.xrRig
+    const child = this.world.camera
+    const feetWorldPos = child.getWorldPosition(v2)
+    feetWorldPos.y -= child.position.y
+    const offset = v1.subVectors(position, feetWorldPos)
+    parent.position.add(offset)
+
+    // const offset = v1.copy(position)
+    // const offset = v1.copy(position)
+    // offset.x -= parent.position.x + child.position.x
+    // offset.y -= parent.position.y
+    // offset.z -= parent.position.z + child.position.z
+    // parent.position.add(offset)
+  }
+
+  turnXRRigAtPlayer(degrees) {
+    const parent = this.xrRig
+    const child = this.world.camera
+    // console.log(child.getWorldPosition(new THREE.Vector3()))
+    const pivotWorld = new THREE.Vector3()
+    child.getWorldPosition(pivotWorld)
+    parent.rotateOnAxis(UP, degrees * THREE.MathUtils.DEG2RAD)
+    const offset = child.position.clone()
+    offset.applyQuaternion(parent.quaternion)
+    parent.position.copy(pivotWorld).sub(offset)
+    // console.log(child.getWorldPosition(new THREE.Vector3()))
+  }
+
   toggleFlying(value) {
     value = isBoolean(value) ? value : !this.flying
     if (this.flying === value) return
@@ -353,6 +420,7 @@ export class PlayerLocal extends Entity {
   }
 
   fixedUpdate(delta) {
+    const xr = this.isXR
     const freeze = this.data.effect?.freeze
     const anchor = this.getAnchorMatrix()
     const snare = this.data.effect?.snare || 0
@@ -709,25 +777,55 @@ export class PlayerLocal extends Entity {
   }
 
   update(delta) {
-    const isXR = this.world.xr?.session
+    const xr = this.isXR
     const freeze = this.data.effect?.freeze
     const anchor = this.getAnchorMatrix()
 
+    // if (xr) return
+    // console.log('update')
+
+    if (xr) {
+      // move the rig so that the ground underneath the camera aligns with the base player
+      this.setXRPlayerPosition(this.base.position)
+      // fetch any physical movement delta
+      this.world.camera.getWorldPosition(v1)
+      v1.y = 0
+      v2.copy(this.xrRig.position)
+      v2.y = 0
+      v3.copy(v1).sub(v2)
+      this.hmdDelta.copy(v3).sub(this.hmdLast)
+      this.hmdLast.copy(v3)
+      // apply physical movement delta to capsule so physics stays with us if we wander
+      const pose = this.capsule.getGlobalPose()
+      v2.copy(pose.p).add(this.hmdDelta)
+      v2.toPxVec3(pose.p)
+      this.capsule.setGlobalPose(pose)
+    }
+
     // update cam look direction
-    if (isXR) {
+    if (xr) {
       // in xr clear camera rotation (handled internally)
       // in xr we only track turn here, which is added to the xr camera later on
-      this.cam.rotation.x = 0
-      this.cam.rotation.z = 0
+      // this.cam.rotation.x = 0
+      // this.cam.rotation.z = 0
       if (this.control.xrRightStick.value.x === 0 && this.didSnapTurn) {
         this.didSnapTurn = false
       } else if (this.control.xrRightStick.value.x > 0 && !this.didSnapTurn) {
-        this.cam.rotation.y -= 45 * DEG2RAD
+        this.turnXRRigAtPlayer(-45)
         this.didSnapTurn = true
       } else if (this.control.xrRightStick.value.x < 0 && !this.didSnapTurn) {
-        this.cam.rotation.y += 45 * DEG2RAD
+        this.turnXRRigAtPlayer(45)
         this.didSnapTurn = true
       }
+      // if we did snap turn, we need to refresh the hmd position to cancel it out
+      if (this.didSnapTurn) {
+        this.world.camera.getWorldPosition(v1)
+        v1.y = 0
+        v2.copy(this.xrRig.position)
+        v2.y = 0
+        v3.copy(v1).sub(v2)
+        this.hmdLast.copy(v3)
+      }
     } else if (this.control.pointer.locked) {
       // or pointer lock, rotate camera with pointer movement
       this.cam.rotation.x += -this.control.pointer.delta.y * POINTER_LOOK_SPEED * delta
@@ -741,25 +839,16 @@ export class PlayerLocal extends Entity {
     }
 
     // ensure we can't look too far up/down
-    if (!isXR) {
+    if (!xr) {
       this.cam.rotation.x = clamp(this.cam.rotation.x, -89 * DEG2RAD, 89 * DEG2RAD)
     }
 
     // zoom camera if scrolling wheel
-    if (!isXR) {
+    if (!xr) {
       this.cam.zoom += -this.control.scrollDelta.value * ZOOM_SPEED * delta
       this.cam.zoom = clamp(this.cam.zoom, MIN_ZOOM, MAX_ZOOM)
     }
 
-    // force zoom in xr to trigger first person (below)
-    if (isXR && !this.xrActive) {
-      this.cam.zoom = 0
-      this.xrActive = true
-    } else if (!isXR && this.xrActive) {
-      this.cam.zoom = 1
-      this.xrActive = false
-    }
-
     // transition in and out of first person
     if (this.cam.zoom < 1 && !this.firstPerson) {
       this.cam.zoom = 0
@@ -777,14 +866,14 @@ export class PlayerLocal extends Entity {
     }
 
     // watch jump presses to either fly or air-jump
-    this.jumpDown = isXR ? this.control.xrRightBtn1.down : this.control.space.down || this.control.touchA.down
-    if (isXR ? this.control.xrRightBtn1.pressed : this.control.space.pressed || this.control.touchA.pressed) {
+    this.jumpDown = xr ? this.control.xrRightBtn1.down : this.control.space.down || this.control.touchA.down
+    if (xr ? this.control.xrRightBtn1.pressed : this.control.space.pressed || this.control.touchA.pressed) {
       this.jumpPressed = true
     }
 
     // get our movement direction
     this.moveDir.set(0, 0, 0)
-    if (isXR) {
+    if (xr) {
       // in xr use controller input
       this.moveDir.x = this.control.xrLeftStick.value.x
       this.moveDir.z = this.control.xrLeftStick.value.z
@@ -830,7 +919,7 @@ export class PlayerLocal extends Entity {
     }
 
     // determine if we're "running"
-    if (this.stick?.active || isXR) {
+    if (this.stick?.active || xr) {
       // touch/xr joysticks at full extent
       this.running = this.moving && this.moveDir.length() > 0.9
     } else {
@@ -842,9 +931,10 @@ export class PlayerLocal extends Entity {
     this.moveDir.normalize()
 
     // flying direction
-    if (isXR) {
+    if (xr) {
       this.flyDir.copy(this.moveDir)
-      this.flyDir.applyQuaternion(this.world.xr.camera.quaternion)
+      this.world.camera.getWorldQuaternion(q1)
+      this.flyDir.applyQuaternion(q1)
     } else {
       this.flyDir.copy(this.moveDir)
       this.flyDir.applyQuaternion(this.cam.quaternion)
@@ -868,9 +958,11 @@ export class PlayerLocal extends Entity {
     if (moveDeg < 0) moveDeg += 360
 
     // rotate direction to face camera Y direction
-    if (isXR) {
-      e1.copy(this.world.xr.camera.rotation).reorder('YXZ')
-      e1.y += this.cam.rotation.y
+    if (xr) {
+      this.world.camera.getWorldQuaternion(q1)
+      e1.setFromQuaternion(q1).reorder('YXZ')
+      // e1.y += this.xrRig.rotation.y
+      // e1.y += this.cam.rotation.y // why not world.camera now?
       const yQuaternion = q1.setFromAxisAngle(UP, e1.y)
       this.moveDir.applyQuaternion(yQuaternion)
     } else {
@@ -881,9 +973,11 @@ export class PlayerLocal extends Entity {
     // get initial facing angle matching camera
     let rotY = 0
     let applyRotY
-    if (isXR) {
-      e1.copy(this.world.xr.camera.rotation).reorder('YXZ')
-      rotY = e1.y + this.cam.rotation.y
+    if (xr) {
+      this.world.camera.getWorldQuaternion(q1)
+      e1.setFromQuaternion(q1).reorder('YXZ')
+      rotY = e1.y
+      // rotY = e1.y + this.cam.rotation.y
     } else {
       rotY = this.cam.rotation.y
     }
@@ -932,8 +1026,9 @@ export class PlayerLocal extends Entity {
     this.mode = mode
 
     // set gaze direction
-    if (isXR) {
-      this.gaze.copy(FORWARD).applyQuaternion(this.world.xr.camera.quaternion)
+    if (xr) {
+      this.world.camera.getWorldQuaternion(q1)
+      this.gaze.copy(FORWARD).applyQuaternion(q1)
     } else {
       this.gaze.copy(FORWARD).applyQuaternion(this.cam.quaternion)
       if (!this.firstPerson) {
@@ -1010,8 +1105,12 @@ export class PlayerLocal extends Entity {
   }
 
   lateUpdate(delta) {
-    const isXR = this.world.xr?.session
+    const xr = this.isXR
     const anchor = this.getAnchorMatrix()
+
+    // if (xr) return
+    // console.log('lateUpdate')
+
     // if we're anchored, force into that pose
     if (anchor) {
       this.base.position.setFromMatrixPosition(anchor)
@@ -1022,7 +1121,7 @@ export class PlayerLocal extends Entity {
     }
     // make camera follow our position horizontally
     this.cam.position.copy(this.base.position)
-    if (isXR) {
+    if (xr) {
       // ...
     } else {
       // and vertically at our vrm model height
@@ -1034,10 +1133,10 @@ export class PlayerLocal extends Entity {
         this.cam.position.add(right.multiplyScalar(0.3))
       }
     }
-    if (this.world.xr?.session) {
+    if (xr) {
       // in vr snap camera
-      this.control.camera.position.copy(this.cam.position)
-      this.control.camera.quaternion.copy(this.cam.quaternion)
+      // this.control.camera.position.copy(this.cam.position)
+      // this.control.camera.quaternion.copy(this.cam.quaternion)
     } else {
       // otherwise interpolate camera towards target
       simpleCamLerp(this.world, this.control.camera, this.cam, delta)

src/core/systems/ClientGraphics.js

@@ -63,8 +63,6 @@ export class ClientGraphics extends System {
     this.renderer.toneMappingExposure = 1
     this.renderer.outputColorSpace = THREE.SRGBColorSpace
     this.renderer.xr.enabled = true
-    this.renderer.xr.setReferenceSpaceType('local-floor')
-    this.renderer.xr.setFoveation(1)
     this.maxAnisotropy = this.renderer.capabilities.getMaxAnisotropy()
     THREE.Texture.DEFAULT_ANISOTROPY = this.maxAnisotropy
     this.usePostprocessing = this.world.prefs.postprocessing

src/core/systems/XR.js

@@ -1,11 +1,5 @@
 import { System } from './System'
 import * as THREE from '../extras/three'
-import { XRControllerModelFactory } from 'three/addons'
-
-const UP = new THREE.Vector3(0, 1, 0)
-
-const v1 = new THREE.Vector3()
-const e1 = new THREE.Euler(0, 0, 0, 'YXZ')
 
 /**
  * XR System
@@ -19,11 +13,8 @@ export class XR extends System {
     super(world)
     this.session = null
     this.camera = null
-    this.controller1Model = null
-    this.controller2Model = null
     this.supportsVR = false
     this.supportsAR = false
-    this.controllerModelFactory = new XRControllerModelFactory()
   }
 
   async init() {
@@ -32,6 +23,8 @@ export class XR extends System {
   }
 
   async enter() {
+    this.world.graphics.renderer.xr.setReferenceSpaceType('local-floor')
+    this.world.graphics.renderer.xr.setFoveation(1)
     const session = await navigator.xr?.requestSession('immersive-vr', {
       requiredFeatures: ['local-floor'],
     })
@@ -43,28 +36,13 @@ export class XR extends System {
     }
     this.world.graphics.renderer.xr.setSession(session)
     session.addEventListener('end', this.onSessionEnd)
-    this.session = session
     this.camera = this.world.graphics.renderer.xr.getCamera()
+    this.session = session
     this.world.emit('xrSession', session)
-
-    this.controller1Model = this.world.graphics.renderer.xr.getControllerGrip(0)
-    this.controller1Model.add(this.controllerModelFactory.createControllerModel(this.controller1Model))
-    this.world.rig.add(this.controller1Model)
-
-    this.controller2Model = this.world.graphics.renderer.xr.getControllerGrip(1)
-    this.controller2Model.add(this.controllerModelFactory.createControllerModel(this.controller2Model))
-    this.world.rig.add(this.controller2Model)
   }
 
   onSessionEnd = () => {
-    this.world.camera.position.set(0, 0, 0)
-    this.world.camera.rotation.set(0, 0, 0)
-    this.world.rig.remove(this.controller1Model)
-    this.world.rig.remove(this.controller2Model)
     this.session = null
-    this.camera = null
-    this.controller1Model = null
-    this.controller2Model = null
     this.world.emit('xrSession', null)
   }
 }