Prediction policies

.github/workflows/prediction-tests.yml

@@ -19,6 +19,11 @@ on:
         description: "Seconds before one profile is treated as hung"
         required: true
         default: "420"
+      jobTempLeakValidation:
+        description: "Enable Unity JobTemp allocation diagnostics (high overhead)"
+        required: false
+        type: boolean
+        default: false
 
 jobs:
   prediction-tests:
@@ -33,9 +38,10 @@ jobs:
         mode: [server, host]
 
     env:
-      CLIENT_COUNT: ${{ inputs.clientCount || '3' }}
+      CLIENT_COUNT: ${{ inputs.clientCount || '5' }}
       CONNECT_DELAY: ${{ inputs.connectionDelaySeconds || '1' }}
       PROFILE_TIMEOUT_SECONDS: ${{ inputs.profileTimeoutSeconds || '420' }}
+      JOB_TEMP_LEAK_VALIDATION: ${{ inputs.jobTempLeakValidation && 'true' || 'false' }}
 
     steps:
       - name: Checkout
@@ -103,6 +109,10 @@ jobs:
 
           RUN=(xvfb-run -a --server-args="-screen 0 1024x768x24")
           FAIL=0
+          DIAGNOSTIC_ARGS=()
+          if [ "$JOB_TEMP_LEAK_VALIDATION" = "true" ]; then
+            DIAGNOSTIC_ARGS=(-diag-job-temp-memory-leak-validation)
+          fi
 
           terminate_process_tree() {
             local pid="$1"
@@ -136,7 +146,9 @@ jobs:
 
             echo "::group::Launching $MODE process ($PROFILE)"
             "${RUN[@]}" "$BIN" -batchmode -nographics \
+              "${DIAGNOSTIC_ARGS[@]}" \
               -role "$MODE" -count "$TOTAL_COUNT" -port "$PORT" \
+              -connectTimeout 180 \
               "${LATENCY_ARGS[@]}" \
               -results "$RESULTS_DIR/$MODE.json" \
               -logFile "$LOGS_DIR/$MODE.log" &
@@ -153,18 +165,44 @@ jobs:
               return
             fi
 
+            READY=0
+            for _ in $(seq 1 60); do
+              if grep -q '\[PredictionTests\].*local connection ready' "$LOGS_DIR/$MODE.log" 2>/dev/null; then
+                READY=1
+                break
+              fi
+              if ! kill -0 "$SERVER_PID" 2>/dev/null; then
+                break
+              fi
+              sleep 1
+            done
+
+            if [ "$READY" -ne 1 ]; then
+              echo "::error::$MODE process did not become locally ready before clients started ($PROFILE)"
+              tail -n 200 "$LOGS_DIR/$MODE.log" || true
+              terminate_process_tree "$SERVER_PID"
+              wait "$SERVER_PID" 2>/dev/null || true
+              FAIL=1
+              return
+            fi
+
             CLIENT_PIDS=()
             for i in $(seq 1 "$EXTERNAL_CLIENTS"); do
               echo "::group::Launching client $i ($PROFILE)"
               "${RUN[@]}" "$BIN" -batchmode -nographics \
-                -role client -port "$PORT" \
+                "${DIAGNOSTIC_ARGS[@]}" \
+                -role client -count "$TOTAL_COUNT" -port "$PORT" \
+                -connectTimeout 180 \
                 "${LATENCY_ARGS[@]}" \
                 -results "$RESULTS_DIR/client-$i.json" \
                 -logFile "$LOGS_DIR/client-$i.log" &
               PID=$!
               CLIENT_PIDS+=("$PID")
               echo "Client $i PID: $PID"
               echo "::endgroup::"
+              if [ "$i" -lt "$EXTERNAL_CLIENTS" ]; then
+                sleep 1
+              fi
             done
 
             ALL_PIDS=("$SERVER_PID" "${CLIENT_PIDS[@]}")
@@ -262,11 +300,13 @@ jobs:
 
           JQ_LIB='
             def humanBytes:
-              if . < 1024 then "\(.) B"
+              if . == null then "—"
+              elif . < 1024 then "\(.) B"
               elif . < 1048576 then "\(((. * 10) / 1024 | floor) / 10) KB"
               else "\(((. * 100) / 1048576 | floor) / 100) MB" end;
             def humanDuration:
-              if . < 1000 then "\(((. * 10) | floor) / 10)ms"
+              if . == null then "—"
+              elif . < 1000 then "\(((. * 10) | floor) / 10)ms"
               else "\(((. * 100) / 1000 | floor) / 100)s" end;
             def statusIcon:
               if . then ":white_check_mark:" else ":x:" end;

Assets/PredictionTests/Bootstrap.unity

@@ -279,6 +279,10 @@ GameObject:
   - component: {fileID: 120003}
   - component: {fileID: 120004}
   - component: {fileID: 120005}
+  - component: {fileID: 120006}
+  - component: {fileID: 120007}
+  - component: {fileID: 120008}
+  - component: {fileID: 120009}
   m_Layer: 0
   m_Name: PredictedSystems
   m_TagString: Untagged
@@ -359,10 +363,71 @@ MonoBehaviour:
   m_Enabled: 1
   m_EditorHideFlags: 0
   m_Script: {fileID: 11500000, guid: e3f8a1c20d934b6e8a51f0c4d7b2101a, type: 3}
-  m_Name: 
-  m_EditorClassIdentifier: 
+  m_Name:
+  m_EditorClassIdentifier:
   _spawnDelay:
     rawValue: 1077936128
+--- !u!114 &120006
+MonoBehaviour:
+  m_ObjectHideFlags: 0
+  m_CorrespondingSourceObject: {fileID: 0}
+  m_PrefabInstance: {fileID: 0}
+  m_PrefabAsset: {fileID: 0}
+  m_GameObject: {fileID: 120000}
+  m_Enabled: 1
+  m_EditorHideFlags: 0
+  m_Script: {fileID: 11500000, guid: e3f8a1c20d934b6e8a51f0c4d7b21024, type: 3}
+  m_Name:
+  m_EditorClassIdentifier:
+  _spawnDelay:
+    rawValue: 1082130432
+  _spawnPosition: {x: 20, y: 5, z: 0}
+--- !u!114 &120007
+MonoBehaviour:
+  m_ObjectHideFlags: 0
+  m_CorrespondingSourceObject: {fileID: 0}
+  m_PrefabInstance: {fileID: 0}
+  m_PrefabAsset: {fileID: 0}
+  m_GameObject: {fileID: 120000}
+  m_Enabled: 1
+  m_EditorHideFlags: 0
+  m_Script: {fileID: 11500000, guid: e3f8a1c20d934b6e8a51f0c4d7b21024, type: 3}
+  m_Name:
+  m_EditorClassIdentifier:
+  _spawnDelay:
+    rawValue: 1084227584
+  _spawnPosition: {x: -20, y: 3, z: 0}
+--- !u!114 &120008
+MonoBehaviour:
+  m_ObjectHideFlags: 0
+  m_CorrespondingSourceObject: {fileID: 0}
+  m_PrefabInstance: {fileID: 0}
+  m_PrefabAsset: {fileID: 0}
+  m_GameObject: {fileID: 120000}
+  m_Enabled: 1
+  m_EditorHideFlags: 0
+  m_Script: {fileID: 11500000, guid: e3f8a1c20d934b6e8a51f0c4d7b21024, type: 3}
+  m_Name:
+  m_EditorClassIdentifier:
+  _spawnDelay:
+    rawValue: 1086324736
+  _spawnPosition: {x: 40, y: 3, z: 0}
+--- !u!114 &120009
+MonoBehaviour:
+  m_ObjectHideFlags: 0
+  m_CorrespondingSourceObject: {fileID: 0}
+  m_PrefabInstance: {fileID: 0}
+  m_PrefabAsset: {fileID: 0}
+  m_GameObject: {fileID: 120000}
+  m_Enabled: 1
+  m_EditorHideFlags: 0
+  m_Script: {fileID: 11500000, guid: e3f8a1c20d934b6e8a51f0c4d7b21024, type: 3}
+  m_Name:
+  m_EditorClassIdentifier:
+  _spawnDelay:
+    rawValue: 1084227584
+  _spawnPosition: {x: 0, y: 8, z: 0}
+  _ownByFirstPlayer: 1
 --- !u!1 &130000
 GameObject:
   m_ObjectHideFlags: 0
@@ -381,6 +446,10 @@ GameObject:
   - component: {fileID: 130008}
   - component: {fileID: 130009}
   - component: {fileID: 130010}
+  - component: {fileID: 130011}
+  - component: {fileID: 130012}
+  - component: {fileID: 130013}
+  - component: {fileID: 130014}
   m_Layer: 0
   m_Name: Bootstrap
   m_TagString: Untagged
@@ -417,7 +486,7 @@ MonoBehaviour:
   m_EditorClassIdentifier: 
   _networkManager: {fileID: 100002}
   _predictionManager: {fileID: 110002}
-  _connectionTimeout: 30
+  _connectionTimeout: 180
   _timeBetweenScenarios: 0.1
   _editorRole: 2
   _editorExpectedConnections: 2
@@ -554,6 +623,70 @@ MonoBehaviour:
   _reconnectTimeout: 30
   _stayDisconnectedSeconds: 1
   _settleSeconds: 3
+--- !u!114 &130011
+MonoBehaviour:
+  m_ObjectHideFlags: 0
+  m_CorrespondingSourceObject: {fileID: 0}
+  m_PrefabInstance: {fileID: 0}
+  m_PrefabAsset: {fileID: 0}
+  m_GameObject: {fileID: 130000}
+  m_Enabled: 1
+  m_EditorHideFlags: 0
+  m_Script: {fileID: 11500000, guid: e3f8a1c20d934b6e8a51f0c4d7b21027, type: 3}
+  m_Name:
+  m_EditorClassIdentifier:
+  _rig: {fileID: 120006}
+  _restSeconds: 3
+  _timeout: 90
+--- !u!114 &130012
+MonoBehaviour:
+  m_ObjectHideFlags: 0
+  m_CorrespondingSourceObject: {fileID: 0}
+  m_PrefabInstance: {fileID: 0}
+  m_PrefabAsset: {fileID: 0}
+  m_GameObject: {fileID: 130000}
+  m_Enabled: 1
+  m_EditorHideFlags: 0
+  m_Script: {fileID: 11500000, guid: e3f8a1c20d934b6e8a51f0c4d7b21028, type: 3}
+  m_Name:
+  m_EditorClassIdentifier:
+  _rig: {fileID: 120007}
+  _minDivergence: 0.3
+  _convergedDistance: 0.15
+  _settleSeconds: 2
+  _timeout: 90
+--- !u!114 &130013
+MonoBehaviour:
+  m_ObjectHideFlags: 0
+  m_CorrespondingSourceObject: {fileID: 0}
+  m_PrefabInstance: {fileID: 0}
+  m_PrefabAsset: {fileID: 0}
+  m_GameObject: {fileID: 130000}
+  m_Enabled: 1
+  m_EditorHideFlags: 0
+  m_Script: {fileID: 11500000, guid: e3f8a1c20d934b6e8a51f0c4d7b2102a, type: 3}
+  m_Name:
+  m_EditorClassIdentifier:
+  _rig: {fileID: 120008}
+  _minDivergence: 0.3
+  _convergedDistance: 0.15
+  _settleSeconds: 2
+  _timeout: 90
+--- !u!114 &130014
+MonoBehaviour:
+  m_ObjectHideFlags: 0
+  m_CorrespondingSourceObject: {fileID: 0}
+  m_PrefabInstance: {fileID: 0}
+  m_PrefabAsset: {fileID: 0}
+  m_GameObject: {fileID: 130000}
+  m_Enabled: 1
+  m_EditorHideFlags: 0
+  m_Script: {fileID: 11500000, guid: e3f8a1c20d934b6e8a51f0c4d7b2102c, type: 3}
+  m_Name:
+  m_EditorClassIdentifier:
+  _rig: {fileID: 120009}
+  _restSeconds: 3
+  _timeout: 90
 --- !u!1660057539 &9223372036854775807
 SceneRoots:
   m_ObjectHideFlags: 0

Assets/PredictionTests/README.md

@@ -15,6 +15,13 @@ Multi-process end-to-end tests for the prediction pipeline, modeled after PurrNe
 | `StaticModuleReuseScenario` | Tick-pooled predicted identities rerun static module setup on reuse and reset list-backed module state |
 | `DynamicModuleShapeScenario` | Reused predicted identities start with no stale dynamic modules, then add/remove different dynamic module shapes |
 | `ProjectileChainReconnectScenario` | A client reconnects during an active projectile/VFX burst, stressing full-sync while dynamic modules and pooled effects churn |
+| `ServerRelayScenario` | ServerRelay bodies remain kinematic on clients and only execute verified ticks |
+| `SoftCorrectionScenario` | 3D soft-corrected bodies tolerate local divergence and converge without replay simulation |
+| `SoftCorrection2DScenario` | 2D soft-corrected bodies follow the same convergence and replay guarantees |
+| `OwnedRelayScenario` | PredictedIfOwned resolves to live prediction for the owner and server relay for non-owners |
+| `SoftCorrectionPoolReuseScenario` | A completed pooled lifetime cannot leak pose-correction accumulators into the next object |
+| `GenericSoftCorrectionScenario` | An opted-in generic state consumes verified deltas and converges without rollback simulation |
+| `ReplayPolicyTransitionScenario` | Entering SoftCorrection during reconcile freezes the body before the replay physics pass |
 
 All scenarios fail on unexpected Unity `Error`, `Assert`, or `Exception` logs during the active scenario. They run with simulated latency (40–80ms by default, configurable on the `Bootstrap` object or via `-latencyMin`/`-latencyMax`; `-latencyMax 0` disables) so rollback depth resembles real conditions instead of a clean localhost.
 
@@ -30,8 +37,11 @@ Build `StandaloneLinux64`/`StandaloneWindows64` with this scene first, then:
 
 ```
 PurrDictionTests -batchmode -nographics -role host -count 3 -results host.json -logFile host.log
-PurrDictionTests -batchmode -nographics -role client -results client-1.json -logFile client-1.log
-PurrDictionTests -batchmode -nographics -role client -results client-2.json -logFile client-2.log
+PurrDictionTests -batchmode -nographics -role client -count 3 -results client-1.json -logFile client-1.log
+PurrDictionTests -batchmode -nographics -role client -count 3 -results client-2.json -logFile client-2.log
 ```
 
 Optional args: `-port`, `-serverHost`, `-connectTimeout`. Exit code is non-zero if any scenario fails. CI runs this via `.github/workflows/prediction-tests.yml` (server and host matrix, IL2CPP).
+
+Policy regression scenarios are included in the normal suite. Pass `-policyRegressionScenariosOnly`
+to run just the bootstrap and the three focused policy scenarios.