diff --git a/README.md b/README.md index 3e2ba0e3a..f0a92adad 100644 --- a/README.md +++ b/README.md @@ -391,6 +391,64 @@ _extends THREE.Camera_ A class indicating that the path tracer should render an equirectangular view. Does not work with three.js raster rendering. +## TemporalResolve + +A class that implements temporal filtering to preserve samples when the camera is moving. This helps reduce noise on camera movement by reprojecting the last frame's samples into the current one. + +### .temporalResolveMix + +```javascript +temporalResolveMix = 0.9 : Number +``` + +How much the last frame should be blended into the current one. Higher values will result in a less noisy look at the cost of more smearing. + +### .clampRadius + +```javascript +clampRadius = 1 : Number +``` + +An integer to set the radius of pixels to be used for neighborhood clamping. Higher values will result in a less noisy look at the cost of more blurring. + +### .newSamplesSmoothing + +```javascript +newSamplesSmoothing = 0.675 : Number +``` + +To reduce noise for pixels that appeared recently, the average of multiple adjacent pixels can be used instead of a pixel itself. This factor determines the influence of the averaged pixel. Higher values will result in less noise but also less sharpness. + +### .newSamplesCorrection + +```javascript +newSamplesCorrection = 1 : Number +``` + +Higher values will make pixels that appeared recently have a greater influence on the output. This will result in more noise but less smearing. + +### .weightTransform + +```javascript +weightTransform = 0 : Number +``` + +This will potentiate the input color by `1 / (1 - weightTransform)` resulting in less contrast and noise when moving the camera. Higher values will highlight darker areas more. + +### .constructor + +```javascript +constructor( ptRenderer : PathTracingRenderer, scene : Object3D, camera : Camera ) +``` + +### .update + +```javascript +update() : void +``` + +Updates the temporal resolve pass for the current frame. + ## PhysicalSpotLight _extends THREE.SpotLight_ diff --git a/example/index.js b/example/index.js index fc68bf1de..fd302d439 100644 --- a/example/index.js +++ b/example/index.js @@ -32,6 +32,7 @@ import { PathTracingSceneWorker } from '../src/workers/PathTracingSceneWorker.js import { PhysicalPathTracingMaterial, PathTracingRenderer, MaterialReducer, BlurredEnvMapGenerator } from '../src/index.js'; import { FullScreenQuad } from 'three/examples/jsm/postprocessing/Pass.js'; import { OrbitControls } from 'three/examples/jsm/controls/OrbitControls.js'; +import { TemporalResolve } from '../src/temporal-resolve/TemporalResolve.js'; const envMaps = { 'Royal Esplanade': 'https://raw.githubusercontent.com/mrdoob/three.js/dev/examples/textures/equirectangular/royal_esplanade_1k.hdr', @@ -71,6 +72,13 @@ const params = { tilesY: 2, samplesPerFrame: 1, + temporalResolve: true, + temporalResolveMix: 0.9, + clampRadius: 1, + newSamplesSmoothing: 0.675, + newSamplesCorrection: 1, + weightTransform: 0, + model: initialModel, envMap: envMaps[ 'Royal Esplanade' ], @@ -104,7 +112,7 @@ const params = { let creditEl, loadingEl, samplesEl; let floorPlane, gui, stats, sceneInfo; let renderer, orthoCamera, perspectiveCamera, activeCamera; -let ptRenderer, fsQuad, controls, scene; +let ptRenderer, fsQuad, controls, scene, temporalResolve; let envMap, envMapGenerator; let loadingModel = false; let delaySamples = 0; @@ -143,6 +151,13 @@ async function init() { ptRenderer.material.bgGradientTop.set( params.bgGradientTop ); ptRenderer.material.bgGradientBottom.set( params.bgGradientBottom ); + temporalResolve = new TemporalResolve( ptRenderer, scene, activeCamera ); + temporalResolve.temporalResolveMix = 0.9; + temporalResolve.clampRadius = 1; + temporalResolve.newSamplesSmoothing = 0.5; + temporalResolve.newSamplesCorrection = 0.75; + temporalResolve.weightTransform = 0; + fsQuad = new FullScreenQuad( new MeshBasicMaterial( { map: ptRenderer.target.texture, blending: CustomBlending @@ -230,6 +245,17 @@ function animate() { } renderer.autoClear = false; + if ( params.temporalResolve ) { + + temporalResolve.update(); + fsQuad.material.map = temporalResolve.target.texture; + + } else { + + fsQuad.material.map = ptRenderer.target.texture; + + } + fsQuad.render( renderer ); renderer.autoClear = true; @@ -245,7 +271,7 @@ function animate() { function resetRenderer() { - if ( params.tilesX * params.tilesY !== 1.0 ) { + if ( ! params.temporalResolve && params.tilesX * params.tilesY !== 1.0 ) { delaySamples = 1; @@ -311,6 +337,24 @@ function buildGui() { } ); + const trFolder = gui.addFolder( 'Temporal Resolve' ); + trFolder.add( params, 'temporalResolve' ); + trFolder + .add( params, 'temporalResolveMix', 0, 1, 0.025 ) + .onChange( ( value ) => ( temporalResolve.temporalResolveMix = value ) ); + trFolder + .add( params, 'clampRadius', 1, 8, 1 ) + .onChange( ( value ) => ( temporalResolve.clampRadius = value ) ); + trFolder + .add( params, 'newSamplesSmoothing', 0, 1, 0.025 ) + .onChange( ( value ) => ( temporalResolve.newSamplesSmoothing = value ) ); + trFolder + .add( params, 'newSamplesCorrection', 0, 1, 0.025 ) + .onChange( ( value ) => ( temporalResolve.newSamplesCorrection = value ) ); + trFolder + .add( params, 'weightTransform', 0, 0.5, 0.025 ) + .onChange( ( value ) => ( temporalResolve.weightTransform = value ) ); + const resolutionFolder = gui.addFolder( 'resolution' ); resolutionFolder.add( params, 'resolutionScale', 0.1, 1.0, 0.01 ).onChange( () => { diff --git a/example/materialBall.js b/example/materialBall.js index afb16869d..05ab52791 100644 --- a/example/materialBall.js +++ b/example/materialBall.js @@ -7,8 +7,9 @@ import { PathTracingSceneWorker } from '../src/workers/PathTracingSceneWorker.js import { RGBELoader } from 'three/examples/jsm/loaders/RGBELoader.js'; import { MeshoptDecoder } from 'three/examples/jsm/libs/meshopt_decoder.module.js'; import { GUI } from 'three/examples/jsm/libs/lil-gui.module.min.js'; +import { TemporalResolve } from '../src/temporal-resolve/TemporalResolve.js'; -let renderer, controls, sceneInfo, ptRenderer, activeCamera, blitQuad, denoiseQuad, materials; +let renderer, controls, sceneInfo, ptRenderer, activeCamera, blitQuad, denoiseQuad, materials, temporalResolve; let perspectiveCamera, orthoCamera, equirectCamera; let envMap, envMapGenerator, scene; let samplesEl; @@ -75,7 +76,6 @@ const params = { matte: false, castShadow: true, }, - multipleImportanceSampling: true, stableNoise: false, denoiseEnabled: true, @@ -90,6 +90,12 @@ const params = { samplesPerFrame: 1, acesToneMapping: true, resolutionScale: 1 / window.devicePixelRatio, + temporalResolve: true, + temporalResolveMix: 0.9, + clampRadius: 2, + newSamplesSmoothing: 0.675, + newSamplesCorrection: 1, + weightTransform: 0, transparentTraversals: 20, filterGlossyFactor: 0.5, tiles: 1, @@ -132,7 +138,7 @@ async function init() { const aspect = window.innerWidth / window.innerHeight; perspectiveCamera = new PhysicalCamera( 75, aspect, 0.025, 500 ); - perspectiveCamera.position.set( - 4, 2, 3 ); + perspectiveCamera.position.set( - 4, 2, 7 ); const orthoHeight = orthoWidth / aspect; orthoCamera = new THREE.OrthographicCamera( orthoWidth / - 2, orthoWidth / 2, orthoHeight / 2, orthoHeight / - 2, 0, 100 ); @@ -167,6 +173,13 @@ async function init() { scene = new THREE.Scene(); + temporalResolve = new TemporalResolve( ptRenderer, scene, activeCamera ); + temporalResolve.temporalResolveMix = 0.9; + temporalResolve.clampRadius = 2; + temporalResolve.newSamplesSmoothing = 0.675; + temporalResolve.newSamplesCorrection = 1; + temporalResolve.weightTransform = 0; + samplesEl = document.getElementById( 'samples' ); envMapGenerator = new BlurredEnvMapGenerator( renderer ); @@ -324,6 +337,23 @@ async function init() { } ); + const trFolder = gui.addFolder( 'Temporal Resolve' ); + trFolder.add( params, 'temporalResolve' ); + trFolder + .add( params, 'temporalResolveMix', 0, 1, 0.025 ) + .onChange( ( value ) => ( temporalResolve.temporalResolveMix = value ) ); + trFolder + .add( params, 'clampRadius', 1, 8, 1 ) + .onChange( ( value ) => ( temporalResolve.clampRadius = value ) ); + trFolder + .add( params, 'newSamplesSmoothing', 0, 1, 0.025 ) + .onChange( ( value ) => ( temporalResolve.newSamplesSmoothing = value ) ); + trFolder + .add( params, 'newSamplesCorrection', 0, 1, 0.025 ) + .onChange( ( value ) => ( temporalResolve.newSamplesCorrection = value ) ); + trFolder + .add( params, 'weightTransform', 0, 0.5, 0.025 ) + .onChange( ( value ) => ( temporalResolve.weightTransform = value ) ); const denoiseFolder = gui.addFolder( 'Denoising' ); denoiseFolder.add( params, 'denoiseEnabled' ); denoiseFolder.add( params, 'denoiseSigma', 0.01, 12.0 ); @@ -647,6 +677,14 @@ function animate() { renderer.autoClear = false; quad.material.map = ptRenderer.target.texture; + + if ( params.temporalResolve ) { + + temporalResolve.update(); + quad.material.map = temporalResolve.target.texture; + + } + quad.render( renderer ); renderer.autoClear = true; @@ -656,4 +694,3 @@ function animate() { - diff --git a/src/index.js b/src/index.js index 102b39ddc..0a45a51a0 100644 --- a/src/index.js +++ b/src/index.js @@ -32,3 +32,6 @@ export * from './materials/PhysicalPathTracingMaterial.js'; export * from './shader/shaderMaterialSampling.js'; export * from './shader/shaderUtils.js'; export * from './shader/shaderStructs.js'; + +// temporal resolve +export * from './temporal-resolve/TemporalResolve.js'; diff --git a/src/temporal-resolve/TemporalResolve.js b/src/temporal-resolve/TemporalResolve.js new file mode 100644 index 000000000..eb575b120 --- /dev/null +++ b/src/temporal-resolve/TemporalResolve.js @@ -0,0 +1,151 @@ +import { FloatType, LinearFilter, WebGLRenderTarget } from 'three'; +import { FullScreenQuad } from 'three/examples/jsm/postprocessing/Pass.js'; +import { ComposeTemporalResolveMaterial } from './materials/ComposeTemporalResolveMaterial.js'; +import { TemporalResolvePass } from './passes/TemporalResolvePass.js'; + +export class TemporalResolve { + + get target() { + + return this.renderTarget; + + } + + constructor( ptRenderer, scene, camera ) { + + this.ptRenderer = ptRenderer; + this.scene = scene; + + // parameters + this.temporalResolveMix = 0.9; + this.clampRadius = 1; + this.newSamplesSmoothing = 0.675; + this.newSamplesCorrection = 1; + + this.fullscreenMaterial = new ComposeTemporalResolveMaterial(); + + this.fsQuad = new FullScreenQuad( this.fullscreenMaterial ); + + this.renderTarget = new WebGLRenderTarget( + typeof window !== 'undefined' ? window.innerWidth : 2000, + typeof window !== 'undefined' ? window.innerHeight : 1000, + { + minFilter: LinearFilter, + magFilter: LinearFilter, + type: FloatType, + depthBuffer: false, + } + ); + + this.lastSize = { width: 0, height: 0 }; + + this.initNewCamera( camera ); + this.initNewSize( window.innerWidth, window.innerHeight ); + + // TODO: move this to a getter / setter + let weightTransform = 0; + Object.defineProperty( this, 'weightTransform', { + set( value ) { + + weightTransform = value; + + this.temporalResolvePass.fullscreenMaterial.defines.WEIGHT_TRANSFORM = ( 1 - value ).toFixed( 5 ); + this.temporalResolvePass.fullscreenMaterial.needsUpdate = true; + + }, + get() { + + return weightTransform; + + } + } ); + + } + + initNewCamera( camera ) { + + this.activeCamera = camera; + + this.temporalResolvePass = new TemporalResolvePass( + this.ptRenderer, + this.scene, + camera + ); + this.temporalResolvePass.fullscreenMaterial.samplesTexture = this.ptRenderer.target.texture; + this.fullscreenMaterial.temporalResolveTexture = this.temporalResolvePass.renderTarget.texture; + + } + + initNewSize( width, height ) { + + this.lastSize.width = width; + this.lastSize.height = height; + + this.renderTarget.setSize( width, height ); + this.temporalResolvePass.setSize( width, height ); + + } + + update() { + + const renderer = this.ptRenderer._renderer; + + // save original values + const origRenderTarget = renderer.getRenderTarget(); + + this.ptRenderer.stableTiles = false; + + const { camera } = this.ptRenderer; + if ( camera !== this.activeCamera ) { + + this.initNewCamera( camera ); + + } + + const { width, height } = this.ptRenderer.target; + if ( width !== this.lastSize.width || height !== this.lastSize.height ) { + + this.initNewSize( width, height ); + + } + + // ensure that the scene's objects' matrices are updated for the VelocityPass + this.scene.updateMatrixWorld(); + + this.scene.traverse( ( c ) => { + + // update the modelViewMatrix which is used by the VelocityPass + c.modelViewMatrix.multiplyMatrices( + this.activeCamera.matrixWorldInverse, + c.matrixWorld + ); + + } ); + + // keep uniforms updated + this.temporalResolvePass.fullscreenMaterial.samples = + this.ptRenderer.samples; + + this.temporalResolvePass.fullscreenMaterial.temporalResolveMix = + this.temporalResolveMix; + + this.temporalResolvePass.fullscreenMaterial.clampRadius = + parseInt( this.clampRadius ); + + this.temporalResolvePass.fullscreenMaterial.newSamplesSmoothing = + this.newSamplesSmoothing; + + this.temporalResolvePass.fullscreenMaterial.newSamplesCorrection = + this.newSamplesCorrection; + + this.temporalResolvePass.render( renderer ); + + renderer.setRenderTarget( this.renderTarget ); + this.fsQuad.render( renderer ); + + // restore original values + renderer.setRenderTarget( origRenderTarget ); + + } + +} diff --git a/src/temporal-resolve/materials/ComposeTemporalResolveMaterial.js b/src/temporal-resolve/materials/ComposeTemporalResolveMaterial.js new file mode 100644 index 000000000..8df027b02 --- /dev/null +++ b/src/temporal-resolve/materials/ComposeTemporalResolveMaterial.js @@ -0,0 +1,44 @@ +import { MaterialBase } from '../../materials/MaterialBase.js'; + +const vertexShader = /* glsl */` + varying vec2 vUv; + + void main() { + + vUv = position.xy * 0.5 + 0.5; + gl_Position = vec4( position.xy, 1.0, 1.0 ); + + } +`; + +const fragmentShader = /* glsl */` + varying vec2 vUv; + + uniform sampler2D temporalResolveTexture; + + void main() { + + vec4 temporalResolveTexel = texture2D( temporalResolveTexture, vUv ); + + bool isBackgroundTile = temporalResolveTexel.a == 0.0; + + gl_FragColor = vec4( temporalResolveTexel.rgb, isBackgroundTile ? 0.0 : 1.0 ); + + } +`; + +export class ComposeTemporalResolveMaterial extends MaterialBase { + + constructor() { + + super( { + vertexShader, + fragmentShader, + uniforms: { + temporalResolveTexture: { value: null }, + }, + } ); + + } + +} diff --git a/src/temporal-resolve/materials/TemporalResolveMaterial.js b/src/temporal-resolve/materials/TemporalResolveMaterial.js new file mode 100644 index 000000000..e257be71f --- /dev/null +++ b/src/temporal-resolve/materials/TemporalResolveMaterial.js @@ -0,0 +1,42 @@ +import { Matrix4 } from 'three'; +import { MaterialBase } from '../../materials/MaterialBase.js'; +import { fragmentShader } from './shaders/temporalResolveFragment.js'; +import { vertexShader } from './shaders/temporalResolveVertex.js'; + +export class TemporalResolveMaterial extends MaterialBase { + + constructor() { + + super( { + type: 'TemporalResolveMaterial', + uniforms: { + samplesTexture: { value: null }, + accumulatedSamplesTexture: { value: null }, + velocityTexture: { value: null }, + lastVelocityTexture: { value: null }, + depthTexture: { value: null }, + lastDepthTexture: { value: null }, + samples: { value: 0 }, + temporalResolveMix: { value: 0 }, + clampRadius: { value: 0 }, + newSamplesSmoothing: { value: 0 }, + newSamplesCorrection: { value: 0 }, + tileCount: { value: 0 }, + curInverseProjectionMatrix: { value: new Matrix4() }, + curCameraMatrixWorld: { value: new Matrix4() }, + prevInverseProjectionMatrix: { value: new Matrix4() }, + prevCameraMatrixWorld: { value: new Matrix4() }, + cameraNear: { value: 0 }, + cameraFar: { value: 0 }, + }, + defines: { + DILATION: '', + WEIGHT_TRANSFORM: '1.0' + }, + vertexShader, + fragmentShader, + } ); + + } + +} diff --git a/src/temporal-resolve/materials/VelocityShader.js b/src/temporal-resolve/materials/VelocityShader.js new file mode 100644 index 000000000..fdef3dfea --- /dev/null +++ b/src/temporal-resolve/materials/VelocityShader.js @@ -0,0 +1,115 @@ +// this shader is from: https://github.com/gkjohnson/threejs-sandbox + +import { ShaderChunk, Matrix4 } from 'three'; + +// Modified ShaderChunk.skinning_pars_vertex to handle +// a second set of bone information from the previous frame +export const prev_skinning_pars_vertex = /* glsl */ ` + #ifdef USE_SKINNING + #ifdef BONE_TEXTURE + uniform sampler2D prevBoneTexture; + mat4 getPrevBoneMatrix( const in float i ) { + + float j = i * 4.0; + float x = mod( j, float( boneTextureSize ) ); + float y = floor( j / float( boneTextureSize ) ); + float dx = 1.0 / float( boneTextureSize ); + float dy = 1.0 / float( boneTextureSize ); + y = dy * ( y + 0.5 ); + vec4 v1 = texture2D( prevBoneTexture, vec2( dx * ( x + 0.5 ), y ) ); + vec4 v2 = texture2D( prevBoneTexture, vec2( dx * ( x + 1.5 ), y ) ); + vec4 v3 = texture2D( prevBoneTexture, vec2( dx * ( x + 2.5 ), y ) ); + vec4 v4 = texture2D( prevBoneTexture, vec2( dx * ( x + 3.5 ), y ) ); + mat4 bone = mat4( v1, v2, v3, v4 ); + return bone; + + } + #else + uniform mat4 prevBoneMatrices[ MAX_BONES ]; + mat4 getPrevBoneMatrix( const in float i ) { + + mat4 bone = prevBoneMatrices[ int(i) ]; + return bone; + + } + #endif + #endif + `; + +// Returns the body of the vertex shader for the velocity buffer and +// outputs the position of the current and last frame positions +export const velocity_vertex = /* glsl */ ` + vec3 transformed; + + // Get the normal + ${ ShaderChunk.skinbase_vertex } + ${ ShaderChunk.beginnormal_vertex } + ${ ShaderChunk.skinnormal_vertex } + ${ ShaderChunk.defaultnormal_vertex } + + // Get the current vertex position + transformed = vec3( position ); + ${ ShaderChunk.skinning_vertex } + newPosition = velocityMatrix * vec4( transformed, 1.0 ); + + // Get the previous vertex position + transformed = vec3( position ); + ${ ShaderChunk.skinbase_vertex.replace( /mat4 /g, '' ).replace( /getBoneMatrix/g, 'getPrevBoneMatrix' ) } + ${ ShaderChunk.skinning_vertex.replace( /vec4 /g, '' ) } + prevPosition = prevVelocityMatrix * vec4( transformed, 1.0 ); + + gl_Position = newPosition; + + `; + +export const VelocityShader = { + uniforms: { + prevVelocityMatrix: { value: new Matrix4() }, + velocityMatrix: { value: new Matrix4() }, + prevBoneTexture: { value: null }, + interpolateGeometry: { value: 0 }, + intensity: { value: 1 }, + boneTexture: { value: null }, + + alphaTest: { value: 0.0 }, + map: { value: null }, + alphaMap: { value: null }, + opacity: { value: 1.0 } + }, + + vertexShader: /* glsl */ ` + #define MAX_BONES 1024 + + ${ShaderChunk.skinning_pars_vertex} + ${prev_skinning_pars_vertex} + + uniform mat4 velocityMatrix; + uniform mat4 prevVelocityMatrix; + uniform float interpolateGeometry; + varying vec4 prevPosition; + varying vec4 newPosition; + + void main() { + + ${ velocity_vertex } + + } + `, + + fragmentShader: /* glsl */ ` + uniform float intensity; + varying vec4 prevPosition; + varying vec4 newPosition; + + void main() { + + vec2 pos0 = ( prevPosition.xy / prevPosition.w ) * 0.5 + 0.5; + vec2 pos1 = ( newPosition.xy / newPosition.w ) * 0.5 + 0.5; + + vec2 vel = pos1 - pos0; + + gl_FragColor = vec4( vel, 1. - gl_FragCoord.z, 0.0 ); + + } + ` +}; diff --git a/src/temporal-resolve/materials/shaders/temporalResolveFragment.js b/src/temporal-resolve/materials/shaders/temporalResolveFragment.js new file mode 100644 index 000000000..25888240c --- /dev/null +++ b/src/temporal-resolve/materials/shaders/temporalResolveFragment.js @@ -0,0 +1,252 @@ +export const fragmentShader = /* glsl */` +uniform sampler2D samplesTexture; +uniform sampler2D accumulatedSamplesTexture; +uniform sampler2D velocityTexture; +uniform sampler2D depthTexture; +uniform sampler2D lastDepthTexture; + +uniform float samples; + +uniform float temporalResolveMix; +uniform float clampRadius; +uniform float newSamplesSmoothing; +uniform float newSamplesCorrection; +uniform float tileCount; + +uniform mat4 curInverseProjectionMatrix; +uniform mat4 curCameraMatrixWorld; + +uniform mat4 prevInverseProjectionMatrix; +uniform mat4 prevCameraMatrixWorld; + +uniform float cameraNear; +uniform float cameraFar; + +varying vec2 vUv; + +#include + +#define FLOAT_EPSILON 0.00001 +#define BLUR_EXPONENT 0.5 +#define MAX_NEIGHBOR_DEPTH_DIFFERENCE 0.001 + +// credits for transforming screen position to world position: https://discourse.threejs.org/t/reconstruct-world-position-in-screen-space-from-depth-buffer/5532/2 +vec3 screenSpaceToWorldSpace( const vec2 uv, const float depth, mat4 inverseProjectionMatrix, mat4 cameraMatrixWorld ) { + vec4 ndc = vec4( + ( uv.x - 0.5 ) * 2.0, + ( uv.y - 0.5 ) * 2.0, + ( depth - 0.5 ) * 2.0, + 1.0 + ); + + vec4 clip = inverseProjectionMatrix * ndc; + vec4 view = cameraMatrixWorld * ( clip / clip.w ); + + return view.xyz; +} + +#ifdef DILATION +// source: https://www.elopezr.com/temporal-aa-and-the-quest-for-the-holy-trail/ (modified to GLSL) +vec4 getDilatedTexture( sampler2D tex, vec2 uv, vec2 texSize ) { + + float closestDepth = 0.0; + vec2 closestUVOffset; + + for ( int x = - 1; x <= 1; ++ x ) { + + for ( int y = - 1; y <= 1; ++ y ) { + + vec2 uvOffset = vec2( x, y ) / texSize; + float neighborDepth = textureLod( tex, vUv + uvOffset, 0.0 ).b; + if ( neighborDepth > closestDepth ) { + + closestUVOffset = uvOffset; + closestDepth = neighborDepth; + + } + + } + + } + + return textureLod( tex, vUv + closestUVOffset, 0.0 ); + +} +#endif + +// idea from: https://www.elopezr.com/temporal-aa-and-the-quest-for-the-holy-trail/ +vec3 transformColor( vec3 color ) { + + return pow( color, vec3( WEIGHT_TRANSFORM ) ); + +} + +vec3 undoColorTransform( vec3 color ) { + + return max( vec3( 0.0 ), pow( color, vec3( 1.0 / WEIGHT_TRANSFORM ) ) ); + +} + +void main() { + + vec4 samplesTexel = texture2D( samplesTexture, vUv ); + samplesTexel.rgb = transformColor( samplesTexel.rgb ); + + ivec2 size = textureSize( samplesTexture, 0 ); + vec2 pxSize = vec2( size.x, size.y ); + +#ifdef DILATION + vec4 velocity = getDilatedTexture( velocityTexture, vUv, pxSize ); + + vec2 velUv = velocity.xy; + vec2 reprojectedUv = vUv - velUv; +#else + vec4 velocity = textureLod( velocityTexture, vUv, 0.0 ); + + vec2 velUv = velocity.xy; + vec2 reprojectedUv = vUv - velUv; +#endif + + bool isTileRendered = samplesTexel.a != 0.0; + + // background doesn't need reprojection + if( velocity.a == 1.0 ) { + + samplesTexel.rgb = undoColorTransform( samplesTexel.rgb ); + gl_FragColor = vec4( samplesTexel.rgb, isTileRendered ? 1.0 : 0.0 ); + + return; + + } + + // depth textures should not be dilated + float unpackedDepth = unpackRGBAToDepth( textureLod( depthTexture, vUv, 0.0 ) ); + float lastUnpackedDepth = unpackRGBAToDepth( textureLod( lastDepthTexture, reprojectedUv, 0.0 ) ); + + float movement = length( velUv ) * 100.0; + + vec3 curWorldPos = screenSpaceToWorldSpace( vUv, unpackedDepth, curInverseProjectionMatrix, curCameraMatrixWorld ); + vec3 lastWorldPos = screenSpaceToWorldSpace( vUv, lastUnpackedDepth, prevInverseProjectionMatrix, prevCameraMatrixWorld ); + float distToLastFrame = length( curWorldPos - lastWorldPos ) * 0.25; + + vec4 accumulatedSamplesTexel; + vec3 outputColor; + float alpha; + + bool canReproject = reprojectedUv.x >= 0.0 && reprojectedUv.x <= 1.0 && reprojectedUv.y >= 0.0 && reprojectedUv.y <= 1.0; + + // check that the reprojected UV is valid + if ( canReproject ) { + + accumulatedSamplesTexel = textureLod( accumulatedSamplesTexture, reprojectedUv, 0.0 ); + accumulatedSamplesTexel.rgb = transformColor( accumulatedSamplesTexel.rgb ); + + alpha = distToLastFrame < 0.25 ? ( accumulatedSamplesTexel.a + 0.05 ) : 0.0; + + // alpha = 0.0 is reserved for the background (to find out if the tile hasn't been rendered yet) + alpha = clamp( alpha, FLOAT_EPSILON, 1.0 ); + + if( samplesTexel.a != 0.0 ) { + + vec2 px = 1.0 / pxSize; + + vec3 boxBlurredColor; + float totalWeight; + + vec3 minNeighborColor = vec3( 1.0, 1.0, 1.0 ); + vec3 maxNeighborColor = vec3( 0.0, 0.0, 0.0 ); + + // use a small ring if there is a lot of movement otherwise there will be more smearing + float radius = movement > 1.0 ? 1.0 : clampRadius; + + vec3 col; + float weight; + vec2 neighborUv; + bool neighborUvValid; + float neighborUnpackedDepth; + + for( float x = - radius; x <= radius; x ++ ) { + + for( float y = - radius; y <= radius; y ++ ) { + + neighborUv = vUv + px * vec2( x, y ); + neighborUvValid = neighborUv.x >= 0.0 && neighborUv.x <= 1.0 && neighborUv.y >= 0.0 && neighborUv.y <= 1.0; + + if( neighborUvValid ) { + + col = textureLod( samplesTexture, neighborUv, 0.0 ).rgb; + col = transformColor( col ); + + neighborUnpackedDepth = unpackRGBAToDepth( textureLod( lastDepthTexture, neighborUv, 0.0 ) ); + + // box blur + if( abs( x ) <= 3.0 && abs( y ) <= 3.0 && abs( unpackedDepth - neighborUnpackedDepth ) < MAX_NEIGHBOR_DEPTH_DIFFERENCE ) { + + weight = 1.0 - abs( dot( col - samplesTexel.rgb, vec3( 0.25 ) ) ); + weight = pow( weight, BLUR_EXPONENT ); + boxBlurredColor += col * weight; + totalWeight += weight; + + } + + minNeighborColor = min( col, minNeighborColor ); + maxNeighborColor = max( col, maxNeighborColor ); + + } + + } + + } + + // clamp the reprojected frame (neighborhood clamping) + accumulatedSamplesTexel.rgb = mix( accumulatedSamplesTexel.rgb, clamp( accumulatedSamplesTexel.rgb, minNeighborColor, maxNeighborColor ), newSamplesCorrection ); + + // let's blur the input color to reduce noise for new samples + if ( newSamplesSmoothing != 0.0 && alpha < 1.0 && totalWeight > FLOAT_EPSILON ) { + + boxBlurredColor /= totalWeight; + samplesTexel.rgb = mix( samplesTexel.rgb, boxBlurredColor, newSamplesSmoothing ); + + } + + } + + } else { + + // reprojected UV coordinates are outside of screen, so just use the current frame for it + alpha = FLOAT_EPSILON; + accumulatedSamplesTexel.rgb = samplesTexel.rgb; + + } + + // in case this pixel is from a tile that wasn't rendered yet + if( !isTileRendered ) { + + gl_FragColor = vec4( undoColorTransform( accumulatedSamplesTexel.rgb ), 0.05); + + return; + + } + + float tileFactor = max( 0.5, 1.0 - ( tileCount - 1.0 ) * 0.025 ); + + float m = ( 1.0 - min( movement * 2.0, 1.0 ) * ( 1.0 - temporalResolveMix ) ) * tileFactor - ( samples - 1.0 ) * 0.0025 - 0.025; + + m = clamp( m, 0.0, 1.0 ); + + outputColor = accumulatedSamplesTexel.rgb * m + samplesTexel.rgb * ( 1.0 - m ); + + // alpha will be below 1 if the pixel is "new" (e.g. it became disoccluded recently) + // so make the final color blend more towards the new pixel + if( alpha < 1.0 ) { + + float correctionMix = min( movement, 0.5 ) * newSamplesCorrection; + outputColor = mix( outputColor, samplesTexel.rgb, correctionMix ); + + } + + outputColor = undoColorTransform( outputColor ); + + gl_FragColor = vec4( outputColor, alpha ); +} +`; diff --git a/src/temporal-resolve/materials/shaders/temporalResolveVertex.js b/src/temporal-resolve/materials/shaders/temporalResolveVertex.js new file mode 100644 index 000000000..d30508647 --- /dev/null +++ b/src/temporal-resolve/materials/shaders/temporalResolveVertex.js @@ -0,0 +1,10 @@ +export const vertexShader = /* glsl */ ` +varying vec2 vUv; + +void main() { + + vUv = position.xy * 0.5 + 0.5; + gl_Position = vec4( position.xy, 1.0, 1.0 ); + +} +`; diff --git a/src/temporal-resolve/passes/TemporalResolvePass.js b/src/temporal-resolve/passes/TemporalResolvePass.js new file mode 100644 index 000000000..a5b67e589 --- /dev/null +++ b/src/temporal-resolve/passes/TemporalResolvePass.js @@ -0,0 +1,151 @@ +import { + Color, + FloatType, + FramebufferTexture, LinearFilter, + MeshDepthMaterial, + NearestFilter, + RGBADepthPacking, + RGBAFormat, + Vector2, + WebGLRenderTarget +} from 'three'; +import { FullScreenQuad } from 'three/examples/jsm/postprocessing/Pass.js'; +import { TemporalResolveMaterial } from '../materials/TemporalResolveMaterial.js'; +import { VelocityPass } from './VelocityPass.js'; + +const zeroVec2 = new Vector2(); +const meshDepthMaterial = new MeshDepthMaterial( { + depthPacking: RGBADepthPacking, +} ); +const blackColor = new Color( 0 ); + +export class TemporalResolvePass { + + constructor( ptRenderer, scene, camera ) { + + this.ptRenderer = ptRenderer; + this.scene = scene; + this.camera = camera; + + this.renderTarget = new WebGLRenderTarget( 1, 1, { + minFilter: LinearFilter, + magFilter: LinearFilter, + type: FloatType, + depthBuffer: false, + } ); + + this.depthRenderTarget = new WebGLRenderTarget( 1, 1, { + minFilter: NearestFilter, + magFilter: NearestFilter, + } ); + + this.velocityPass = new VelocityPass( scene, camera ); + + this.fullscreenMaterial = new TemporalResolveMaterial(); + + this.fullscreenMaterial.velocityTexture = this.velocityPass.renderTarget.texture; + this.fullscreenMaterial.depthTexture = this.depthRenderTarget.texture; + + this.fsQuad = new FullScreenQuad( null ); + this.fsQuad.material = this.fullscreenMaterial; + + this.setSize( window.innerWidth, window.innerHeight ); + + } + + dispose() { + + this.renderTarget.dispose(); + this.accumulatedSamplesTexture.dispose(); + this.lastDepthTexture.dispose(); + this.fullscreenMaterial.dispose(); + + } + + setSize( width, height ) { + + this.renderTarget.setSize( width, height ); + this.depthRenderTarget.setSize( width, height ); + this.velocityPass.setSize( width, height ); + + this.createFramebuffers( width, height ); + + } + + createFramebuffers( width, height ) { + + if ( this.accumulatedSamplesTexture ) + this.accumulatedSamplesTexture.dispose(); + if ( this.lastDepthTexture ) this.lastDepthTexture.dispose(); + + this.accumulatedSamplesTexture = new FramebufferTexture( + width, + height, + RGBAFormat + ); + this.accumulatedSamplesTexture.minFilter = LinearFilter; + this.accumulatedSamplesTexture.magFilter = LinearFilter; + this.accumulatedSamplesTexture.type = FloatType; + + this.lastDepthTexture = new FramebufferTexture( width, height, RGBAFormat ); + this.lastDepthTexture.minFilter = NearestFilter; + this.lastDepthTexture.magFilter = NearestFilter; + + this.fullscreenMaterial.accumulatedSamplesTexture = this.accumulatedSamplesTexture; + this.fullscreenMaterial.lastDepthTexture = this.lastDepthTexture; + + this.fullscreenMaterial.needsUpdate = true; + + } + + render( renderer ) { + + // render depth + this.scene.overrideMaterial = meshDepthMaterial; + renderer.setRenderTarget( this.depthRenderTarget ); + renderer.clear(); + + const { background } = this.scene; + this.scene.background = blackColor; + + renderer.render( this.scene, this.camera ); + + this.scene.background = background; + this.scene.overrideMaterial = null; + + // render velocity + this.velocityPass.render( renderer ); + + // update uniforms of this pass + this.fullscreenMaterial.tileCount = this.ptRenderer.tiles.x * this.ptRenderer.tiles.y; + this.fullscreenMaterial.curInverseProjectionMatrix.copy( + this.camera.projectionMatrixInverse + ); + this.fullscreenMaterial.curCameraMatrixWorld.copy( + this.camera.matrixWorld + ); + this.fullscreenMaterial.cameraNear = this.camera.near; + this.fullscreenMaterial.cameraFar = this.camera.far; + + // now render this fullscreen pass + renderer.setRenderTarget( this.renderTarget ); + this.fsQuad.render( renderer ); + + // save all buffers for use in the next frame + renderer.copyFramebufferToTexture( zeroVec2, this.accumulatedSamplesTexture ); + + renderer.setRenderTarget( this.depthRenderTarget ); + renderer.copyFramebufferToTexture( zeroVec2, this.lastDepthTexture ); + + this.fullscreenMaterial.prevInverseProjectionMatrix.copy( + this.camera.projectionMatrixInverse + ); + this.fullscreenMaterial.prevCameraMatrixWorld.copy( + this.camera.matrixWorld + ); + + renderer.setRenderTarget( null ); + + } + +} diff --git a/src/temporal-resolve/passes/VelocityPass.js b/src/temporal-resolve/passes/VelocityPass.js new file mode 100644 index 000000000..c798a1946 --- /dev/null +++ b/src/temporal-resolve/passes/VelocityPass.js @@ -0,0 +1,163 @@ +import { + Color, + DataTexture, + FloatType, + HalfFloatType, + LinearFilter, + RGBAFormat, + ShaderMaterial, + UniformsUtils, + WebGLRenderTarget +} from 'three'; +import { VelocityShader } from '../materials/VelocityShader.js'; + +const backgroundColor = new Color( 0 ); +const updateProperties = [ 'visible', 'wireframe', 'side' ]; + +export class VelocityPass { + + constructor( scene, camera ) { + + this.scene = scene; + this.camera = camera; + + this.cachedMaterials = new WeakMap(); + + this.renderTarget = new WebGLRenderTarget( + typeof window !== 'undefined' ? window.innerWidth : 2000, + typeof window !== 'undefined' ? window.innerHeight : 1000, + { + minFilter: LinearFilter, + magFilter: LinearFilter, + type: HalfFloatType, + } + ); + + } + + setVelocityMaterialInScene() { + + this.scene.traverse( c => { + + if ( c.material ) { + + const originalMaterial = c.material; + + // eslint-disable-next-line prefer-const + let [ cachedOriginalMaterial, velocityMaterial ] = this.cachedMaterials.get( c ) || []; + + if ( originalMaterial !== cachedOriginalMaterial ) { + + velocityMaterial = new ShaderMaterial( { + uniforms: UniformsUtils.clone( VelocityShader.uniforms ), + vertexShader: VelocityShader.vertexShader, + fragmentShader: VelocityShader.fragmentShader + } ); + + c.material = velocityMaterial; + + if ( c.skeleton && c.skeleton.boneTexture ) this.saveBoneTexture( c ); + + this.cachedMaterials.set( c, [ originalMaterial, velocityMaterial ] ); + + } + + velocityMaterial.uniforms.velocityMatrix.value.multiplyMatrices( + this.camera.projectionMatrix, + c.modelViewMatrix + ); + + for ( const prop of updateProperties ) velocityMaterial[ prop ] = originalMaterial[ prop ]; + + if ( c.skeleton ) { + + velocityMaterial.defines.USE_SKINNING = ''; + velocityMaterial.defines.BONE_TEXTURE = ''; + + velocityMaterial.uniforms.boneTexture.value = c.skeleton.boneTexture; + + } + + c.material = velocityMaterial; + + } + + } ); + + } + + saveBoneTexture( object ) { + + let boneTexture = object.material.uniforms.prevBoneTexture.value; + + if ( boneTexture && boneTexture.image.width === object.skeleton.boneTexture.width ) { + + boneTexture = object.material.uniforms.prevBoneTexture.value; + boneTexture.image.data.set( object.skeleton.boneTexture.image.data ); + + } else { + + if ( boneTexture ) boneTexture.dispose(); + + const boneMatrices = object.skeleton.boneTexture.image.data.slice(); + const size = object.skeleton.boneTexture.image.width; + + boneTexture = new DataTexture( boneMatrices, size, size, RGBAFormat, FloatType ); + object.material.uniforms.prevBoneTexture.value = boneTexture; + + boneTexture.needsUpdate = true; + + } + + } + + unsetVelocityMaterialInScene() { + + this.scene.traverse( c => { + + if ( c.material ) { + + c.material.uniforms.prevVelocityMatrix.value.multiplyMatrices( this.camera.projectionMatrix, c.modelViewMatrix ); + + if ( c.skeleton && c.skeleton.boneTexture ) this.saveBoneTexture( c ); + + const [ originalMaterial ] = this.cachedMaterials.get( c ); + + c.material = originalMaterial; + + } + + } ); + + } + + dispose() { + + this.renderTarget.dispose(); + + } + + setSize( width, height ) { + + this.renderTarget.setSize( width, height ); + + } + + render( renderer ) { + + this.setVelocityMaterialInScene(); + + renderer.setRenderTarget( this.renderTarget ); + renderer.clear(); + const { background } = this.scene; + this.scene.background = backgroundColor; + + renderer.render( this.scene, this.camera ); + + this.scene.background = background; + + this.unsetVelocityMaterialInScene(); + + } + +}