diff --git a/examples/hello/hello.fs b/examples/hello/hello.fs index 771b913..1571407 100644 --- a/examples/hello/hello.fs +++ b/examples/hello/hello.fs @@ -90,21 +90,22 @@ let init (_args: array) = // let barrelModel = Model.file("ExplodingBarrel.glb"); // let renderModel = (Graphics.Scene3D.model barrelModel) |> Transform.scale 1f; // let renderModel = Model.file ("ExplodingBarrel.glb") |> Graphics.Scene3D.model |> Transform.scale 0.5f; - let modify = Model.modify (MeshSelector.all ()) (MeshOverride.material (textureMaterial)); - let renderModel = Model.file ("vr_glove_model2.glb") |> modify |> Graphics.Scene3D.model |> Transform.scale 5f; + // let modify = Model.modify (MeshSelector.all ()) (MeshOverride.material (textureMaterial)); + // let renderModel = Model.file ("vr_glove_model2.glb") |> modify |> Graphics.Scene3D.model |> Transform.scale 5f; - // let renderModel = - // "shark.glb" - // |> Model.file - // |> Graphics.Scene3D.model - // |> Transform.scale 0.001f; + let renderModel = + "shark.glb" + |> Model.file + |> Graphics.Scene3D.model + |> Transform.translateZ 10.0f + |> Transform.scale 0.004f; group([| material (textureMaterial, [| cylinder() |> Transform.translateY -1.0f; renderModel - |> Transform.rotateY (Math.Angle.degrees (frameTime.tts * 20.0f)) - |> Transform.rotateZ (Math.Angle.degrees (frameTime.tts * 40.0f)) + |> Transform.rotateY (Math.Angle.degrees (180.0f + 10.0f * frameTime.tts * 0.5f)) + |> Transform.rotateX (Math.Angle.degrees (0.0f * sin frameTime.tts * 2.0f)) |]) |> Transform.translateZ ((sin (frameTime.tts * 5.0f)) * 1.0f) |]) diff --git a/runtime/functor-runtime-common/src/animation.rs b/runtime/functor-runtime-common/src/animation.rs new file mode 100644 index 0000000..795d3a7 --- /dev/null +++ b/runtime/functor-runtime-common/src/animation.rs @@ -0,0 +1,36 @@ +use cgmath::{Quaternion, Vector3}; + +pub struct Animation { + pub name: String, + pub channels: Vec, + pub duration: f32, +} + +pub enum AnimationProperty { + Translation, + Rotation, + Scale, + // TODO: Morph target + Weights, +} + +pub struct AnimationChannel { + pub target_node_index: usize, + pub target_property: AnimationProperty, + pub keyframes: Vec, + // interpolation: TODO? +} + +#[derive(Clone)] +pub struct Keyframe { + pub time: f32, + pub value: AnimationValue, +} + +#[derive(Clone)] +pub enum AnimationValue { + Translation(Vector3), + Rotation(Quaternion), + Scale(Vector3), + Weights(Vec), +} diff --git a/runtime/functor-runtime-common/src/asset/pipelines/model_pipeline.rs b/runtime/functor-runtime-common/src/asset/pipelines/model_pipeline.rs index fb74730..2dd7b17 100644 --- a/runtime/functor-runtime-common/src/asset/pipelines/model_pipeline.rs +++ b/runtime/functor-runtime-common/src/asset/pipelines/model_pipeline.rs @@ -1,10 +1,12 @@ use std::io::Cursor; use cgmath::num_traits::ToPrimitive; -use cgmath::{vec2, vec3, Matrix4}; +use cgmath::{vec2, vec3, Matrix4, Quaternion}; use gltf::{buffer::Source as BufferSource, image::Source as ImageSource}; +use crate::animation::{Animation, AnimationChannel, AnimationProperty, AnimationValue, Keyframe}; use crate::model::{Model, ModelMesh, Skeleton, SkeletonBuilder}; + use crate::render::VertexPositionTexture; use crate::{ asset::{AssetCache, AssetPipeline}, @@ -84,17 +86,22 @@ impl AssetPipeline for ModelPipeline { } } - process_animations(&document, &buffers_data); + let animations = process_animations(&document, &buffers_data); let skeleton = maybe_skeleton.unwrap_or(Skeleton::empty()); - Model { meshes, skeleton } + Model { + meshes, + skeleton, + animations, + } } fn unloaded_asset(&self, _context: crate::asset::AssetPipelineContext) -> Model { Model { meshes: vec![], skeleton: Skeleton::empty(), + animations: vec![], } } } @@ -238,46 +245,108 @@ fn process_joints( } } -fn process_animations(document: &gltf::Document, buffers: &[gltf::buffer::Data]) { +fn process_animations(document: &gltf::Document, buffers: &[gltf::buffer::Data]) -> Vec { + let mut animations = Vec::new(); // Load animations // From: https://whoisryosuke.com/blog/2022/importing-gltf-with-wgpu-and-rust for animation in document.animations() { - // println!("!! Animation: {:?}", animation.name()); + let animation_name = animation.name().unwrap_or("Unnamed Animation").to_owned(); + let mut channels = Vec::new(); + let mut max_time = 0.0; + for channel in animation.channels() { + // TODO: Proper interpolation + //let sampler = channel.sampler(); + let target = channel.target(); + let node_index = target.node().index(); + let property = match target.property() { + gltf::animation::Property::Translation => AnimationProperty::Translation, + gltf::animation::Property::Rotation => AnimationProperty::Rotation, + gltf::animation::Property::Scale => AnimationProperty::Scale, + gltf::animation::Property::MorphTargetWeights => AnimationProperty::Weights, + }; + let reader = channel.reader(|buffer| Some(&buffers[buffer.index()])); - // TODO: - let _keyframe_timestamps = if let Some(inputs) = reader.read_inputs() { - match inputs { - gltf::accessor::Iter::Standard(times) => { - let _times: Vec = times.collect(); - // println!("Time: {}", times.len()); - // dbg!(times); + + let input_times: Vec = reader + .read_inputs() + .expect("Failed to read animation input") + .collect(); + + let output_values = reader + .read_outputs() + .expect("Failed to read animation output"); + + let mut keyframes = Vec::new(); + max_time = input_times + .iter() + .cloned() + .fold(max_time, |a: f32, b: f32| a.max(b)); + + match output_values { + gltf::animation::util::ReadOutputs::Translations(translations) => { + for (i, translation) in translations.enumerate() { + let time = input_times[i]; + keyframes.push(Keyframe { + time, + value: AnimationValue::Translation(vec3( + translation[0], + translation[1], + translation[2], + )), + }); } - gltf::accessor::Iter::Sparse(_) => { - println!("Sparse keyframes not supported"); + } + gltf::animation::util::ReadOutputs::Rotations(rotations) => { + for (i, rotation) in rotations.into_f32().enumerate() { + let time = input_times[i]; + keyframes.push(Keyframe { + time, + // TODO: Does w come first or last? + value: AnimationValue::Rotation(Quaternion { + v: vec3(rotation[0], rotation[1], rotation[2]), + s: rotation[3], + }), + }); } } - }; - - // TODO: - let mut keyframes_vec: Vec> = Vec::new(); - let _keyframes = if let Some(outputs) = reader.read_outputs() { - match outputs { - gltf::animation::util::ReadOutputs::Translations(translation) => { - translation.for_each(|tr| { - // println!("Translation:"); - // dbg!(tr); - let vector: Vec = tr.into(); - keyframes_vec.push(vector); + gltf::animation::util::ReadOutputs::Scales(scales) => { + for (i, scale) in scales.enumerate() { + let time = input_times[i]; + keyframes.push(Keyframe { + time, + value: AnimationValue::Scale(vec3(scale[0], scale[1], scale[2])), }); } - _other => (), // gltf::animation::util::ReadOutputs::Rotations(_) => todo!(), - // gltf::animation::util::ReadOutputs::Scales(_) => todo!(), - // gltf::animation::util::ReadOutputs::MorphTargetWeights(_) => todo!(), } - }; + gltf::animation::util::ReadOutputs::MorphTargetWeights(_weights) => { + // TODO: + println!("WARN: ignoring morph target weights; not implemented"); + // for (i, weight) in weights.enumerate() { + // let time = input_times[i]; + // keyframes.push(Keyframe { + // time, + // value: AnimationValue::Weights(weight.to_vec()), + // }); + // } + } + } - // println!("Keyframes: {}", keyframes_vec.len()); + channels.push(AnimationChannel { + target_node_index: node_index, + target_property: property, + keyframes, + // TODO: + //interpolation, + }); } + + animations.push(Animation { + name: animation_name, + channels, + duration: max_time, + }); } + // panic!("animations"); + animations } diff --git a/runtime/functor-runtime-common/src/lib.rs b/runtime/functor-runtime-common/src/lib.rs index b2200f0..b2876c5 100644 --- a/runtime/functor-runtime-common/src/lib.rs +++ b/runtime/functor-runtime-common/src/lib.rs @@ -48,6 +48,7 @@ impl OpaqueState { } } +pub mod animation; pub mod asset; mod frame_time; pub mod geometry; diff --git a/runtime/functor-runtime-common/src/model/mod.rs b/runtime/functor-runtime-common/src/model/mod.rs index 09df775..5f58268 100644 --- a/runtime/functor-runtime-common/src/model/mod.rs +++ b/runtime/functor-runtime-common/src/model/mod.rs @@ -1,6 +1,8 @@ mod skeleton; -use crate::{geometry::IndexedMesh, render::VertexPositionTexture, texture::Texture2D}; +use crate::{ + animation::Animation, geometry::IndexedMesh, render::VertexPositionTexture, texture::Texture2D, +}; use cgmath::Matrix4; pub use skeleton::*; @@ -18,4 +20,6 @@ pub struct Model { pub meshes: Vec, pub skeleton: Skeleton, + + pub animations: Vec, } diff --git a/runtime/functor-runtime-common/src/model/skeleton.rs b/runtime/functor-runtime-common/src/model/skeleton.rs index a5fa51b..51f6985 100644 --- a/runtime/functor-runtime-common/src/model/skeleton.rs +++ b/runtime/functor-runtime-common/src/model/skeleton.rs @@ -1,15 +1,17 @@ use std::collections::{HashMap, HashSet}; -use cgmath::{Matrix4, SquareMatrix}; +use cgmath::{InnerSpace, Matrix3, Matrix4, Quaternion, SquareMatrix, Vector3, VectorSpace}; -#[derive(Debug)] +use crate::animation::{Animation, AnimationValue, Keyframe}; + +#[derive(Clone, Debug)] pub struct Joint { pub name: String, pub transform: Matrix4, pub parent: Option, } -#[derive(Debug)] +#[derive(Clone, Debug)] pub struct Skeleton { num_joints: i32, @@ -27,6 +29,88 @@ impl Skeleton { } } + pub fn animate(skeleton: &Skeleton, animation: &Animation, time: f32) -> Skeleton { + // Start by cloning the existing skeleton + let mut new_skeleton = skeleton.clone(); + + // For each joint, store the base T, R, S, and the animated T, R, S + let mut joint_base_trs = HashMap::new(); + let mut joint_animated_trs = HashMap::new(); + + // First, for each joint, extract base T, R, S + for (&joint_index, joint) in &skeleton.joint_info { + // Extract translation + let base_t = joint.transform.w.truncate(); // last column + + // Extract the upper-left 3x3 matrix + let m = Matrix3::from_cols( + joint.transform.x.truncate(), + joint.transform.y.truncate(), + joint.transform.z.truncate(), + ); + + // Extract scale factors + let scale_x = m.x.magnitude(); + let scale_y = m.y.magnitude(); + let scale_z = m.z.magnitude(); + let base_s = Vector3::new(scale_x, scale_y, scale_z); + + // Normalize the columns to get the rotation matrix + let rotation_matrix = Matrix3::from_cols(m.x / scale_x, m.y / scale_y, m.z / scale_z); + + // Convert rotation matrix to Quaternion + let base_r = Quaternion::from(rotation_matrix); + + joint_base_trs.insert(joint_index, (base_t, base_r, base_s)); + // Initialize animated T, R, S to base T, R, S + joint_animated_trs.insert(joint_index, (base_t, base_r, base_s)); + } + + // For each animation channel + for channel in &animation.channels { + let target_node_index = channel.target_node_index as i32; + // Get the animated value at time t + if let Some(value) = interpolate_keyframes(&channel.keyframes, time) { + // Update the animated T, R, S for the joint + if let Some((animated_t, animated_r, animated_s)) = + joint_animated_trs.get_mut(&target_node_index) + { + match value { + AnimationValue::Translation(translation) => { + *animated_t = translation; + } + AnimationValue::Rotation(rotation) => { + *animated_r = rotation; + } + AnimationValue::Scale(scale) => { + *animated_s = scale; + } + _ => {} + } + } + } + } + + // For each joint, reconstruct the animated transform + for (&joint_index, joint) in new_skeleton.joint_info.iter_mut() { + if let Some((animated_t, animated_r, animated_s)) = joint_animated_trs.get(&joint_index) + { + // Construct the transform from animated_T, animated_R, animated_S + let transform = Matrix4::from_translation(*animated_t) + * Matrix4::from(*animated_r) + * Matrix4::from_nonuniform_scale(animated_s.x, animated_s.y, animated_s.z); + joint.transform = transform; + } + } + + // Recompute absolute transforms + new_skeleton.joint_absolute_transform = + compute_absolute_transforms(&new_skeleton.joint_info); + + // Return the new skeleton + new_skeleton + } + pub fn get_joint_count(&self) -> i32 { self.num_joints } @@ -163,6 +247,50 @@ fn compute_joint_absolute_transform( abs_transform } +fn interpolate_keyframes(keyframes: &Vec, time: f32) -> Option { + if keyframes.is_empty() { + return None; + } + // If time is before the first keyframe, return the first value + if time <= keyframes[0].time { + return Some(keyframes[0].value.clone()); + } + // If time is after the last keyframe, return the last value + if time >= keyframes[keyframes.len() - 1].time { + return Some(keyframes[keyframes.len() - 1].value.clone()); + } + // Find the keyframes surrounding the given time + for i in 0..keyframes.len() - 1 { + let kf0 = &keyframes[i]; + let kf1 = &keyframes[i + 1]; + if time >= kf0.time && time <= kf1.time { + let t = (time - kf0.time) / (kf1.time - kf0.time); + return Some(interpolate_values(&kf0.value, &kf1.value, t)); + } + } + None +} +fn interpolate_values(v0: &AnimationValue, v1: &AnimationValue, t: f32) -> AnimationValue { + match (v0, v1) { + (AnimationValue::Translation(tr0), AnimationValue::Translation(tr1)) => { + let value = tr0.lerp(*tr1, t); + AnimationValue::Translation(value) + } + (AnimationValue::Rotation(r0), AnimationValue::Rotation(r1)) => { + let value = r0.slerp(*r1, t); + AnimationValue::Rotation(value) + } + (AnimationValue::Scale(s0), AnimationValue::Scale(s1)) => { + let value = s0.lerp(*s1, t); + AnimationValue::Scale(value) + } + _ => { + // Unsupported interpolation + v0.clone() + } + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/runtime/functor-runtime-common/src/render_context.rs b/runtime/functor-runtime-common/src/render_context.rs index 49c7f90..a9afcd1 100644 --- a/runtime/functor-runtime-common/src/render_context.rs +++ b/runtime/functor-runtime-common/src/render_context.rs @@ -1,9 +1,10 @@ use std::sync::Arc; -use crate::asset::AssetCache; +use crate::{asset::AssetCache, FrameTime}; pub struct RenderContext<'a> { pub gl: &'a glow::Context, pub shader_version: &'a str, pub asset_cache: Arc, + pub frame_time: FrameTime, } diff --git a/runtime/functor-runtime-common/src/scene3d/mod.rs b/runtime/functor-runtime-common/src/scene3d/mod.rs index a44e6c2..cda779f 100644 --- a/runtime/functor-runtime-common/src/scene3d/mod.rs +++ b/runtime/functor-runtime-common/src/scene3d/mod.rs @@ -14,7 +14,7 @@ use crate::{ geometry::{self, Geometry, Mesh}, material::{BasicMaterial, ColorMaterial, Material}, math::Angle, - model::Model, + model::{Model, Skeleton}, texture::{RuntimeTexture, Texture2D}, RenderContext, }; @@ -226,30 +226,44 @@ impl Scene3D { ); }; + // TODO: Bring back drawing mesh.mesh.draw(&render_context.gl) } // TEMPORARY: Render joints - let joints = hydrated_model.skeleton.get_transforms(); - for joint_transform in joints { - let mut color_material = - ColorMaterial::create(vec4(0.0, 1.0, 0.0, 1.0)); - color_material.initialize(&render_context); - - let xform = &(matrix * joint_transform); - let point = xform.transform_point(point3(0.0, 0.0, 0.0)); - let xform2 = Matrix4::from_translation(vec3(point.x, point.y, point.z)) - * Matrix4::from_scale(0.1); - - color_material.draw_opaque( - &render_context, - projection_matrix, - view_matrix, - &xform2, - &[], + let maybe_animation = hydrated_model.animations.get(0); + if let Some(animation) = maybe_animation { + let time = render_context.frame_time.tts % animation.duration; + let animated_skeleton = + Skeleton::animate(&hydrated_model.skeleton, animation, time); + + println!( + "Animating {} {} {}", + animation.name, animation.duration, time, ); - scene_context.sphere.borrow().draw(render_context.gl); + let joints = animated_skeleton.get_transforms(); + for joint_transform in joints { + let mut color_material = + ColorMaterial::create(vec4(0.0, 1.0, 0.0, 1.0)); + color_material.initialize(&render_context); + + let xform = &(matrix * joint_transform); + let point = xform.transform_point(point3(0.0, 0.0, 0.0)); + let xform2 = + Matrix4::from_translation(vec3(point.x, point.y, point.z)) + * Matrix4::from_scale(0.1); + + color_material.draw_opaque( + &render_context, + projection_matrix, + view_matrix, + &xform2, + &[], + ); + + scene_context.sphere.borrow().draw(render_context.gl); + } } } } diff --git a/runtime/functor-runtime-desktop/src/main.rs b/runtime/functor-runtime-desktop/src/main.rs index 3773d8a..e34b610 100644 --- a/runtime/functor-runtime-desktop/src/main.rs +++ b/runtime/functor-runtime-desktop/src/main.rs @@ -150,6 +150,7 @@ pub async fn main() { gl: &gl, shader_version, asset_cache: asset_cache.clone(), + frame_time: time.clone(), }; let scene = game.render(time.clone()); diff --git a/runtime/functor-runtime-web/src/lib.rs b/runtime/functor-runtime-web/src/lib.rs index 42f2592..7b9a0e3 100644 --- a/runtime/functor-runtime-web/src/lib.rs +++ b/runtime/functor-runtime-web/src/lib.rs @@ -176,10 +176,18 @@ async fn run_async() -> Result<(), JsValue> { let scene_context = SceneContext::new(); *g.borrow_mut() = Some(Closure::new(move || { + let now = performance.now() as f32; + let frame_time = FrameTime { + dts: (now - last_time) / 1000.0, + tts: (now - initial_time) / 1000.0, + }; + + last_time = now; let render_ctx = RenderContext { gl: &gl, shader_version, asset_cache: asset_cache.clone(), + frame_time: frame_time.clone(), }; let projection_matrix: Matrix4 = @@ -198,14 +206,7 @@ async fn run_async() -> Result<(), JsValue> { // let scene = Scene3D::cube(); - let now = performance.now() as f32; - let frameTime = FrameTime { - dts: (now - last_time) / 1000.0, - tts: (now - initial_time) / 1000.0, - }; - last_time = now; - - let val = game_render(functor_runtime_common::to_js_value(&frameTime)); + let val = game_render(functor_runtime_common::to_js_value(&frame_time)); web_sys::console::log_2(&JsValue::from_str("calling render"), &val); let scene: Scene3D = functor_runtime_common::from_js_value(val);