Read + render animation data

examples/hello/hello.fs

@@ -90,21 +90,22 @@ let init (_args: array<string>) =
         // let barrelModel = Model.file("ExplodingBarrel.glb");
         // let renderModel = (Graphics.Scene3D.model barrelModel) |> Transform.scale 1f;
         // let renderModel = Model.file ("ExplodingBarrel.glb") |> Graphics.Scene3D.model |> Transform.scale 0.5f;
-        let modify = Model.modify (MeshSelector.all ()) (MeshOverride.material (textureMaterial));
-        let renderModel = Model.file ("vr_glove_model2.glb") |> modify |> Graphics.Scene3D.model |> Transform.scale 5f;
+        // let modify = Model.modify (MeshSelector.all ()) (MeshOverride.material (textureMaterial));
+        // let renderModel = Model.file ("vr_glove_model2.glb") |> modify |> Graphics.Scene3D.model |> Transform.scale 5f;
 
-        // let renderModel = 
-        //     "shark.glb"
-        //     |> Model.file 
-        //     |> Graphics.Scene3D.model 
-        //     |> Transform.scale 0.001f;
+        let renderModel = 
+            "shark.glb"
+            |> Model.file 
+            |> Graphics.Scene3D.model 
+            |> Transform.translateZ 10.0f
+            |> Transform.scale 0.004f;
 
         group([|
             material (textureMaterial, [|
                 cylinder() |> Transform.translateY -1.0f;
                 renderModel
-                |> Transform.rotateY (Math.Angle.degrees (frameTime.tts * 20.0f))
-                |> Transform.rotateZ (Math.Angle.degrees (frameTime.tts * 40.0f))
+                |> Transform.rotateY (Math.Angle.degrees (180.0f + 10.0f * frameTime.tts * 0.5f))
+                |> Transform.rotateX (Math.Angle.degrees (0.0f * sin frameTime.tts * 2.0f))
             |])
             |> Transform.translateZ ((sin (frameTime.tts * 5.0f)) * 1.0f)
         |])

runtime/functor-runtime-common/src/animation.rs

@@ -0,0 +1,36 @@
+use cgmath::{Quaternion, Vector3};
+
+pub struct Animation {
+    pub name: String,
+    pub channels: Vec<AnimationChannel>,
+    pub duration: f32,
+}
+
+pub enum AnimationProperty {
+    Translation,
+    Rotation,
+    Scale,
+    // TODO: Morph target
+    Weights,
+}
+
+pub struct AnimationChannel {
+    pub target_node_index: usize,
+    pub target_property: AnimationProperty,
+    pub keyframes: Vec<Keyframe>,
+    // interpolation: TODO?
+}
+
+#[derive(Clone)]
+pub struct Keyframe {
+    pub time: f32,
+    pub value: AnimationValue,
+}
+
+#[derive(Clone)]
+pub enum AnimationValue {
+    Translation(Vector3<f32>),
+    Rotation(Quaternion<f32>),
+    Scale(Vector3<f32>),
+    Weights(Vec<f32>),
+}

runtime/functor-runtime-common/src/asset/pipelines/model_pipeline.rs

@@ -1,9 +1,11 @@
 use std::io::Cursor;
 
-use cgmath::{vec2, vec3, Matrix4};
+use cgmath::num_traits::ToPrimitive;
+use cgmath::{vec2, vec3, Matrix4, Quaternion};
 use gltf::{buffer::Source as BufferSource, image::Source as ImageSource};
 
-use crate::model::{Model, ModelMesh};
+use crate::animation::{Animation, AnimationChannel, AnimationProperty, AnimationValue, Keyframe};
+use crate::model::{Model, ModelMesh, Skeleton, SkeletonBuilder};
 use crate::{
     asset::{AssetCache, AssetPipeline},
     geometry::IndexedMesh,
@@ -67,19 +69,39 @@ impl AssetPipeline<Model> for ModelPipeline {
 
         let mut meshes = Vec::new();
 
+        let mut maybe_skeleton: Option<Skeleton> = None;
+
         for scene in document.scenes() {
             println!("Scene {}", scene.index());
             for node in scene.nodes() {
                 println!("- Node: {:?}", node.name());
-                process_node(&node, &buffers_data, &images_data, &mut meshes);
+                process_node(
+                    &node,
+                    &buffers_data,
+                    &images_data,
+                    &mut meshes,
+                    &mut maybe_skeleton,
+                );
             }
         }
 
-        Model { meshes }
+        let animations = process_animations(&document, &buffers_data);
+
+        let skeleton = maybe_skeleton.unwrap_or(Skeleton::empty());
+
+        Model {
+            meshes,
+            skeleton,
+            animations,
+        }
     }
 
     fn unloaded_asset(&self, _context: crate::asset::AssetPipelineContext) -> Model {
-        Model { meshes: vec![] }
+        Model {
+            meshes: vec![],
+            skeleton: Skeleton::empty(),
+            animations: vec![],
+        }
     }
 }
 
@@ -88,6 +110,7 @@ fn process_node(
     buffers: &[gltf::buffer::Data],
     images: &[TextureData],
     meshes: &mut Vec<ModelMesh>,
+    maybe_skeleton: &mut Option<Skeleton>,
 ) {
     if let Some(mesh) = node.mesh() {
         let transform_array = node.transform().matrix();
@@ -115,6 +138,16 @@ fn process_node(
 
             let scale = 1.0;
 
+            let joints = reader
+                .read_joints(0)
+                .map(|v| v.into_u16().collect::<Vec<_>>())
+                .unwrap_or_default();
+
+            let weights = reader
+                .read_weights(0)
+                .map(|v| v.into_f32().collect::<Vec<_>>())
+                .unwrap_or_default();
+
             let vertices: Vec<VertexPositionTexture> = positions
                 .iter()
                 .zip(tex_coords.into_iter())
@@ -124,10 +157,12 @@ fn process_node(
                 })
                 .collect();
             println!(
-                "-- Mesh: {:?} vertices: {} indices: {}",
+                "-- Mesh: {:?} vertices: {} indices: {} joints: {} weights: {}",
                 mesh.name(),
                 vertices.len(),
-                indices.len()
+                indices.len(),
+                joints.len(),
+                weights.len(),
             );
 
             // Parse material
@@ -167,7 +202,152 @@ fn process_node(
             meshes.push(model_mesh);
         }
     }
+
+    // Process skinning data
+    if let Some(skin) = node.skin() {
+        let reader = skin.reader(|buffer| Some(&buffers[buffer.index()]));
+
+        // TODO: Save inverse bind matrices with model
+        let _inverse_bind_matrices = reader
+            .read_inverse_bind_matrices()
+            .map(|v| v.collect::<Vec<_>>())
+            .unwrap_or_default();
+
+        let mut skeleton_builder = SkeletonBuilder::create();
+
+        let maybe_root = skin.skeleton();
+
+        if let Some(root) = maybe_root {
+            process_joints(&root, None, &mut skeleton_builder);
+        }
+
+        *maybe_skeleton = Some(skeleton_builder.build());
+    }
+
     for child in node.children() {
-        process_node(&child, buffers, images, meshes);
+        process_node(&child, buffers, images, meshes, maybe_skeleton);
+    }
+}
+
+fn process_joints(
+    node: &gltf::Node,
+    parent_id: Option<i32>,
+    skeleton_builder: &mut SkeletonBuilder,
+) {
+    let id = node.index().to_i32().unwrap();
+    let name = node.name().unwrap_or("None");
+    let transform = node.transform().matrix().into();
+    skeleton_builder.add_joint(id, name.to_owned(), parent_id, transform);
+
+    for node in node.children() {
+        process_joints(&node, Some(id), skeleton_builder);
+    }
+}
+
+fn process_animations(document: &gltf::Document, buffers: &[gltf::buffer::Data]) -> Vec<Animation> {
+    let mut animations = Vec::new();
+    // Load animations
+    // From: https://whoisryosuke.com/blog/2022/importing-gltf-with-wgpu-and-rust
+    let len = document.animations().len();
+    for animation in document.animations() {
+        let animation_name = animation.name().unwrap_or("Unnamed Animation").to_owned();
+        let mut channels = Vec::new();
+        let mut max_time = 0.0;
+
+        for channel in animation.channels() {
+            let sampler = channel.sampler();
+            let target = channel.target();
+            let node_index = target.node().index();
+            let property = match target.property() {
+                gltf::animation::Property::Translation => AnimationProperty::Translation,
+                gltf::animation::Property::Rotation => AnimationProperty::Rotation,
+                gltf::animation::Property::Scale => AnimationProperty::Scale,
+                gltf::animation::Property::MorphTargetWeights => AnimationProperty::Weights,
+            };
+
+            let reader = channel.reader(|buffer| Some(&buffers[buffer.index()]));
+
+            let input_times: Vec<f32> = reader
+                .read_inputs()
+                .expect("Failed to read animation input")
+                .collect();
+
+            let output_values = reader
+                .read_outputs()
+                .expect("Failed to read animation output");
+
+            let interpolation = sampler.interpolation();
+
+            let mut keyframes = Vec::new();
+            max_time = input_times
+                .iter()
+                .cloned()
+                .fold(max_time, |a: f32, b: f32| a.max(b));
+
+            match output_values {
+                gltf::animation::util::ReadOutputs::Translations(translations) => {
+                    for (i, translation) in translations.enumerate() {
+                        let time = input_times[i];
+                        keyframes.push(Keyframe {
+                            time,
+                            value: AnimationValue::Translation(vec3(
+                                translation[0],
+                                translation[1],
+                                translation[2],
+                            )),
+                        });
+                    }
+                }
+                gltf::animation::util::ReadOutputs::Rotations(rotations) => {
+                    for (i, rotation) in rotations.into_f32().enumerate() {
+                        let time = input_times[i];
+                        keyframes.push(Keyframe {
+                            time,
+                            // TODO: Does w come first or last?
+                            value: AnimationValue::Rotation(Quaternion {
+                                v: vec3(rotation[0], rotation[1], rotation[2]),
+                                s: rotation[3],
+                            }),
+                        });
+                    }
+                }
+                gltf::animation::util::ReadOutputs::Scales(scales) => {
+                    for (i, scale) in scales.enumerate() {
+                        let time = input_times[i];
+                        keyframes.push(Keyframe {
+                            time,
+                            value: AnimationValue::Scale(vec3(scale[0], scale[1], scale[2])),
+                        });
+                    }
+                }
+                gltf::animation::util::ReadOutputs::MorphTargetWeights(weights) => {
+                    // TODO:
+                    println!("WARN: ignoring morph target weights; not implemented");
+                    // for (i, weight) in weights.enumerate() {
+                    //     let time = input_times[i];
+                    //     keyframes.push(Keyframe {
+                    //         time,
+                    //         value: AnimationValue::Weights(weight.to_vec()),
+                    //     });
+                    // }
+                }
+            }
+
+            channels.push(AnimationChannel {
+                target_node_index: node_index,
+                target_property: property,
+                keyframes,
+                // TODO:
+                //interpolation,
+            });
+        }
+
+        animations.push(Animation {
+            name: animation_name,
+            channels,
+            duration: max_time,
+        });
     }
+    // panic!("animations");
+    animations
 }

runtime/functor-runtime-common/src/lib.rs

@@ -48,6 +48,7 @@ impl OpaqueState {
     }
 }
 
+pub mod animation;
 pub mod asset;
 mod frame_time;
 pub mod geometry;

runtime/functor-runtime-common/src/model/mod.rs

@@ -1,6 +1,12 @@
+mod skeleton;
+
+use crate::{
+    animation::Animation, geometry::IndexedMesh, render::vertex::VertexPositionTexture,
+    texture::Texture2D,
+};
 use cgmath::Matrix4;
 
-use crate::{geometry::IndexedMesh, render::vertex::VertexPositionTexture, texture::Texture2D};
+pub use skeleton::*;
 
 pub struct ModelMesh {
     // Material info
@@ -13,4 +19,8 @@ pub struct ModelMesh {
 
 pub struct Model {
     pub meshes: Vec<ModelMesh>,
+
+    pub skeleton: Skeleton,
+
+    pub animations: Vec<Animation>,
 }