chore(WebGPU): bug fixes and more developement

Many fixes to bind groups and the move to MapperHelper

Better texture handing

Cleanup samplers from texture views

Fixed an issue in the Volume Mapper FSQ

Added error checking for bind groups

Sources/Rendering/WebGPU/BindGroup/index.js

@@ -8,14 +8,22 @@ function vtkWebGPUBindGroup(publicAPI, model) {
   // Set our className
   model.classHierarchy.push('vtkWebGPUBindGroup');
 
-  publicAPI.addBindable = (bindable) => {
-    // only add new bindables
-    for (let i = 0; i < model.bindables.length; i++) {
-      if (model.bindables[i] === bindable) {
+  publicAPI.setBindables = (bindables) => {
+    // is there a difference between the old and new list?
+    if (model.bindables.length === bindables.length) {
+      let allMatch = true;
+      for (let i = 0; i < model.bindables.length; i++) {
+        if (model.bindables[i] !== bindables[i]) {
+          allMatch = false;
+        }
+      }
+      if (allMatch) {
         return;
       }
     }
-    model.bindables.push(bindable);
+
+    // there is a difference
+    model.bindables = bindables;
     publicAPI.modified();
   };
 
@@ -33,7 +41,7 @@ function vtkWebGPUBindGroup(publicAPI, model) {
     // check mtime
     let mtime = publicAPI.getMTime();
     for (let i = 0; i < model.bindables.length; i++) {
-      const tm = model.bindables[i].getBindGroupTime();
+      const tm = model.bindables[i].getBindGroupTime().getMTime();
       mtime = tm > mtime ? tm : mtime;
     }
     if (mtime < model.bindGroupTime.getMTime()) {

Sources/Rendering/WebGPU/Device/index.js

@@ -53,6 +53,17 @@ function vtkWebGPUDevice(publicAPI, model) {
     return layout;
   };
 
+  publicAPI.getBindGroupLayoutDescription = (layout) => {
+    for (let i = 0; i < model.bindGroupLayouts.length; i++) {
+      if (model.bindGroupLayouts[i].layout === layout) {
+        return model.bindGroupLayouts[i].sval;
+      }
+    }
+    vtkErrorMacro('layout not found');
+    console.trace();
+    return null;
+  };
+
   publicAPI.getPipeline = (hash) => {
     if (hash in model.pipelines) {
       return model.pipelines[hash];

Sources/Rendering/WebGPU/MapperHelper/index.js

@@ -267,23 +267,24 @@ function vtkWebGPUMapperHelper(publicAPI, model) {
 
     model.pipeline = device.getPipeline(model.pipelineHash);
 
-    // todo handle removing a bindable
+    const bindables = [];
     if (model.UBO) {
-      model.bindGroup.addBindable(model.UBO);
+      bindables.push(model.UBO);
     }
 
     if (model.SSBO) {
-      model.bindGroup.addBindable(model.SSBO);
+      bindables.push(model.SSBO);
     }
 
     // add texture BindGroupLayouts
     for (let t = 0; t < model.textureViews.length; t++) {
-      model.bindGroup.addBindable(model.textureViews[t]);
+      bindables.push(model.textureViews[t]);
       const samp = model.textureViews[t].getSampler();
       if (samp) {
-        model.bindGroup.addBindable(samp);
+        bindables.push(samp);
       }
     }
+    model.bindGroup.setBindables(bindables);
 
     // build VBO for this primitive
     // build the pipeline if needed

Sources/Rendering/WebGPU/Pipeline/index.js

@@ -61,6 +61,8 @@ function vtkWebGPUPipeline(publicAPI, model) {
     });
   };
 
+  publicAPI.getBindGroupLayout = (idx) => model.layouts[idx].layout;
+
   publicAPI.getBindGroupLayoutCount = (lname) => {
     for (let i = 0; i < model.layouts.length; i++) {
       if (model.layouts[i].name === lname) {

Sources/Rendering/WebGPU/README.md

@@ -31,15 +31,16 @@ is needed.
 - cropping planes for polydata mapper
 - update widgets to use the new async hardware selector API
 
+- create new volume renderer built for multivolume rendering - in progress
+  - traverse all volumes and register with volume pass - done
+  - render all volumes hexahedra to get depth buffer near and far
+    merged with opaque pass depth buffer - done
+  - render all volumes in single mapper using prior near/far depth textures - in progress
+
 Waiting on fixes/dev in WebGPU spec
 
 - 3d textures (as of April 21 2021 Dawn lacks support for 1d and 3d)
 - image display (use 3d texture)
-- create new volume renderer built for multivolume rendering
-  - traverse all volumes and register with volume pass
-  - render all volumes hexahedra to get depth buffer near and far
-    merged with opaque pass depth buffer
-  - render all volumes in single mapper using prior near/far depth textures
 - more cross platform testing and bug fixing
 - single volume rendering (abandon)
 
@@ -53,7 +54,7 @@ Waiting on fixes/dev in WebGPU spec
 
 # Developer Notes
 
-If you want to extend WebGPU most of the work is done in the mapper classes such as WebGPUPolyDataMapper so probably that is where you should start. It has some subclasses that extend it such as Sphere/Stick/Glyph3d mapper. There is also an example of user code creating a new mapper in the CustomWebGPUCone example. If you are interested in render passes then ForwardPass is the main entry point and makes use of other passes by default.
+If you want to extend WebGPU most of the work is done in the mapper classes. The simplest mapper is WebGPUMapperHelper which is the base for all mappers (either directly as a superclass or as a member variable) so probably that is where you should start. It has some subclasses that extend it or use it such as Sphere/Stick/Glyph3d mapper. There is also an example of user code creating a new mapper in the CustomWebGPUCone example. If you are interested in render passes then ForwardPass is the main entry point and makes use of other passes by default.
 
 Here are some quick notes on the WebGPU classes and how they work together. Classes that typically have one instance are described as such even though you can have multiple instances of them.
 
@@ -67,7 +68,9 @@ Here are some quick notes on the WebGPU classes and how they work together. Clas
 
 - Texture - many instances, a structured chunk of memory typically 1 to 3 dimensions with optional support for mipmapping, etc. Can be created from a buffer or a JS image. Often created by mappers or render passes.
 
-- Sampler - many instances - something that can be used to sample a texture, typically linear or nearest, etc. Requested often by mappers using textures.
+- TextureView - many instances, a view of a texture, lightweight and a bindable
+
+- Sampler - many instances - something that can be used to sample a texture, typically linear or nearest, etc. Requested often by mappers using textures. a bindable
 
 - ShaderCache - one instance, caches many shader modules, owned by the Device. Requested typically by mappers.
 
@@ -90,9 +93,12 @@ can be used to run a pipeline on those fragment destinations.
 
 - Mapper - maps vtkDataSet to graphics primitives (draws them) Creates many objects to get the job done including VertexInputs, Pipelines, Buffers, Textures, Samplers. Typically sets up everything and then registers pipelines to call it back when they render. For example, a single mapper when it renders with lines and triangles would request two pipelines and set up their vertex input etc, and then register a reuqest for those pipelines to call it back when the pipelines render. Later on after all mappers have "rendered" the resulting pipelines would be executed by the renderer and for each pipeline all mappers using that pipeline would get a callback so they can bind and draw their primitives. This is different from OpenGL where each mapper would draw during its render pass lines then triangles. With WebGPU (essentially) all lines are drawn together for all mappers, then all triangles for all mappers.
 
-- UniformBuffer - a UBO in a class, mappers and renderers have them by default
+- Bind Group - hold bindables (textures samplers, UBOs SSBOs) and organizes them
+so they can be bound as needed. Typically one for each renderer and one for each mapper.
+
+- UniformBuffer - a UBO in a class, mappers and renderers have them by default, a bindable
 
-- StorageBuffer - a SSBO that can be used when you need a SSBO
+- StorageBuffer - a SSBO that can be used when you need a SSBO, a bindable
 
 The buffer and texture managers also cache their objects so that these large GPU objects can be shared betwen mappers. Both of them take a request and return something from the cache. In both cases the source property of the request indicates what object is holding onto the buffer/texture.
 
@@ -101,12 +107,12 @@ what WebGPU uses. So to avoid confusion we call WebGPU render passes "render enc
 This matches WebGPU terminology as they are encoders and sometimes called render pass
 encoders in the WebGPU spec.
 
-There is a notion of bindable things in this implementation. The mapper helper keeps an array of
-bindable things that it uses/manages. Right now these unclude UBOs, SSBOs, and TextureViews. TextureViews
-may also include a sampler in them. A bindable thing must answer to the following interface
+There is a notion of bindable things in this implementation. BingGroups keep an array of
+bindable things that it uses/manages. Right now these unclude UBOs, SSBOs, and TextureViews and Smaplers. A bindable thing must answer to the following interface
 ```
 set/getName
-getBindGroupLayout(device)
-getBindGroup()
-getShaderCode(pipeline)
+getBindGroupLayoutEntry()
+getBindGroupEntry()
+getBindGroupTime()
+getShaderCode(group, binding)
 ```

Sources/Rendering/WebGPU/RenderEncoder/index.js

@@ -21,6 +21,11 @@ function vtkWebGPURenderEncoder(publicAPI, model) {
 
   publicAPI.setPipeline = (pl) => {
     model.handle.setPipeline(pl.getHandle());
+    // todo check attachment state?
+    // console.log(
+    //   `bound pipeline for ${model.pipelineHash} ${JSON.stringify(pl.get())}`
+    // );
+    // console.log(model.description);
     model.boundPipeline = pl;
   };
 
@@ -36,11 +41,22 @@ function vtkWebGPURenderEncoder(publicAPI, model) {
   };
 
   publicAPI.activateBindGroup = (bg) => {
+    const device = model.boundPipeline.getDevice();
     const midx = model.boundPipeline.getBindGroupLayoutCount(bg.getName());
-    model.handle.setBindGroup(
-      midx,
-      bg.getBindGroup(model.boundPipeline.getDevice())
+    model.handle.setBindGroup(midx, bg.getBindGroup(device));
+    // verify bind group layout matches
+    const bgl1 = device.getBindGroupLayoutDescription(
+      bg.getBindGroupLayout(device)
     );
+    const bgl2 = device.getBindGroupLayoutDescription(
+      model.boundPipeline.getBindGroupLayout(midx)
+    );
+    if (bgl1 !== bgl2) {
+      console.log(
+        `renderEncoder ${model.pipelineHash} mismatched bind group layouts bind group has\n${bgl1}\n versus pipeline\n${bgl2}\n`
+      );
+      console.trace();
+    }
   };
 
   publicAPI.attachTextureViews = () => {

Sources/Rendering/WebGPU/Renderer/index.js

@@ -199,7 +199,6 @@ function vtkWebGPURenderer(publicAPI, model) {
       model.pipelineCallbacks = [];
 
       model.renderEncoder.begin(model.parent.getCommandEncoder());
-
       publicAPI.updateUBO();
     } else {
       publicAPI.scissorAndViewport(model.renderEncoder);
@@ -250,7 +249,6 @@ function vtkWebGPURenderer(publicAPI, model) {
     if (prepass) {
       // clear last pipelines
       model.pipelineCallbacks = [];
-
       model.renderEncoder.begin(model.parent.getCommandEncoder());
     } else {
       publicAPI.scissorAndViewport(model.renderEncoder);
@@ -276,7 +274,6 @@ function vtkWebGPURenderer(publicAPI, model) {
     if (prepass) {
       // clear last pipelines
       model.pipelineCallbacks = [];
-
       model.renderEncoder.begin(model.parent.getCommandEncoder());
     } else {
       publicAPI.scissorAndViewport(model.renderEncoder);
@@ -410,7 +407,7 @@ export function extend(publicAPI, model, initialValues = {}) {
 
   model.bindGroup = vtkWebGPUBindGroup.newInstance();
   model.bindGroup.setName('rendererBG');
-  model.bindGroup.addBindable(model.UBO);
+  model.bindGroup.setBindables([model.UBO]);
 
   model.tmpMat4 = mat4.identity(new Float64Array(16));
 

Sources/Rendering/WebGPU/StorageBuffer/index.js

@@ -163,7 +163,7 @@ function vtkWebGPUStorageBuffer(publicAPI, model) {
     }
   };
 
-  publicAPI.getSendTime = () => model.sendTime.getMTime();
+  publicAPI.getSendTime = () => model._sendTime.getMTime();
   publicAPI.getShaderCode = (binding, group) => {
     const lines = [`struct ${model.name}StructEntry\n{`];
     for (let i = 0; i < model.bufferEntries.length; i++) {

Sources/Rendering/WebGPU/Texture/index.js

@@ -1,6 +1,7 @@
 import macro from 'vtk.js/Sources/macro';
 import vtkWebGPUBufferManager from 'vtk.js/Sources/Rendering/WebGPU/BufferManager';
 import vtkWebGPUTextureView from 'vtk.js/Sources/Rendering/WebGPU/TextureView';
+import vtkWebGPUTypes from 'vtk.js/Sources/Rendering/WebGPU/Types';
 
 const { BufferUsage } = vtkWebGPUBufferManager;
 
@@ -57,32 +58,49 @@ function vtkWebGPUTexture(publicAPI, model) {
   // set the data
   publicAPI.writeImageData = (req) => {
     let bufferBytesPerRow = model.width * 4;
-    if (req.dataArray) {
+    if (req.dataArray || req.nativeArray) {
       // create and write the buffer
       const buffRequest = {
-        dataArray: req.dataArray,
-        time: req.dataArray.getMTime(),
         /* eslint-disable no-undef */
         usage: BufferUsage.Texture,
         /* eslint-enable no-undef */
+        // todo this needs to be computed from the texture format
         format: 'unorm8x4',
       };
 
+      if (req.dataArray) {
+        buffRequest.dataArray = req.dataArray;
+        buffRequest.time = req.dataArray.getMTime();
+      }
+      if (req.nativeArray) {
+        buffRequest.nativeArray = req.nativeArray;
+      }
+
       // bytesPerRow must be a multiple of 256 so we might need to rebuild
-      // the data here before passing to the buffer. If it is unorm8x4 then
+      // the data here before passing to the buffer. e.g. if it is unorm8x4 then
       // we need to have width be a multiple of 64
-      if (model.width % 64) {
+      const tDetails = vtkWebGPUTypes.getDetailsFromTextureFormat(model.format);
+      const currWidthInBytes = model.width * tDetails.stride;
+      if (currWidthInBytes % 256) {
         const oArray = req.dataArray.getData();
-        const bufferWidth = 64 * Math.floor((model.width + 63) / 64);
-        const nArray = new Uint8Array(bufferWidth * model.height * 4);
+        const bufferWidthInBytes =
+          256 * Math.floor((currWidthInBytes + 255) / 256);
+        const bufferWidth = bufferWidthInBytes / oArray.BYTES_PER_ELEMENT;
+        const oWidth = currWidthInBytes / oArray.BYTES_PER_ELEMENT;
+
+        const nArray = macro.newTypedArray(
+          oArray.name,
+          bufferWidth * model.height
+        );
+
         for (let v = 0; v < model.height; v++) {
           nArray.set(
-            oArray.subarray(v * 4 * model.width, (v + 1) * 4 * model.width),
-            v * 4 * bufferWidth
+            oArray.subarray(v * oWidth, (v + 1) * oWidth),
+            v * bufferWidth
           );
         }
         buffRequest.nativeArray = nArray;
-        bufferBytesPerRow = bufferWidth * 4;
+        bufferBytesPerRow = bufferWidthInBytes;
       }
       const buff = model.device.getBufferManager().getBuffer(buffRequest);
       model.buffer = buff;