From 83b095519d78896cc2ccfe14fc3a04a45d74250a Mon Sep 17 00:00:00 2001 From: Carifio24 Date: Mon, 9 Dec 2024 17:24:28 -0500 Subject: [PATCH 01/11] Combine scatter spheres into one mesh per color. --- glue_ar/common/scatter_gltf.py | 167 +++++++++++++++++++++++---------- 1 file changed, 117 insertions(+), 50 deletions(-) diff --git a/glue_ar/common/scatter_gltf.py b/glue_ar/common/scatter_gltf.py index 1cdb2c9..4d2d3de 100644 --- a/glue_ar/common/scatter_gltf.py +++ b/glue_ar/common/scatter_gltf.py @@ -1,4 +1,5 @@ -from gltflib import AccessorType, BufferTarget, ComponentType, PrimitiveMode +from collections import defaultdict +from gltflib import Accessor, AccessorType, BufferTarget, ComponentType, PrimitiveMode from glue_vispy_viewers.common.viewer_state import Vispy3DViewerState from glue_vispy_viewers.scatter.layer_state import ScatterLayerState from numpy import isfinite, ndarray @@ -12,7 +13,7 @@ from glue_ar.common.shapes import cone_triangles, cone_points, cylinder_points, cylinder_triangles, \ normalize, rectangular_prism_triangulation, sphere_triangles from glue_ar.gltf_utils import add_points_to_bytearray, add_triangles_to_bytearray, index_mins, index_maxes -from glue_ar.utils import Viewer3DState, get_stretches, iterable_has_nan, hex_to_components, layer_color, \ +from glue_ar.utils import Viewer3DState, get_stretches, iterable_has_nan, hex_to_components, layer_color, offset_triangles, \ unique_id, xyz_bounds, xyz_for_layer, Bounds from glue_ar.common.gltf_builder import GLTFBuilder from glue_ar.common.scatter import Scatter3DLayerState, ScatterLayerState3D, \ @@ -227,33 +228,7 @@ def add_scatter_layer_gltf(builder: GLTFBuilder, scaled=True) data = data[:, [1, 2, 0]] - barr = bytearray() - add_triangles_to_bytearray(barr, triangles) - triangles_len = len(barr) - max_index = max(idx for tri in triangles for idx in tri) - - buffer = builder.buffer_count - builder.add_buffer_view( - buffer=buffer, - byte_length=triangles_len, - byte_offset=0, - target=BufferTarget.ELEMENT_ARRAY_BUFFER, - ) - builder.add_accessor( - buffer_view=builder.buffer_view_count-1, - component_type=ComponentType.UNSIGNED_INT, - count=len(triangles)*3, - type=AccessorType.SCALAR, - mins=[0], - maxes=[max_index], - ) - sphere_triangles_accessor = builder.accessor_count - 1 - first_material_index = builder.material_count - if fixed_color: - color = layer_color(layer_state) - color_components = hex_to_components(color) - builder.add_material(color=color_components, opacity=layer_state.alpha) buffer = builder.buffer_count cmap = layer_state.cmap @@ -264,44 +239,136 @@ def add_scatter_layer_gltf(builder: GLTFBuilder, uri = f"layer_{unique_id()}.bin" sizes = sizes_for_scatter_layer(layer_state, bounds, mask) - for i, point in enumerate(data): - - prev_len = len(barr) - size = radius if fixed_size else sizes[i] - pts = points_getter(point, size) - add_points_to_bytearray(barr, pts) - point_mins = index_mins(pts) - point_maxes = index_maxes(pts) - - if not fixed_color: - cval = cmap_vals[i] - normalized = max(min((cval - layer_state.cmap_vmin) / crange, 1), 0) - cindex = int(normalized * 255) - color = cmap(cindex) - builder.add_material(color, layer_state.alpha) + barr = bytearray() + if fixed_color: + points = [] + tris = [] + triangle_offset = 0 + for i, point in enumerate(data): + size = radius if fixed_size else sizes[i] + pts = points_getter(point, size) + points.append(pts) + pt_triangles = offset_triangles(triangles, triangle_offset) + triangle_offset += len(pts) + tris.append(pt_triangles) + + mesh_points = [pt for pts in points for pt in pts] + mesh_triangles = [tri for sphere in tris for tri in sphere] + add_points_to_bytearray(barr, mesh_points) + points_len = len(barr) + add_triangles_to_bytearray(barr, mesh_triangles) + point_mins = index_mins(mesh_points) + point_maxes = index_maxes(mesh_points) + + buffer = builder.buffer_count builder.add_buffer_view( buffer=buffer, - byte_length=len(barr)-prev_len, - byte_offset=prev_len, + byte_length=points_len, + byte_offset=0, target=BufferTarget.ARRAY_BUFFER, ) builder.add_accessor( buffer_view=builder.buffer_view_count-1, component_type=ComponentType.FLOAT, - count=len(pts), + count=len(mesh_points), type=AccessorType.VEC3, mins=point_mins, maxes=point_maxes, ) + builder.add_buffer_view( + buffer=buffer, + byte_length=len(barr)-points_len, + byte_offset=points_len, + target=BufferTarget.ELEMENT_ARRAY_BUFFER, + ) + builder.add_accessor( + buffer_view=builder.buffer_view_count-1, + component_type=ComponentType.UNSIGNED_INT, + count=len(mesh_triangles)*3, + type=AccessorType.SCALAR, + mins=[0], + maxes=[max(idx for tri in mesh_triangles for idx in tri)], + ) - material_index = builder.material_count - 1 + color = layer_color(layer_state) + color_components = hex_to_components(color) + builder.add_material(color=color_components, opacity=layer_state.alpha) builder.add_mesh( - position_accessor=builder.accessor_count-1, - indices_accessor=sphere_triangles_accessor, - material=material_index, + position_accessor=builder.accessor_count-2, + indices_accessor=builder.accessor_count-1, + material=builder.material_count-1, ) + else: + points_by_color = defaultdict(list) + triangles_by_color = defaultdict(list) + triangle_offsets = defaultdict(int) + color_materials = defaultdict(int) + for i, point in enumerate(data): + cval = cmap_vals[i] + normalized = max(min((cval - layer_state.cmap_vmin) / crange, 1), 0) + cindex = int(normalized * 255) + color = cmap(cindex) + builder.add_material(color, layer_state.alpha) + color_materials[color] = builder.material_count - 1 + + size = radius if fixed_size else sizes[i] + pts = points_getter(point, size) + pt_triangles = offset_triangles(triangles, triangle_offsets[color]) + triangle_offsets[color] += len(pts) + points_by_color[color].append(pts) + triangles_by_color[color].append(pt_triangles) + + for color, points in points_by_color.items(): + tris = triangles_by_color[color] + mesh_points = [pt for pts in points for pt in pts] + mesh_triangles = [tri for sphere in tris for tri in sphere] + + points_offset = len(barr) + add_points_to_bytearray(barr, mesh_points) + triangles_offset = len(barr) + add_triangles_to_bytearray(barr, mesh_triangles) + point_mins = index_mins(mesh_points) + point_maxes = index_maxes(mesh_points) + + buffer = builder.buffer_count + builder.add_buffer_view( + buffer=buffer, + byte_length=triangles_offset-points_offset, + byte_offset=points_offset, + target=BufferTarget.ARRAY_BUFFER, + ) + builder.add_accessor( + buffer_view=builder.buffer_view_count-1, + component_type=ComponentType.FLOAT, + count=len(mesh_points), + type=AccessorType.VEC3, + mins=point_mins, + maxes=point_maxes, + ) + builder.add_buffer_view( + buffer=buffer, + byte_length=len(barr)-triangles_offset, + byte_offset=triangles_offset, + target=BufferTarget.ELEMENT_ARRAY_BUFFER, + ) + builder.add_accessor( + buffer_view=builder.buffer_view_count-1, + component_type=ComponentType.UNSIGNED_INT, + count=len(mesh_triangles), + type=AccessorType.SCALAR, + mins=[0], + maxes=[max(idx for tri in mesh_triangles for idx in tri)], + ) + + material = color_materials[color] + builder.add_mesh( + position_accessor=builder.accessor_count-2, + indices_accessor=builder.accessor_count-1, + material=material, + ) + builder.add_buffer(byte_length=len(barr), uri=uri) builder.add_file_resource(uri, data=barr) From 040a4e242d16780e4c0a9efef06ba807d7da9870 Mon Sep 17 00:00:00 2001 From: Carifio24 Date: Mon, 9 Dec 2024 17:33:58 -0500 Subject: [PATCH 02/11] Fix incorrect size in colormapped meshes. --- glue_ar/common/scatter_gltf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/glue_ar/common/scatter_gltf.py b/glue_ar/common/scatter_gltf.py index 4d2d3de..18ef66c 100644 --- a/glue_ar/common/scatter_gltf.py +++ b/glue_ar/common/scatter_gltf.py @@ -356,7 +356,7 @@ def add_scatter_layer_gltf(builder: GLTFBuilder, builder.add_accessor( buffer_view=builder.buffer_view_count-1, component_type=ComponentType.UNSIGNED_INT, - count=len(mesh_triangles), + count=len(mesh_triangles)*3, type=AccessorType.SCALAR, mins=[0], maxes=[max(idx for tri in mesh_triangles for idx in tri)], From f9f27245deb1c289441468092dc8695c05c7d231 Mon Sep 17 00:00:00 2001 From: Carifio24 Date: Tue, 10 Dec 2024 12:44:15 -0500 Subject: [PATCH 03/11] Don't create redundant materials in glTF scatter. --- glue_ar/common/scatter_gltf.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/glue_ar/common/scatter_gltf.py b/glue_ar/common/scatter_gltf.py index 18ef66c..1eaf44c 100644 --- a/glue_ar/common/scatter_gltf.py +++ b/glue_ar/common/scatter_gltf.py @@ -310,8 +310,11 @@ def add_scatter_layer_gltf(builder: GLTFBuilder, normalized = max(min((cval - layer_state.cmap_vmin) / crange, 1), 0) cindex = int(normalized * 255) color = cmap(cindex) - builder.add_material(color, layer_state.alpha) - color_materials[color] = builder.material_count - 1 + material_index = color_materials.get(color, None) + if material_index is None: + builder.add_material(color, layer_state.alpha) + material_index = builder.material_count - 1 + color_materials[color] = material_index size = radius if fixed_size else sizes[i] pts = points_getter(point, size) From 28384eaa4e3afec8251fbb89b8529ab83e150752 Mon Sep 17 00:00:00 2001 From: Carifio24 Date: Wed, 11 Dec 2024 10:52:53 -0500 Subject: [PATCH 04/11] Work on implementing ability to chunk glTF meshes into a specified number of points. --- glue_ar/common/scatter_gltf.py | 137 +++++++++++++++++++++++---------- glue_ar/gltf_utils.py | 9 ++- 2 files changed, 103 insertions(+), 43 deletions(-) diff --git a/glue_ar/common/scatter_gltf.py b/glue_ar/common/scatter_gltf.py index 1eaf44c..46b0f85 100644 --- a/glue_ar/common/scatter_gltf.py +++ b/glue_ar/common/scatter_gltf.py @@ -1,5 +1,5 @@ from collections import defaultdict -from gltflib import Accessor, AccessorType, BufferTarget, ComponentType, PrimitiveMode +from gltflib import AccessorType, BufferTarget, ComponentType, PrimitiveMode from glue_vispy_viewers.common.viewer_state import Vispy3DViewerState from glue_vispy_viewers.scatter.layer_state import ScatterLayerState from numpy import isfinite, ndarray @@ -12,7 +12,7 @@ from glue_ar.common.scatter_export_options import ARIpyvolumeScatterExportOptions, ARVispyScatterExportOptions from glue_ar.common.shapes import cone_triangles, cone_points, cylinder_points, cylinder_triangles, \ normalize, rectangular_prism_triangulation, sphere_triangles -from glue_ar.gltf_utils import add_points_to_bytearray, add_triangles_to_bytearray, index_mins, index_maxes +from glue_ar.gltf_utils import SHORT_MAX, add_points_to_bytearray, add_triangles_to_bytearray, index_mins, index_maxes from glue_ar.utils import Viewer3DState, get_stretches, iterable_has_nan, hex_to_components, layer_color, offset_triangles, \ unique_id, xyz_bounds, xyz_for_layer, Bounds from glue_ar.common.gltf_builder import GLTFBuilder @@ -209,7 +209,8 @@ def add_scatter_layer_gltf(builder: GLTFBuilder, points_getter: PointsGetter, triangles: List[Tuple[int, int, int]], bounds: Bounds, - clip_to_bounds: bool = True): + clip_to_bounds: bool = True, + points_per_mesh: Optional[int] = None): if layer_state is None: return @@ -228,8 +229,6 @@ def add_scatter_layer_gltf(builder: GLTFBuilder, scaled=True) data = data[:, [1, 2, 0]] - - buffer = builder.buffer_count cmap = layer_state.cmap cmap_attr = "cmap_attribute" if vispy_layer_state else "cmap_att" @@ -241,9 +240,22 @@ def add_scatter_layer_gltf(builder: GLTFBuilder, sizes = sizes_for_scatter_layer(layer_state, bounds, mask) barr = bytearray() + n_points = len(data) + + # If points per mesh is not specified, + # we don't do any mesh chunking. + # Note that this will work for colormapping as well + if points_per_mesh is None: + points_per_mesh = n_points + if fixed_color: points = [] tris = [] + + color = layer_color(layer_state) + color_components = hex_to_components(color) + builder.add_material(color=color_components, opacity=layer_state.alpha) + triangle_offset = 0 for i, point in enumerate(data): size = radius if fixed_size else sizes[i] @@ -253,52 +265,92 @@ def add_scatter_layer_gltf(builder: GLTFBuilder, triangle_offset += len(pts) tris.append(pt_triangles) - mesh_points = [pt for pts in points for pt in pts] - mesh_triangles = [tri for sphere in tris for tri in sphere] - add_points_to_bytearray(barr, mesh_points) - points_len = len(barr) - add_triangles_to_bytearray(barr, mesh_triangles) - point_mins = index_mins(mesh_points) - point_maxes = index_maxes(mesh_points) - buffer = builder.buffer_count - builder.add_buffer_view( - buffer=buffer, - byte_length=points_len, - byte_offset=0, - target=BufferTarget.ARRAY_BUFFER, - ) - builder.add_accessor( - buffer_view=builder.buffer_view_count-1, - component_type=ComponentType.FLOAT, - count=len(mesh_points), - type=AccessorType.VEC3, - mins=point_mins, - maxes=point_maxes, - ) + pts_count = len(points_getter((0, 0, 0), 1)) + for _ in range(min(points_per_mesh, n_points)): + pt_triangles = offset_triangles(triangles, triangle_offset) + triangle_offset += pts_count + tris.append(pt_triangles) + + mesh_triangles = [tri for sphere in tris for tri in sphere] + max_triangle_index = max(idx for tri in mesh_triangles for idx in tri) + use_short = max_triangle_index <= SHORT_MAX + triangles_start = len(barr) + add_triangles_to_bytearray(barr, mesh_triangles, short=use_short) + triangles_len = len(barr) builder.add_buffer_view( buffer=buffer, - byte_length=len(barr)-points_len, - byte_offset=points_len, + byte_length=triangles_len-triangles_start, + byte_offset=triangles_start, target=BufferTarget.ELEMENT_ARRAY_BUFFER, ) + component_type = ComponentType.UNSIGNED_SHORT if use_short else ComponentType.UNSIGNED_INT builder.add_accessor( buffer_view=builder.buffer_view_count-1, - component_type=ComponentType.UNSIGNED_INT, + component_type=component_type, count=len(mesh_triangles)*3, type=AccessorType.SCALAR, mins=[0], - maxes=[max(idx for tri in mesh_triangles for idx in tri)], + maxes=max_triangle_index, ) - color = layer_color(layer_state) - color_components = hex_to_components(color) - builder.add_material(color=color_components, opacity=layer_state.alpha) - builder.add_mesh( - position_accessor=builder.accessor_count-2, - indices_accessor=builder.accessor_count-1, - material=builder.material_count-1, - ) + start = 0 + buffer = builder.buffer_count + triangles_accessor = builder.accessor_count - 1 + while start < n_points: + mesh_points = [pt for pts in points[start:points_per_mesh] for pt in pts] + barr_offset = len(barr) + add_points_to_bytearray(barr, mesh_points) + point_mins = index_mins(mesh_points) + point_maxes = index_maxes(mesh_points) + + builder.add_buffer_view( + buffer=buffer, + byte_length=len(barr)-barr_offset, + byte_offset=0, + target=BufferTarget.ARRAY_BUFFER, + ) + builder.add_accessor( + buffer_view=builder.buffer_view_count-1, + component_type=ComponentType.FLOAT, + count=len(mesh_points), + type=AccessorType.VEC3, + mins=point_mins, + maxes=point_maxes, + ) + points_accessor = builder.accessor_count - 1 + + count = n_points - start + triangles_accessor = triangles_accessor + # This should only happen on the final iteration + # or not at all, if points_per_mesh is a divisor of count + # But in this case we do need a separate accessor as the + # byte length is different + if count < points_per_mesh: + byte_length = n_points * triangles_len / count + builder.add_buffer_view( + buffer=buffer, + byte_length=byte_length, + byte_offset=triangles_start, + target=BufferTarget.ELEMENT_ARRAY_BUFFER, + ) + component_type = ComponentType.UNSIGNED_SHORT if use_short else ComponentType.UNSIGNED_INT + builder.add_accessor( + buffer_view=builder.buffer_view_count-1, + component_type=component_type, + count=len(mesh_triangles)*3, + type=AccessorType.SCALAR, + mins=[0], + maxes=max_triangle_index, + ) + triangles_accessor = builder.accessor_count - 1 + + builder.add_mesh( + position_accessor=points_accessor, + indices_accessor=triangles_accessor, + material=builder.material_count-1, + ) + start += points_per_mesh else: points_by_color = defaultdict(list) @@ -327,11 +379,13 @@ def add_scatter_layer_gltf(builder: GLTFBuilder, tris = triangles_by_color[color] mesh_points = [pt for pts in points for pt in pts] mesh_triangles = [tri for sphere in tris for tri in sphere] + max_triangle_index = max(idx for tri in mesh_triangles for idx in tri) + use_short = max_triangle_index <= SHORT_MAX points_offset = len(barr) add_points_to_bytearray(barr, mesh_points) triangles_offset = len(barr) - add_triangles_to_bytearray(barr, mesh_triangles) + add_triangles_to_bytearray(barr, mesh_triangles, short=use_short) point_mins = index_mins(mesh_points) point_maxes = index_maxes(mesh_points) @@ -356,9 +410,10 @@ def add_scatter_layer_gltf(builder: GLTFBuilder, byte_offset=triangles_offset, target=BufferTarget.ELEMENT_ARRAY_BUFFER, ) + component_type = ComponentType.UNSIGNED_SHORT if use_short else ComponentType.UNSIGNED_INT builder.add_accessor( buffer_view=builder.buffer_view_count-1, - component_type=ComponentType.UNSIGNED_INT, + component_type=component_type, count=len(mesh_triangles)*3, type=AccessorType.SCALAR, mins=[0], diff --git a/glue_ar/gltf_utils.py b/glue_ar/gltf_utils.py index a0e592e..ceab78c 100644 --- a/glue_ar/gltf_utils.py +++ b/glue_ar/gltf_utils.py @@ -18,6 +18,8 @@ "meshoptimizer": "EXT_meshopt_compression", } +SHORT_MAX = 65_535 + def create_material_for_color( color: List[int], @@ -40,10 +42,13 @@ def add_points_to_bytearray(arr: bytearray, points: Iterable[Iterable[Union[int, arr.extend(struct.pack('f', coordinate)) -def add_triangles_to_bytearray(arr: bytearray, triangles: Iterable[Iterable[int]]): +def add_triangles_to_bytearray(arr: bytearray, + triangles: Iterable[Iterable[int]], + short: bool = False): + format = "H" if short else "I" for triangle in triangles: for index in triangle: - arr.extend(struct.pack('I', index)) + arr.extend(struct.pack(format, index)) T = TypeVar("T", bound=Union[int, float]) From fb7df3f216ea0616c15e5ebc44e3e105d2933671 Mon Sep 17 00:00:00 2001 From: Carifio24 Date: Wed, 11 Dec 2024 13:30:34 -0500 Subject: [PATCH 05/11] Fix up some issues with the fixed-color points-per-mesh implementation. --- glue_ar/common/scatter_gltf.py | 34 ++++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/glue_ar/common/scatter_gltf.py b/glue_ar/common/scatter_gltf.py index 46b0f85..e42e54d 100644 --- a/glue_ar/common/scatter_gltf.py +++ b/glue_ar/common/scatter_gltf.py @@ -12,7 +12,7 @@ from glue_ar.common.scatter_export_options import ARIpyvolumeScatterExportOptions, ARVispyScatterExportOptions from glue_ar.common.shapes import cone_triangles, cone_points, cylinder_points, cylinder_triangles, \ normalize, rectangular_prism_triangulation, sphere_triangles -from glue_ar.gltf_utils import SHORT_MAX, add_points_to_bytearray, add_triangles_to_bytearray, index_mins, index_maxes +from glue_ar.gltf_utils import SHORT_MAX, add_points_to_bytearray, add_triangles_to_bytearray, index_mins, index_maxes, SHORT_MAX from glue_ar.utils import Viewer3DState, get_stretches, iterable_has_nan, hex_to_components, layer_color, offset_triangles, \ unique_id, xyz_bounds, xyz_for_layer, Bounds from glue_ar.common.gltf_builder import GLTFBuilder @@ -261,10 +261,6 @@ def add_scatter_layer_gltf(builder: GLTFBuilder, size = radius if fixed_size else sizes[i] pts = points_getter(point, size) points.append(pts) - pt_triangles = offset_triangles(triangles, triangle_offset) - triangle_offset += len(pts) - tris.append(pt_triangles) - pts_count = len(points_getter((0, 0, 0), 1)) for _ in range(min(points_per_mesh, n_points)): @@ -291,14 +287,15 @@ def add_scatter_layer_gltf(builder: GLTFBuilder, count=len(mesh_triangles)*3, type=AccessorType.SCALAR, mins=[0], - maxes=max_triangle_index, + maxes=[max_triangle_index], ) start = 0 buffer = builder.buffer_count triangles_accessor = builder.accessor_count - 1 while start < n_points: - mesh_points = [pt for pts in points[start:points_per_mesh] for pt in pts] + print(start, n_points) + mesh_points = [pt for pts in points[start:start+points_per_mesh] for pt in pts] barr_offset = len(barr) add_points_to_bytearray(barr, mesh_points) point_mins = index_mins(mesh_points) @@ -307,7 +304,7 @@ def add_scatter_layer_gltf(builder: GLTFBuilder, builder.add_buffer_view( buffer=buffer, byte_length=len(barr)-barr_offset, - byte_offset=0, + byte_offset=barr_offset, target=BufferTarget.ARRAY_BUFFER, ) builder.add_accessor( @@ -320,14 +317,18 @@ def add_scatter_layer_gltf(builder: GLTFBuilder, ) points_accessor = builder.accessor_count - 1 - count = n_points - start - triangles_accessor = triangles_accessor # This should only happen on the final iteration # or not at all, if points_per_mesh is a divisor of count # But in this case we do need a separate accessor as the # byte length is different + count = n_points - start if count < points_per_mesh: - byte_length = n_points * triangles_len / count + triangles_count = len(tris) + byte_length = count * triangles_len // triangles_count + print(triangles_len, triangles_count, count) + print(byte_length) + mesh_triangles = [tri for sphere in tris[:count] for tri in sphere] + max_triangle_index = max(idx for tri in mesh_triangles for idx in tri) builder.add_buffer_view( buffer=buffer, byte_length=byte_length, @@ -338,10 +339,10 @@ def add_scatter_layer_gltf(builder: GLTFBuilder, builder.add_accessor( buffer_view=builder.buffer_view_count-1, component_type=component_type, - count=len(mesh_triangles)*3, + count=len(triangles)*3*count, type=AccessorType.SCALAR, mins=[0], - maxes=max_triangle_index, + maxes=[max_triangle_index], ) triangles_accessor = builder.accessor_count - 1 @@ -484,13 +485,18 @@ def add_vispy_scatter_layer_gltf(builder: GLTFBuilder, points_getter = sphere_points_getter(theta_resolution=theta_resolution, phi_resolution=phi_resolution) + triangles_per_point = len(triangles) + # points_per_mesh = SHORT_MAX // (triangles_per_point * 3) + points_per_mesh = 100 + add_scatter_layer_gltf(builder=builder, viewer_state=viewer_state, layer_state=layer_state, points_getter=points_getter, triangles=triangles, bounds=bounds, - clip_to_bounds=clip_to_bounds) + clip_to_bounds=clip_to_bounds, + points_per_mesh=points_per_mesh) @ar_layer_export(Scatter3DLayerState, "Scatter", ARIpyvolumeScatterExportOptions, ("gltf", "glb")) From 06cfab7412d95bebc5d0b0cdbf9f6ac8b579e965 Mon Sep 17 00:00:00 2001 From: Carifio24 Date: Wed, 11 Dec 2024 16:40:06 -0500 Subject: [PATCH 06/11] Use gltf-transform for compression. --- glue_ar/__init__.py | 2 +- glue_ar/compression.py | 11 +++++------ js/package.json | 3 +-- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/glue_ar/__init__.py b/glue_ar/__init__.py index a6f3850..ac5c27f 100644 --- a/glue_ar/__init__.py +++ b/glue_ar/__init__.py @@ -11,7 +11,7 @@ def setup_common(): from .common.usd_builder import USDBuilder # noqa: F401 from .common.stl_builder import STLBuilder # noqa: F401 - from .compression import compress_gltfpack, compress_gltf_pipeline # noqa: F401 + from .compression import compress_draco, compress_meshoptimizer # noqa: F401 def setup_qt(): diff --git a/glue_ar/compression.py b/glue_ar/compression.py index 1e49686..0941fcc 100644 --- a/glue_ar/compression.py +++ b/glue_ar/compression.py @@ -6,15 +6,14 @@ NODE_MODULES_DIR = join(PACKAGE_DIR, "js", "node_modules") -GLTF_PIPELINE_FILEPATH = join(NODE_MODULES_DIR, "gltf-pipeline", "bin", "gltf-pipeline.js") -GLTFPACK_FILEPATH = join(NODE_MODULES_DIR, "gltfpack", "cli.js") +GLTF_TRANSFORM_FILEPATH = join(NODE_MODULES_DIR, "@gltf-transform", "cli", "bin", "cli.js") @compressor("draco") -def compress_gltf_pipeline(filepath: str): - run(["node", GLTF_PIPELINE_FILEPATH, "-i", filepath, "-o", filepath, "-d"], capture_output=True) +def compress_draco(filepath: str): + run([GLTF_TRANSFORM_FILEPATH, "optimize", filepath, filepath, "--compress", "draco"], capture_output=True) @compressor("meshoptimizer") -def compress_gltfpack(filepath: str): - run(["node", GLTFPACK_FILEPATH, "-i", filepath, "-o", filepath], capture_output=True) +def compress_meshoptimizer(filepath: str): + run([GLTF_TRANSFORM_FILEPATH, "optimize", filepath, filepath], capture_output=True) diff --git a/js/package.json b/js/package.json index 527f9eb..f97edf2 100644 --- a/js/package.json +++ b/js/package.json @@ -1,7 +1,6 @@ { "dependencies": { - "gltf-pipeline": "^4.1.0", - "gltfpack": "^0.21.0" + "@gltf-transform/cli": "^4.1.1" }, "scripts": { "glue-ar-export": "npm install && node glue-ar-export.js" From 66cbc6a035879e044dffaae9913dc3155d3d8fe1 Mon Sep 17 00:00:00 2001 From: Carifio24 Date: Wed, 11 Dec 2024 16:41:46 -0500 Subject: [PATCH 07/11] Implement mesh-chunking for colormaps. --- glue_ar/common/scatter_gltf.py | 151 +++++++++++++++++++++------------ 1 file changed, 99 insertions(+), 52 deletions(-) diff --git a/glue_ar/common/scatter_gltf.py b/glue_ar/common/scatter_gltf.py index e42e54d..6fbff05 100644 --- a/glue_ar/common/scatter_gltf.py +++ b/glue_ar/common/scatter_gltf.py @@ -256,12 +256,15 @@ def add_scatter_layer_gltf(builder: GLTFBuilder, color_components = hex_to_components(color) builder.add_material(color=color_components, opacity=layer_state.alpha) - triangle_offset = 0 for i, point in enumerate(data): size = radius if fixed_size else sizes[i] pts = points_getter(point, size) points.append(pts) + # If n_points is less than our designated chunk size, we only want + # to make triangles for that many points (and put everything in one mesh). + # This is both more space-efficient and necessary to be glTF spec-compliant + triangle_offset = 0 pts_count = len(points_getter((0, 0, 0), 1)) for _ in range(min(points_per_mesh, n_points)): pt_triangles = offset_triangles(triangles, triangle_offset) @@ -291,10 +294,8 @@ def add_scatter_layer_gltf(builder: GLTFBuilder, ) start = 0 - buffer = builder.buffer_count triangles_accessor = builder.accessor_count - 1 while start < n_points: - print(start, n_points) mesh_points = [pt for pts in points[start:start+points_per_mesh] for pt in pts] barr_offset = len(barr) add_points_to_bytearray(barr, mesh_points) @@ -320,13 +321,13 @@ def add_scatter_layer_gltf(builder: GLTFBuilder, # This should only happen on the final iteration # or not at all, if points_per_mesh is a divisor of count # But in this case we do need a separate accessor as the - # byte length is different + # byte length is different. + # Note that if we're on the first chunk (start == 0) + # there's no need to do this - we can use the buffer view that we just created count = n_points - start - if count < points_per_mesh: + if start != 0 and count < points_per_mesh: triangles_count = len(tris) byte_length = count * triangles_len // triangles_count - print(triangles_len, triangles_count, count) - print(byte_length) mesh_triangles = [tri for sphere in tris[:count] for tri in sphere] max_triangle_index = max(idx for tri in mesh_triangles for idx in tri) builder.add_buffer_view( @@ -355,60 +356,51 @@ def add_scatter_layer_gltf(builder: GLTFBuilder, else: points_by_color = defaultdict(list) - triangles_by_color = defaultdict(list) - triangle_offsets = defaultdict(int) color_materials = defaultdict(int) + for i, point in enumerate(data): cval = cmap_vals[i] normalized = max(min((cval - layer_state.cmap_vmin) / crange, 1), 0) cindex = int(normalized * 255) color = cmap(cindex) - material_index = color_materials.get(color, None) + + material_index = color_materials.get(cindex, None) if material_index is None: builder.add_material(color, layer_state.alpha) material_index = builder.material_count - 1 - color_materials[color] = material_index + color_materials[cindex] = material_index size = radius if fixed_size else sizes[i] pts = points_getter(point, size) - pt_triangles = offset_triangles(triangles, triangle_offsets[color]) - triangle_offsets[color] += len(pts) - points_by_color[color].append(pts) - triangles_by_color[color].append(pt_triangles) + points_by_color[cindex].append(pts) + + + + for cindex, points in points_by_color.items(): + + # If the maximum number of points in any one color is less than our designated chunk size, + # we only want to make triangles for that many points (and put everything in one mesh). + # This is both more space-efficient and necessary to be glTF spec-compliant + triangle_offset = 0 + tris = [] + pts_count = len(points_getter((0, 0, 0), 1)) + for _ in range(min(points_per_mesh, len(points))): + pt_triangles = offset_triangles(triangles, triangle_offset) + triangle_offset += pts_count + tris.append(pt_triangles) - for color, points in points_by_color.items(): - tris = triangles_by_color[color] mesh_points = [pt for pts in points for pt in pts] mesh_triangles = [tri for sphere in tris for tri in sphere] max_triangle_index = max(idx for tri in mesh_triangles for idx in tri) use_short = max_triangle_index <= SHORT_MAX - - points_offset = len(barr) - add_points_to_bytearray(barr, mesh_points) - triangles_offset = len(barr) + triangles_start = len(barr) add_triangles_to_bytearray(barr, mesh_triangles, short=use_short) - point_mins = index_mins(mesh_points) - point_maxes = index_maxes(mesh_points) + triangles_len = len(barr) - buffer = builder.buffer_count - builder.add_buffer_view( - buffer=buffer, - byte_length=triangles_offset-points_offset, - byte_offset=points_offset, - target=BufferTarget.ARRAY_BUFFER, - ) - builder.add_accessor( - buffer_view=builder.buffer_view_count-1, - component_type=ComponentType.FLOAT, - count=len(mesh_points), - type=AccessorType.VEC3, - mins=point_mins, - maxes=point_maxes, - ) builder.add_buffer_view( buffer=buffer, - byte_length=len(barr)-triangles_offset, - byte_offset=triangles_offset, + byte_length=triangles_len-triangles_start, + byte_offset=triangles_start, target=BufferTarget.ELEMENT_ARRAY_BUFFER, ) component_type = ComponentType.UNSIGNED_SHORT if use_short else ComponentType.UNSIGNED_INT @@ -418,15 +410,71 @@ def add_scatter_layer_gltf(builder: GLTFBuilder, count=len(mesh_triangles)*3, type=AccessorType.SCALAR, mins=[0], - maxes=[max(idx for tri in mesh_triangles for idx in tri)], + maxes=[max_triangle_index], ) - material = color_materials[color] - builder.add_mesh( - position_accessor=builder.accessor_count-2, - indices_accessor=builder.accessor_count-1, - material=material, - ) + start = 0 + triangles_accessor = builder.accessor_count - 1 + n_points = len(points) + while start < n_points: + mesh_points = [pt for pts in points[start:start+points_per_mesh] for pt in pts] + barr_offset = len(barr) + add_points_to_bytearray(barr, mesh_points) + point_mins = index_mins(mesh_points) + point_maxes = index_maxes(mesh_points) + + builder.add_buffer_view( + buffer=buffer, + byte_length=len(barr)-barr_offset, + byte_offset=barr_offset, + target=BufferTarget.ARRAY_BUFFER, + ) + builder.add_accessor( + buffer_view=builder.buffer_view_count-1, + component_type=ComponentType.FLOAT, + count=len(mesh_points), + type=AccessorType.VEC3, + mins=point_mins, + maxes=point_maxes, + ) + points_accessor = builder.accessor_count - 1 + + # This should only happen on the final iteration + # or not at all, if points_per_mesh is a divisor of count + # But in this case we do need a separate accessor as the + # byte length is different. + # Note that if we're on the first chunk (start == 0) + # there's no need to do this - we can use the buffer view that we just created + count = n_points - start + if start != 0 and count < points_per_mesh: + triangles_count = len(tris) + byte_length = count * triangles_len // triangles_count + mesh_triangles = [tri for sphere in tris[:count] for tri in sphere] + max_triangle_index = max(idx for tri in mesh_triangles for idx in tri) + builder.add_buffer_view( + buffer=buffer, + byte_length=byte_length, + byte_offset=triangles_start, + target=BufferTarget.ELEMENT_ARRAY_BUFFER, + ) + component_type = ComponentType.UNSIGNED_SHORT if use_short else ComponentType.UNSIGNED_INT + builder.add_accessor( + buffer_view=builder.buffer_view_count-1, + component_type=component_type, + count=len(triangles)*3*count, + type=AccessorType.SCALAR, + mins=[0], + maxes=[max_triangle_index], + ) + triangles_accessor = builder.accessor_count - 1 + + material = color_materials[cindex] + builder.add_mesh( + position_accessor=points_accessor, + indices_accessor=triangles_accessor, + material=material, + ) + start += points_per_mesh builder.add_buffer(byte_length=len(barr), uri=uri) builder.add_file_resource(uri, data=barr) @@ -484,10 +532,7 @@ def add_vispy_scatter_layer_gltf(builder: GLTFBuilder, points_getter = sphere_points_getter(theta_resolution=theta_resolution, phi_resolution=phi_resolution) - - triangles_per_point = len(triangles) - # points_per_mesh = SHORT_MAX // (triangles_per_point * 3) - points_per_mesh = 100 + points_per_mesh = None add_scatter_layer_gltf(builder=builder, viewer_state=viewer_state, @@ -511,6 +556,7 @@ def add_ipyvolume_scatter_layer_gltf(builder: GLTFBuilder, triangle_getter = IPYVOLUME_TRIANGLE_GETTERS.get(geometry, rectangular_prism_triangulation) triangles = triangle_getter() points_getter = IPYVOLUME_POINTS_GETTERS.get(geometry, box_points_getter) + points_per_mesh = None add_scatter_layer_gltf(builder=builder, viewer_state=viewer_state, @@ -518,4 +564,5 @@ def add_ipyvolume_scatter_layer_gltf(builder: GLTFBuilder, points_getter=points_getter, triangles=triangles, bounds=bounds, - clip_to_bounds=clip_to_bounds) + clip_to_bounds=clip_to_bounds, + points_per_mesh=points_per_mesh) From 2eedb94d459fa2f84ff6c62bbdb8288726866244 Mon Sep 17 00:00:00 2001 From: Carifio24 Date: Mon, 16 Dec 2024 02:55:50 -0500 Subject: [PATCH 08/11] Fix codestyle issues and a missing reference. --- glue_ar/common/scatter_gltf.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/glue_ar/common/scatter_gltf.py b/glue_ar/common/scatter_gltf.py index 6fbff05..6a5bb91 100644 --- a/glue_ar/common/scatter_gltf.py +++ b/glue_ar/common/scatter_gltf.py @@ -12,9 +12,10 @@ from glue_ar.common.scatter_export_options import ARIpyvolumeScatterExportOptions, ARVispyScatterExportOptions from glue_ar.common.shapes import cone_triangles, cone_points, cylinder_points, cylinder_triangles, \ normalize, rectangular_prism_triangulation, sphere_triangles -from glue_ar.gltf_utils import SHORT_MAX, add_points_to_bytearray, add_triangles_to_bytearray, index_mins, index_maxes, SHORT_MAX -from glue_ar.utils import Viewer3DState, get_stretches, iterable_has_nan, hex_to_components, layer_color, offset_triangles, \ - unique_id, xyz_bounds, xyz_for_layer, Bounds +from glue_ar.gltf_utils import SHORT_MAX, add_points_to_bytearray, add_triangles_to_bytearray, \ + index_mins, index_maxes +from glue_ar.utils import Viewer3DState, get_stretches, iterable_has_nan, hex_to_components, \ + layer_color, offset_triangles, unique_id, xyz_bounds, xyz_for_layer, Bounds from glue_ar.common.gltf_builder import GLTFBuilder from glue_ar.common.scatter import Scatter3DLayerState, ScatterLayerState3D, \ PointsGetter, box_points_getter, IPYVOLUME_POINTS_GETTERS, \ @@ -248,6 +249,7 @@ def add_scatter_layer_gltf(builder: GLTFBuilder, if points_per_mesh is None: points_per_mesh = n_points + first_material_index = builder.material_count if fixed_color: points = [] tris = [] @@ -327,7 +329,7 @@ def add_scatter_layer_gltf(builder: GLTFBuilder, count = n_points - start if start != 0 and count < points_per_mesh: triangles_count = len(tris) - byte_length = count * triangles_len // triangles_count + byte_length = count * triangles_len // triangles_count mesh_triangles = [tri for sphere in tris[:count] for tri in sphere] max_triangle_index = max(idx for tri in mesh_triangles for idx in tri) builder.add_buffer_view( @@ -374,8 +376,6 @@ def add_scatter_layer_gltf(builder: GLTFBuilder, pts = points_getter(point, size) points_by_color[cindex].append(pts) - - for cindex, points in points_by_color.items(): # If the maximum number of points in any one color is less than our designated chunk size, From be53b947d557c3ba834d9a08177deeba31f62988 Mon Sep 17 00:00:00 2001 From: Carifio24 Date: Mon, 16 Dec 2024 15:11:06 -0500 Subject: [PATCH 09/11] Don't change current behavior until we add a UI option. --- glue_ar/common/scatter_gltf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/glue_ar/common/scatter_gltf.py b/glue_ar/common/scatter_gltf.py index 6a5bb91..b394035 100644 --- a/glue_ar/common/scatter_gltf.py +++ b/glue_ar/common/scatter_gltf.py @@ -532,7 +532,7 @@ def add_vispy_scatter_layer_gltf(builder: GLTFBuilder, points_getter = sphere_points_getter(theta_resolution=theta_resolution, phi_resolution=phi_resolution) - points_per_mesh = None + points_per_mesh = 1 add_scatter_layer_gltf(builder=builder, viewer_state=viewer_state, @@ -556,7 +556,7 @@ def add_ipyvolume_scatter_layer_gltf(builder: GLTFBuilder, triangle_getter = IPYVOLUME_TRIANGLE_GETTERS.get(geometry, rectangular_prism_triangulation) triangles = triangle_getter() points_getter = IPYVOLUME_POINTS_GETTERS.get(geometry, box_points_getter) - points_per_mesh = None + points_per_mesh = 1 add_scatter_layer_gltf(builder=builder, viewer_state=viewer_state, From e49931f242ccb39c8d72f8e1a6a6e50b24e0efbf Mon Sep 17 00:00:00 2001 From: Carifio24 Date: Mon, 16 Dec 2024 15:15:46 -0500 Subject: [PATCH 10/11] Update libegl1 package. --- .github/workflows/ci-workflows.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-workflows.yml b/.github/workflows/ci-workflows.yml index e99fa01..6e88069 100644 --- a/.github/workflows/ci-workflows.yml +++ b/.github/workflows/ci-workflows.yml @@ -29,7 +29,7 @@ jobs: apt: - '^libxcb.*-dev' - libxkbcommon-x11-0 - - libegl1-mesa + - libegl1-mesa-dev - libhdf5-dev envs: | - linux: py39-test-all From 35d786afb00944554283bbc96bdfd2e8eb77d068 Mon Sep 17 00:00:00 2001 From: Carifio24 Date: Mon, 16 Dec 2024 16:37:57 -0500 Subject: [PATCH 11/11] Update test and test utilities to account for using shorts. --- glue_ar/common/tests/gltf_helpers.py | 7 ++++--- glue_ar/common/tests/test_scatter_gltf.py | 8 +++++--- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/glue_ar/common/tests/gltf_helpers.py b/glue_ar/common/tests/gltf_helpers.py index 85476bf..4903a12 100644 --- a/glue_ar/common/tests/gltf_helpers.py +++ b/glue_ar/common/tests/gltf_helpers.py @@ -9,7 +9,7 @@ from glue_ar.utils import iterator_count -BufferFormat = Union[Literal["f"], Literal["I"]] +BufferFormat = Union[Literal["f"], Literal["I"], Literal["H"]] def get_data(gltf: GLTF, buffer: Buffer, buffer_view: Optional[BufferView] = None) -> bytes: @@ -41,8 +41,9 @@ def count_vertices(gltf: GLTF, buffer: Buffer, buffer_view: BufferView): return count_points(gltf, buffer, buffer_view, 'f') -def count_indices(gltf: GLTF, buffer: Buffer, buffer_view: BufferView): - return count_points(gltf, buffer, buffer_view, 'I') +def count_indices(gltf: GLTF, buffer: Buffer, buffer_view: BufferView, use_short=False): + format = 'H' if use_short else 'I' + return count_points(gltf, buffer, buffer_view, format) def unpack_points(gltf: GLTF, diff --git a/glue_ar/common/tests/test_scatter_gltf.py b/glue_ar/common/tests/test_scatter_gltf.py index dabc985..d6dc4b8 100644 --- a/glue_ar/common/tests/test_scatter_gltf.py +++ b/glue_ar/common/tests/test_scatter_gltf.py @@ -10,6 +10,7 @@ from glue_ar.common.tests.gltf_helpers import count_indices, count_vertices, unpack_vertices from glue_ar.common.tests.helpers import APP_VIEWER_OPTIONS from glue_ar.common.tests.test_scatter import BaseScatterTest +from glue_ar.gltf_utils import SHORT_MAX from glue_ar.utils import export_label_for_layer, hex_to_components, layers_to_export, mask_for_bounds, \ xyz_bounds, xyz_for_layer @@ -68,8 +69,8 @@ def test_basic_export(self, app_type: str, viewer_type: str): phi_resolution=phi_resolution) points_count = sphere_points_count(theta_resolution=theta_resolution, phi_resolution=phi_resolution) - - assert count_indices(gltf, model.buffers[0], model.bufferViews[0]) == triangles_count + use_short = points_count <= SHORT_MAX + assert count_indices(gltf, model.buffers[0], model.bufferViews[0], use_short=use_short) == triangles_count assert count_vertices(gltf, model.buffers[0], model.bufferViews[1]) == points_count assert model.bufferViews[0].target == BufferTarget.ELEMENT_ARRAY_BUFFER.value @@ -77,7 +78,8 @@ def test_basic_export(self, app_type: str, viewer_type: str): indices_accessor = model.accessors[0] assert indices_accessor.bufferView == 0 - assert indices_accessor.componentType == ComponentType.UNSIGNED_INT.value + expected_indices_type = ComponentType.UNSIGNED_SHORT if use_short else ComponentType.UNSIGNED_INT + assert indices_accessor.componentType == expected_indices_type.value assert indices_accessor.count == triangles_count * 3 assert indices_accessor.type == AccessorType.SCALAR.value assert indices_accessor.min == [0]