Annotation workflow

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "examples/data_streaming/dependencies/gaussian-splatting"]
+	path = examples/data_streaming/dependencies/gaussian-splatting
+	url = https://github.com/graphdeco-inria/gaussian-splatting.git

README.md

@@ -1,4 +1,4 @@
-# SuperSplat - 3D Gaussian Splat Editor
+# Fork of: SuperSplat - 3D Gaussian Splat Editor
 
 SuperSplat is a free and open source tool for inspecting and editing 3D Gaussian Splats. It is built on web technologies and runs in the browser, so there's nothing to download or install.
 

TODO.txt

@@ -0,0 +1,87 @@
+1. Hook (API) for loading Gaussian splat scene
+  1.1 One-time load
+  1.2 Stretch goal - stream gaussians live (visualize training)
+2. Additional gaussian metadata (classification)
+  2.1 additional parameter for each gaussian (class color)
+  2.2 Option to toggle visualization of metadata:
+   2.2.1 Visualize different classes by Setting a unique color (color picker?) for each class
+   2.2.2 Select all gaussians corresponding to that class (dropdown list with cheeckboxes)
+3. Bounding box visualization
+  3.1 wireframe rendering
+  3.2 minmaxx,minmaxy coordinate calculation from selected gaussians
+      playcanvas.d.ts
+      calcAabb(result: BoundingBox, pred?: (i: number) => boolean): boolean;
+    /**
+     * Calculate exact scene aabb taking into account splat size
+     *
+     * @param {BoundingBox} result - Where to store the resulting bounding box.
+     * @param {(i: number) => boolean} [pred] - Optional predicate function to filter splats.
+     * @returns {boolean} - Whether the calculation was successful.
+     */
+
+```
+     GSplatInstance -> GSplat -> GSplatData ->:
+    this.aabb = new BoundingBox();
+    gsplatData.calcAabb(this.aabb);
+```
+
+
+1. One time Load:
+GUI:
+ Scene:
+   Add "Start Data Server"
+   menu.ts -> sceneMenuPanel
+
+   file-handler:
+   scene.open -> const file = await handle.getFile();
+                    const url = URL.createObjectURL(file);
+                    await scene.loadModel(url, file.name);
+                    URL.revokeObjectURL(url);
+
+              const model = await this.assetLoader.loadModel({ url, filename });
+            this.add(model);
+
+
+
+ asset.on('load', () => {
+                    stopSpinner();
+
+                    // support loading 2d splats by adding scale_2 property with almost 0 scale
+                    const splatData = asset.resource.splatData;
+                    if (splatData.getProp('scale_0') && splatData.getProp('scale_1') && !splatData.getProp('scale_2')) {
+                        const scale2 = new Float32Array(splatData.numSplats).fill(Math.log(1e-6));
+                        splatData.addProp('scale_2', scale2);
+
+                        // place the new scale_2 property just after scale_1
+                        const props = splatData.getElement('vertex').properties;
+                        props.splice(props.findIndex((prop: any) => prop.name === 'scale_1') + 1, 0, props.splice(props.length - 1, 1)[0]);
+                    }
+
+                    // check the PLY contains minimal set of we expect
+                    const required = [
+                        'x', 'y', 'z',
+                        'scale_0', 'scale_1', 'scale_2',
+                        'rot_0', 'rot_1', 'rot_2', 'rot_3',
+                        'f_dc_0', 'f_dc_1', 'f_dc_2', 'opacity'
+                    ];
+                    const missing = required.filter(x => !splatData.getProp(x));
+                    if (missing.length > 0) {
+                        reject(`This file does not contain gaussian splatting data. The following properties are missing: ${missing.join(', ')}`);
+                    } else {
+                        resolve(new Splat(asset));
+                    }
+                });
+
+
+
+# Update value of splat state:
+a = scene.elements[3].splatData.getProp('state')
+a.fill(1) (Or State enumerator values)
+
+# Update scene with new state values, 1 signifies changed state
+scene.elements[3].updateState(1)
+
+
+Wants:
+1. Show bounding box for selection
+Linearly separate the splats based on features (expand in z coordinate for example)

examples/data_streaming/.gitignore

@@ -0,0 +1 @@
+__pycache__

examples/data_streaming/README.md

@@ -0,0 +1,22 @@
+# SuperSplat Annotation and Hot-Reload Demo
+
+## Installation:
+Python version supporting gaussian-splatting: 3.7.16
+We use conda to manage our dependencies
+
+```
+conda create -n SuperSplatDemo python=3.7.16
+conda activate SuperSplatDemo
+```
+
+Fetch and initialize all submodules (may require installing cuda libraries for gaussian-splatting)- This will probably fail, but will install enough libraries to run the demo!
+```
+cd ./examples/data_streaming
+git submodule update --init --recursive
+
+conda env update --name SuperSplatDemo --file ./dependencies/gaussian-splatting/environment.yml
+```
+
+## Usage:
+Run demo through Splat_annotation.ipynb notebook.
+Additionally, you will need to run the dataserver in external process, build and run this version of SuperSplat.

examples/data_streaming/Splat_annotation.ipynb

@@ -0,0 +1,181 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Install dependecies\n",
+    "%pip install pillow numpy opencv-python scikit-learn websockets flask"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "import sys\n",
+    "import numpy as np\n",
+    "\n",
+    "# Import external library dependencies\n",
+    "module_paths = [\n",
+    "    os.path.abspath(os.path.join('./dependencies/gaussian-splatting')),\n",
+    "]\n",
+    "for module_path in module_paths:\n",
+    "    if module_path not in sys.path:\n",
+    "        sys.path.append(module_path)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Running websocket proxy server:\n",
+    "```\n",
+    "conda activate SuperSplatDemo\n",
+    "cd .\\examples\\data_streaming\n",
+    "python .\\dataserver.py\n",
+    "```\n",
+    "\n",
+    "### Join websocket in SuperSplat web:\n",
+    "`Click Scene -> Start Data Server`"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from annotatedmodel import AnnotatedModel, DatasetParams\n",
+    "\n",
+    "# backCover\n",
+    "demo_dataset_params = DatasetParams(\".\\\\demo\\\\fruit_drops_minimal.ply\",\n",
+    "                                         \".\\\\demo\\\\fruit_drops_minimal_out.ply\",\n",
+    "                                         \".\\\\demo\\\\fruit_drops_minimal.json\",\n",
+    "                                         \".\\\\demo\\\\fruit_drops_minimal_out.json\"\n",
+    "                                         )\n",
+    "\n",
+    "demo = AnnotatedModel(demo_dataset_params.sh_degree)\n",
+    "demo.load_ply(demo_dataset_params.ply_source)\n",
+    "\n",
+    "# Load existing labels with format: https://docs.segments.ai/reference/label-types#segmentation-label \n",
+    "demo.load_labels(demo_dataset_params.label_source)\n",
+    "\n",
+    "# Initialize new labels\n",
+    "#demo.initialize_empty_labels()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Reupload model\n",
+    "def reupload_model(model:AnnotatedModel):\n",
+    "    model.upload_ply()\n",
+    "    model.upload_labels()\n",
+    "\n",
+    "reupload_model(demo)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[3 0 0 ... 2 2 1]\n"
+     ]
+    }
+   ],
+   "source": [
+    "# K-Means\n",
+    "from sklearn.cluster import KMeans\n",
+    "\n",
+    "def kmeans(model:AnnotatedModel, k=5):\n",
+    "    X = model._xyz.cpu().detach()\n",
+    "    clustering = KMeans(n_clusters=k).fit(X)\n",
+    "    labels = clustering.labels_\n",
+    "    print(labels)\n",
+    "    model.load_np_labels(labels)\n",
+    "    model.labelData.generate_n_categories(k-1)\n",
+    "    model.upload_labels()\n",
+    "\n",
+    "kmeans(demo, 5)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Found K: 9\n",
+      "[0 0 0 ... 0 0 0]\n"
+     ]
+    }
+   ],
+   "source": [
+    "# DBSCAN\n",
+    "from sklearn.cluster import DBSCAN\n",
+    "\n",
+    "def dbscan(model:AnnotatedModel, eps=0.02, min_samples=5):\n",
+    "    X = model._xyz.cpu().detach()\n",
+    "    clustering = DBSCAN(eps=eps, min_samples=min_samples).fit(X)\n",
+    "    labels = clustering.labels_\n",
+    "    # DBSCAN uses -1 for non-clustered points\n",
+    "    labels = labels + 1\n",
+    "    k = len(np.unique(labels))\n",
+    "    print(f\"Found K: {k}\")\n",
+    "    print(labels)\n",
+    "    model.load_np_labels(labels)\n",
+    "    model.labelData.generate_n_categories(k)\n",
+    "    model.upload_labels()\n",
+    "\n",
+    "dbscan(demo, 0.015, 5)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Export labels in new annotations file\n",
+    "demo.save_labels(demo_dataset_params.label_dest)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "SuperSplatDemo",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.7.12"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}