WIP

packages/h3-filter-ingest/build-cell-pmtiles.ts

@@ -0,0 +1,111 @@
+// Takes an input cell csv file and downsamples it to multiple h3resolutions based on the Stop configuration, ultimately creating a pmtiles archive that can be used to base filtering visualizations on.
+import * as h3 from "h3-js";
+import * as gdal from "gdal-async";
+import { createReadStream, readdirSync, readFileSync } from "node:fs";
+import * as cliProgress from "cli-progress";
+import { stops } from "./src/stops";
+import { execSync } from "node:child_process";
+
+const MIN_ZOOM = 0;
+
+const usage = `
+npx ts-node build-cell-pmtiles.ts <path-to-cells.csv> <path-to-output.pmtiles>
+`;
+
+const filePath = process.argv[2];
+if (!filePath) {
+  console.error("Missing path to cells csv");
+  console.error(usage);
+  process.exit(1);
+}
+
+const outputPath = process.argv[3];
+if (!outputPath) {
+  console.error("Missing path to output pmtiles");
+  console.error(usage);
+  process.exit(1);
+}
+
+const MIN_RESOLUTION = 6;
+
+(async () => {
+  // First, for each stop build a flatgeobuf file
+  const cells = new Set<string>();
+  for (const stop of stops) {
+    const parents = new Set<string>();
+    const isFirstStop = stops.indexOf(stop) === 0;
+    const isLastStop = stops.indexOf(stop) === stops.length - 1;
+    if (isFirstStop) {
+      console.log("First stop, reading cells from input file");
+      // read the input file line-by-line. There are no columns so we
+      // don't need a csv parser
+      const stream = createReadStream(filePath);
+      for await (const line of stream) {
+        const ids = line
+          .toString()
+          .trim()
+          .split("\n")
+          .map((id: string) => id.trim());
+        for (const id of ids) {
+          cells.add(id);
+        }
+      }
+      console.log(
+        `Starting processing with ${cells.size.toLocaleString()} cells`
+      );
+    }
+    const driver = gdal.drivers.get("FlatGeobuf");
+    const ds = driver.create(`output/cells-${stop.h3Resolution}.fgb`);
+    const layer = ds.layers.create(
+      "cells",
+      gdal.SpatialReference.fromEPSG(4326),
+      gdal.wkbPolygon
+    );
+    layer.fields.add(new gdal.FieldDefn("id", gdal.OFTString));
+    layer.fields.add(new gdal.FieldDefn("parent_id", gdal.OFTString));
+    layer.fields.add(new gdal.FieldDefn("grandparent_id", gdal.OFTString));
+    const progressBar = new cliProgress.SingleBar(
+      {
+        format: `cells-${stop.h3Resolution}.fgb | {bar} | {percentage}% | {eta}s || {value}/{total} cells processed`,
+      },
+      cliProgress.Presets.shades_classic
+    );
+    progressBar.start(cells.size, 0);
+    cells.forEach((cell) => {
+      const id = cell;
+      const parent_id = h3.cellToParent(id, stop.h3Resolution - 1);
+      const grandparent_id = h3.cellToParent(parent_id, stop.h3Resolution - 2);
+      const multipolygon = h3.cellsToMultiPolygon([id], true);
+      const feature = new gdal.Feature(layer);
+      feature.setGeometry(
+        gdal.Geometry.fromGeoJson({
+          type: "Polygon",
+          coordinates: multipolygon[0],
+        })
+      );
+      feature.fields.set("id", id);
+      feature.fields.set("parent_id", parent_id);
+      parents.add(parent_id);
+      feature.fields.set("grandparent_id", grandparent_id);
+      layer.features.add(feature);
+      progressBar.increment();
+    });
+    progressBar.stop();
+    cells.clear();
+    parents.forEach((parent) => cells.add(parent));
+    console.log(`create tiles for r${stop.h3Resolution}, z${stop.zoomLevel}`);
+    const maxZoom = stop.zoomLevel;
+    let minZoom = maxZoom;
+    const nextStop = stops[stops.indexOf(stop) + 1];
+    if (nextStop) {
+      minZoom = nextStop.zoomLevel + 1;
+    } else {
+      minZoom = MIN_ZOOM;
+    }
+    const resolution = stop.h3Resolution;
+
+    execSync(
+      `tippecanoe --force -l cells -z ${maxZoom} -Z ${minZoom} -o output/cells-${resolution}-z${minZoom}-z${maxZoom}.pmtiles output/cells-${resolution}.fgb`
+    );
+  }
+})();

packages/h3-filter-ingest/src/server.ts

@@ -44,7 +44,8 @@ const server = createServer(async (req, res) => {
     console.time(`${z}/${x}/${y}.${ext}`);
     if (ext === "txt") {
       try {
-        const data = await getText(z, x, y);
+        console.log("filters", filters);
+        const data = await getText(z, x, y, filters || {});
         console.timeEnd(`${z}/${x}/${y}.${ext}`);
         // add cors headers to allow all origins
         res.setHeader("Access-Control-Allow-Origin", "*");
@@ -89,7 +90,6 @@ async function getMVT(
 ) {
   const resolution = getResolutionForZoom(z);
   const f = buildWhereClauses(filters || {});
-  console.log(filters, f);
   const q = `
   with mvtgeom as 
   (select 
@@ -127,19 +127,31 @@ async function getMVT(
   }
 }
 
-async function getText(z: number, x: number, y: number) {
+async function getText(
+  z: number,
+  x: number,
+  y: number,
+  filters?: { [column: string]: Filter }
+) {
   const resolution = getResolutionForZoom(z);
+  const f = buildWhereClauses(filters || {});
+  console.log(f);
   const data = await pool.query({
-    name: "mvt-cell-ids-r" + resolution,
+    name:
+      "mvt-text-r" +
+      resolution +
+      createHash("md5").update(JSON.stringify(filters)).digest("hex"),
     text: `
       select 
-        distinct(r${resolution}_id) as id 
+        distinct(${resolution === 11 ? "id" : `r${resolution}_id`}) as id
       from 
         cells
       where 
-        ST_INTERSECTS(geom, ST_TileEnvelope($1,$2,$3))
+        ST_INTERSECTS(geom, ST_TileEnvelope($1,$2,$3)) ${
+          f.values.length > 0 ? "AND " + f.where : ""
+        }
   `,
-    values: [z, x, y],
+    values: [z, x, y, ...f.values],
   });
   if (data.rows.length === 0) {
     return "";
@@ -165,8 +177,6 @@ const stops: Stop[] = [
   // { h3Resolution: 5, zoomLevel: 5 },
 ].sort((a, b) => a.zoomLevel - b.zoomLevel);
 
-console.log(stops);
-
 function getResolutionForZoom(zoom: number) {
   const idx = stops.findIndex((stop) => stop.zoomLevel > zoom);
   if (idx === -1) {

packages/h3-filter-ingest/src/stops.ts

@@ -0,0 +1,21 @@
+export type Stop = {
+  h3Resolution: number;
+  zoomLevel: number;
+};
+
+/**
+ * These stops represent the zoom levels at which each h3 resolution should be
+ * displayed. The algorithm will fill in the gaps, starting at the highest zoom
+ * level and working its way down to MIN_ZOOM.
+ */
+export const stops: Stop[] = [
+  { h3Resolution: 11, zoomLevel: 14 },
+  { h3Resolution: 10, zoomLevel: 13 },
+  { h3Resolution: 9, zoomLevel: 12 },
+  { h3Resolution: 9, zoomLevel: 11 },
+  { h3Resolution: 8, zoomLevel: 10 },
+  // { h3Resolution: 7, zoomLevel: 8 },
+  { h3Resolution: 7, zoomLevel: 8 },
+  { h3Resolution: 6, zoomLevel: 6 },
+  // { h3Resolution: 5, zoomLevel: 5 },
+].sort((a, b) => b.zoomLevel - a.zoomLevel);