From 0ee4cee883718678c7fc938a84fe274212830b0e Mon Sep 17 00:00:00 2001 From: Chad Burt Date: Mon, 14 Oct 2024 07:54:33 -0700 Subject: [PATCH] WIP --- .../h3-filter-ingest/build-cell-pmtiles.ts | 111 ++++++++++++++++++ packages/h3-filter-ingest/src/server.ts | 28 +++-- packages/h3-filter-ingest/src/stops.ts | 21 ++++ 3 files changed, 151 insertions(+), 9 deletions(-) create mode 100644 packages/h3-filter-ingest/build-cell-pmtiles.ts create mode 100644 packages/h3-filter-ingest/src/stops.ts diff --git a/packages/h3-filter-ingest/build-cell-pmtiles.ts b/packages/h3-filter-ingest/build-cell-pmtiles.ts new file mode 100644 index 00000000..33a177fc --- /dev/null +++ b/packages/h3-filter-ingest/build-cell-pmtiles.ts @@ -0,0 +1,111 @@ +// Takes an input cell csv file and downsamples it to multiple h3resolutions based on the Stop configuration, ultimately creating a pmtiles archive that can be used to base filtering visualizations on. +import * as h3 from "h3-js"; +import * as gdal from "gdal-async"; +import { createReadStream, readdirSync, readFileSync } from "node:fs"; +import * as cliProgress from "cli-progress"; +import { stops } from "./src/stops"; +import { execSync } from "node:child_process"; + +const MIN_ZOOM = 0; + +const usage = ` +npx ts-node build-cell-pmtiles.ts +`; + +const filePath = process.argv[2]; +if (!filePath) { + console.error("Missing path to cells csv"); + console.error(usage); + process.exit(1); +} + +const outputPath = process.argv[3]; +if (!outputPath) { + console.error("Missing path to output pmtiles"); + console.error(usage); + process.exit(1); +} + +const MIN_RESOLUTION = 6; + +(async () => { + // First, for each stop build a flatgeobuf file + const cells = new Set(); + for (const stop of stops) { + const parents = new Set(); + const isFirstStop = stops.indexOf(stop) === 0; + const isLastStop = stops.indexOf(stop) === stops.length - 1; + if (isFirstStop) { + console.log("First stop, reading cells from input file"); + // read the input file line-by-line. There are no columns so we + // don't need a csv parser + const stream = createReadStream(filePath); + for await (const line of stream) { + const ids = line + .toString() + .trim() + .split("\n") + .map((id: string) => id.trim()); + for (const id of ids) { + cells.add(id); + } + } + console.log( + `Starting processing with ${cells.size.toLocaleString()} cells` + ); + } + const driver = gdal.drivers.get("FlatGeobuf"); + const ds = driver.create(`output/cells-${stop.h3Resolution}.fgb`); + const layer = ds.layers.create( + "cells", + gdal.SpatialReference.fromEPSG(4326), + gdal.wkbPolygon + ); + layer.fields.add(new gdal.FieldDefn("id", gdal.OFTString)); + layer.fields.add(new gdal.FieldDefn("parent_id", gdal.OFTString)); + layer.fields.add(new gdal.FieldDefn("grandparent_id", gdal.OFTString)); + const progressBar = new cliProgress.SingleBar( + { + format: `cells-${stop.h3Resolution}.fgb | {bar} | {percentage}% | {eta}s || {value}/{total} cells processed`, + }, + cliProgress.Presets.shades_classic + ); + progressBar.start(cells.size, 0); + cells.forEach((cell) => { + const id = cell; + const parent_id = h3.cellToParent(id, stop.h3Resolution - 1); + const grandparent_id = h3.cellToParent(parent_id, stop.h3Resolution - 2); + const multipolygon = h3.cellsToMultiPolygon([id], true); + const feature = new gdal.Feature(layer); + feature.setGeometry( + gdal.Geometry.fromGeoJson({ + type: "Polygon", + coordinates: multipolygon[0], + }) + ); + feature.fields.set("id", id); + feature.fields.set("parent_id", parent_id); + parents.add(parent_id); + feature.fields.set("grandparent_id", grandparent_id); + layer.features.add(feature); + progressBar.increment(); + }); + progressBar.stop(); + cells.clear(); + parents.forEach((parent) => cells.add(parent)); + console.log(`create tiles for r${stop.h3Resolution}, z${stop.zoomLevel}`); + const maxZoom = stop.zoomLevel; + let minZoom = maxZoom; + const nextStop = stops[stops.indexOf(stop) + 1]; + if (nextStop) { + minZoom = nextStop.zoomLevel + 1; + } else { + minZoom = MIN_ZOOM; + } + const resolution = stop.h3Resolution; + + execSync( + `tippecanoe --force -l cells -z ${maxZoom} -Z ${minZoom} -o output/cells-${resolution}-z${minZoom}-z${maxZoom}.pmtiles output/cells-${resolution}.fgb` + ); + } +})(); diff --git a/packages/h3-filter-ingest/src/server.ts b/packages/h3-filter-ingest/src/server.ts index 3dae1dcf..10d5f7e6 100644 --- a/packages/h3-filter-ingest/src/server.ts +++ b/packages/h3-filter-ingest/src/server.ts @@ -44,7 +44,8 @@ const server = createServer(async (req, res) => { console.time(`${z}/${x}/${y}.${ext}`); if (ext === "txt") { try { - const data = await getText(z, x, y); + console.log("filters", filters); + const data = await getText(z, x, y, filters || {}); console.timeEnd(`${z}/${x}/${y}.${ext}`); // add cors headers to allow all origins res.setHeader("Access-Control-Allow-Origin", "*"); @@ -89,7 +90,6 @@ async function getMVT( ) { const resolution = getResolutionForZoom(z); const f = buildWhereClauses(filters || {}); - console.log(filters, f); const q = ` with mvtgeom as (select @@ -127,19 +127,31 @@ async function getMVT( } } -async function getText(z: number, x: number, y: number) { +async function getText( + z: number, + x: number, + y: number, + filters?: { [column: string]: Filter } +) { const resolution = getResolutionForZoom(z); + const f = buildWhereClauses(filters || {}); + console.log(f); const data = await pool.query({ - name: "mvt-cell-ids-r" + resolution, + name: + "mvt-text-r" + + resolution + + createHash("md5").update(JSON.stringify(filters)).digest("hex"), text: ` select - distinct(r${resolution}_id) as id + distinct(${resolution === 11 ? "id" : `r${resolution}_id`}) as id from cells where - ST_INTERSECTS(geom, ST_TileEnvelope($1,$2,$3)) + ST_INTERSECTS(geom, ST_TileEnvelope($1,$2,$3)) ${ + f.values.length > 0 ? "AND " + f.where : "" + } `, - values: [z, x, y], + values: [z, x, y, ...f.values], }); if (data.rows.length === 0) { return ""; @@ -165,8 +177,6 @@ const stops: Stop[] = [ // { h3Resolution: 5, zoomLevel: 5 }, ].sort((a, b) => a.zoomLevel - b.zoomLevel); -console.log(stops); - function getResolutionForZoom(zoom: number) { const idx = stops.findIndex((stop) => stop.zoomLevel > zoom); if (idx === -1) { diff --git a/packages/h3-filter-ingest/src/stops.ts b/packages/h3-filter-ingest/src/stops.ts new file mode 100644 index 00000000..def5edad --- /dev/null +++ b/packages/h3-filter-ingest/src/stops.ts @@ -0,0 +1,21 @@ +export type Stop = { + h3Resolution: number; + zoomLevel: number; +}; + +/** + * These stops represent the zoom levels at which each h3 resolution should be + * displayed. The algorithm will fill in the gaps, starting at the highest zoom + * level and working its way down to MIN_ZOOM. + */ +export const stops: Stop[] = [ + { h3Resolution: 11, zoomLevel: 14 }, + { h3Resolution: 10, zoomLevel: 13 }, + { h3Resolution: 9, zoomLevel: 12 }, + { h3Resolution: 9, zoomLevel: 11 }, + { h3Resolution: 8, zoomLevel: 10 }, + // { h3Resolution: 7, zoomLevel: 8 }, + { h3Resolution: 7, zoomLevel: 8 }, + { h3Resolution: 6, zoomLevel: 6 }, + // { h3Resolution: 5, zoomLevel: 5 }, +].sort((a, b) => b.zoomLevel - a.zoomLevel);