From e188b13f325e397eed9e57bcdf3d2a2c6b59ff29 Mon Sep 17 00:00:00 2001 From: Rob Savoye Date: Fri, 13 Oct 2023 13:45:42 -0600 Subject: [PATCH] fix: Add simple pages for all the utilities, add doc on the Overture data files --- docs/geofabrik.md | 10 +++++ docs/overture.md | 101 ++++++++++++++++++++++++++++++++++++---------- docs/postgres.md | 17 ++++++++ mkdocs.yml | 3 ++ 4 files changed, 109 insertions(+), 22 deletions(-) create mode 100644 docs/geofabrik.md create mode 100644 docs/postgres.md diff --git a/docs/geofabrik.md b/docs/geofabrik.md new file mode 100644 index 0000000..622ad65 --- /dev/null +++ b/docs/geofabrik.md @@ -0,0 +1,10 @@ +# Geofabrik + +This is a simple utility to download country data files from +[GeoFabrik](https://download.geofabrik.de/). + + options: + --help(-h) show this help message and exit + --verbose(-v) verbose output + --file(-f) FILE The country or US state to download + --list(-l) List all files on GeoFabrik diff --git a/docs/overture.md b/docs/overture.md index 7b1831c..af03da4 100644 --- a/docs/overture.md +++ b/docs/overture.md @@ -1,22 +1,79 @@ -{"id": -"tmp_7732383330373539373640313132392E3236393938333635363337383036", -"updatetime": "2014-05-20T08:11:54.000Z", "version": 0, "names": [], -"level": null, "height": null, "numfloors": null, "class": null, -"sources": [[["dataset", "OpenStreetMap"], ["property", ""], -["recordId", "w283075976@1"]]], "bbox": {"minx": 2.6939908, "maxx": -2.694063, "miny": 47.9845676, "maxy": 47.9846009}, "geometry": - -- Austin Building Footprints Year 2013 2D Buildings -- Boston BPDA 3D Buildings -- City of Cambridge, MA Open Data 3D Buildings -- Denver Regional Council of Governments 2D Buildings -- Esri Buildings | Austin Building Footprints Year 2013 2D Buildings -- Esri Buildings | Denver Regional Council of Governments 2D Buildings -- Esri Community Maps -- Miami-Dade County Open Data 3D Buildings -- OpenStreetMap -- Microsoft ML Buildings -- NYC Open Data 3D Buildings -- Portland Building Footprint 2D Buildings -- USGS Lidar -- Washington DC Open Data 3D Buildings +# Overture Map Data + +The Overture Foundation (https://www.overturemaps.org) has been +recently formed to build a competitor to Google Maps. The plan is to +use OpenStreetMap (OSM) data as a baselayer, and layer other datasets +on top. The currently available data (July 2023) has 13 different +datasets in addition to the OSM data. It is [available +here](https://overturemaps.org/download/). It also includes a snapshot +of OSM data from the same time frame. Other than the OSM data and [MS +Footprints](https://github.com/microsoft/GlobalMLBuildingFootprints), +all the current additional data is US specific, and often contains +multiple copies of the same dataset, but from different organization. + +The Overture files are in [Parquet](https://parquet.apache.org/) +format, which uses [Arrow](https://arrow.apache.org/) to store the +data in a tabular fashion. The files are large, about 1.2G most of the +time. Each file has features spread across the planet, instead of a +subset in a geographical region. If you wish to get all the data for a +region, you have to load all 120 files into a database. + +## Schema + +The schema used in the Overture data files is [documented here]( +https://docs.overturemaps.org/reference). This document is just a +summary with some implementation details. + +### Buildings + +* id: tmp_[Giant HEX number] +* updatetime: The last time a feature was updated +* version: The version of the feature +* names: The names of the buiding +* height: The heigth of the feature in meters +* numfloors: The numbers of floors in the building +* class: The type of building, residential, commericial, etc... +* geometry: The feature geometry +* sources: A list of dataset sources with optional recordId +* level: This appears to be unused +* bbox: A bounding box of the feature + +The current list of buildings datasets is: + +* Austin Building Footprints Year 2013 2D Buildings +* Boston BPDA 3D Buildings +* City of Cambridge, MA Open Data 3D Buildings +* Denver Regional Council of Governments 2D Buildings +* Esri Buildings | Austin Building Footprints Year 2013 2D Buildings +* Esri Buildings | Denver Regional Council of Governments 2D Buildings +* Esri Community Maps +* Miami-Dade County Open Data 3D Buildings +* OpenStreetMap +* Microsoft ML Buildings +* NYC Open Data 3D Buildings +* Portland Building Footprint 2D Buildings +* USGS Lidar +* Washington DC Open Data 3D Buildings + +# Special Columns + +## names + +The names column can have 4 variations on the name. Each may also have +a language value as well. + +* common +* official +* alternate +* ahort + +## sources + +The sources column is an array of with two entries. The first entry is +the name of the dataset, and where it exists, a *recordID* to +reference the source dataset. For OSM data, the recordID has 3 +sub-fields. The first character is the type, *w* (way), *n* (node), or +*l* (line). The second is the OSM ID, and the third with a *v* is the +version of the feature in OSM. + +For example: *w***123456**v2 is a way with ID 123456 and is version 2. diff --git a/docs/postgres.md b/docs/postgres.md new file mode 100644 index 0000000..63e7c45 --- /dev/null +++ b/docs/postgres.md @@ -0,0 +1,17 @@ +# Postgres + +This is a simple command line interface that uses the contained +classes to make a data extract from OSM. This program extracts data +from a local postgres data, or the remote Underpass one. A boundary +polygon is used to define the area to be covered in the +extract. Optionally a data file can be used. + + options: + --help(-h) show this help message and exit + --verbose(-v) verbose output + --uri(-u) URI Database URI + --boundary(-b) BOUNDARY Boundary polygon to limit the data size + --sql(-s) SQL Custom SQL query to execute against the database + --all(-a) ALL All the geometry or just centroids + --config(-c) CONFIG The config file for the query (json or yaml) + --outfile(-o) OUTFILE The output file diff --git a/mkdocs.yml b/mkdocs.yml index 3a1c173..641792d 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -60,8 +60,11 @@ nav: - About: about.md - Utilities: - Importer: importer.md + - Postgres: postgres.md + - Geofabrik: geofabrik.md - File Formats: - JSON: json.md - YAML: yaml.md + - Overture: overture.md - API: api.md - License: LICENSE.md