Once features can't possibly fit in a tile, stop trying (#9)

* Stop adding features to a tile if it can't possibly work

* Add --integer and --fraction options to tippecanoe-decode

* Carry the strategies field from tileset metadata through tile-join

* Update changelog

* Assign different codes to different kinds of error exits

Author: e-n-f

CHANGELOG.md

@@ -1,6 +1,12 @@
+## 2.6.2
+
+* Stop adding features to a tile if it can't possibly work, to limit memory use
+* Add --integer and --fraction options to tippecanoe-decode
+* Carry `strategies` field from tileset metadata through tile-join
+
 ## 2.6.1
 
-Upgrade protozero to version 1.7.1
+* Upgrade protozero to version 1.7.1
 
 ## 2.6.0
 

Makefile

@@ -160,10 +160,14 @@ decode-test:
 	mkdir -p tests/muni/decode
 	./tippecanoe -q -z11 -Z11 -f -o tests/muni/decode/multi.mbtiles tests/muni/*.json
 	./tippecanoe-decode -x generator -l subway tests/muni/decode/multi.mbtiles > tests/muni/decode/multi.mbtiles.json.check
+	./tippecanoe-decode -x generator -l subway --integer tests/muni/decode/multi.mbtiles > tests/muni/decode/multi.mbtiles.integer.json.check
+	./tippecanoe-decode -x generator -l subway --fraction tests/muni/decode/multi.mbtiles > tests/muni/decode/multi.mbtiles.fraction.json.check
 	./tippecanoe-decode -x generator -c tests/muni/decode/multi.mbtiles > tests/muni/decode/multi.mbtiles.pipeline.json.check
 	./tippecanoe-decode -x generator tests/muni/decode/multi.mbtiles 11 327 791 > tests/muni/decode/multi.mbtiles.onetile.json.check
 	./tippecanoe-decode -x generator --stats tests/muni/decode/multi.mbtiles > tests/muni/decode/multi.mbtiles.stats.json.check
 	cmp tests/muni/decode/multi.mbtiles.json.check tests/muni/decode/multi.mbtiles.json
+	cmp tests/muni/decode/multi.mbtiles.integer.json.check tests/muni/decode/multi.mbtiles.integer.json
+	cmp tests/muni/decode/multi.mbtiles.fraction.json.check tests/muni/decode/multi.mbtiles.fraction.json
 	cmp tests/muni/decode/multi.mbtiles.pipeline.json.check tests/muni/decode/multi.mbtiles.pipeline.json
 	cmp tests/muni/decode/multi.mbtiles.onetile.json.check tests/muni/decode/multi.mbtiles.onetile.json
 	cmp tests/muni/decode/multi.mbtiles.stats.json.check tests/muni/decode/multi.mbtiles.stats.json

README.md

@@ -880,6 +880,8 @@ resolutions.
  * `-c` or `--tag-layer-and-zoom`: Include each feature's layer and zoom level as part of its `tippecanoe` object rather than as a FeatureCollection wrapper
  * `-S` or `--stats`: Just report statistics about each tile's size and the number of features in it, as a JSON structure.
  * `-f` or `--force`: Decode tiles even if polygon ring order or closure problems are detected
+ * `-I` or `--integer`: Report coordinates in integer tile coordinates
+ * `-F` or `--fraction`: Report coordinates as a fraction of the tile extent
 
 tippecanoe-json-tool
 ====================

csv.cpp

@@ -1,5 +1,6 @@
 #include "csv.hpp"
 #include "text.hpp"
+#include "errors.hpp"
 
 std::vector<std::string> csv_split(const char *s) {
 	std::vector<std::string> ret;
@@ -68,15 +69,15 @@ void readcsv(const char *fn, std::vector<std::string> &header, std::map<std::str
 	FILE *f = fopen(fn, "r");
 	if (f == NULL) {
 		perror(fn);
-		exit(EXIT_FAILURE);
+		exit(EXIT_OPEN);
 	}
 
 	std::string s;
 	if ((s = csv_getline(f)).size() > 0) {
 		std::string err = check_utf8(s);
 		if (err != "") {
 			fprintf(stderr, "%s: %s\n", fn, err.c_str());
-			exit(EXIT_FAILURE);
+			exit(EXIT_UTF8);
 		}
 
 		header = csv_split(s.c_str());
@@ -89,7 +90,7 @@ void readcsv(const char *fn, std::vector<std::string> &header, std::map<std::str
 		std::string err = check_utf8(s);
 		if (err != "") {
 			fprintf(stderr, "%s: %s\n", fn, err.c_str());
-			exit(EXIT_FAILURE);
+			exit(EXIT_UTF8);
 		}
 
 		std::vector<std::string> line = csv_split(s.c_str());
@@ -105,7 +106,7 @@ void readcsv(const char *fn, std::vector<std::string> &header, std::map<std::str
 
 	if (fclose(f) != 0) {
 		perror("fclose");
-		exit(EXIT_FAILURE);
+		exit(EXIT_CLOSE);
 	}
 }
 

decode.cpp

@@ -22,6 +22,7 @@
 #include "write_json.hpp"
 #include "jsonpull/jsonpull.h"
 #include "dirtiles.hpp"
+#include "errors.hpp"
 
 int minzoom = 0;
 int maxzoom = 32;
@@ -85,18 +86,18 @@ void do_stats(mvt_tile &tile, size_t size, bool compressed, int z, unsigned x, u
 	state.json_write_newline();
 }
 
-void handle(std::string message, int z, unsigned x, unsigned y, std::set<std::string> const &to_decode, bool pipeline, bool stats, json_writer &state) {
+void handle(std::string message, int z, unsigned x, unsigned y, std::set<std::string> const &to_decode, bool pipeline, bool stats, json_writer &state, int coordinate_mode) {
 	mvt_tile tile;
 	bool was_compressed;
 
 	try {
 		if (!tile.decode(message, was_compressed)) {
 			fprintf(stderr, "Couldn't parse tile %d/%u/%u\n", z, x, y);
-			exit(EXIT_FAILURE);
+			exit(EXIT_MVT);
 		}
 	} catch (std::exception const &e) {
 		fprintf(stderr, "PBF decoding error in tile %d/%u/%u\n", z, x, y);
-		exit(EXIT_FAILURE);
+		exit(EXIT_PROTOBUF);
 	}
 
 	if (stats) {
@@ -159,7 +160,7 @@ void handle(std::string message, int z, unsigned x, unsigned y, std::set<std::st
 
 		if (layer.extent <= 0) {
 			fprintf(stderr, "Impossible layer extent %lld in mbtiles\n", layer.extent);
-			exit(EXIT_FAILURE);
+			exit(EXIT_IMPOSSIBLE);
 		}
 
 		if (to_decode.size() != 0 && !to_decode.count(layer.name)) {
@@ -202,10 +203,16 @@ void handle(std::string message, int z, unsigned x, unsigned y, std::set<std::st
 		// X and Y are unsigned, so no need to check <0
 		if (x > (1ULL << z) || y > (1ULL << z)) {
 			fprintf(stderr, "Impossible tile %d/%u/%u\n", z, x, y);
-			exit(EXIT_FAILURE);
+			exit(EXIT_IMPOSSIBLE);
 		}
 
-		layer_to_geojson(layer, z, x, y, !pipeline, pipeline, pipeline, false, 0, 0, 0, !force, state);
+		double scale = 0;
+		if (coordinate_mode == 1) { // fraction
+			scale = layer.extent;
+		} else if (coordinate_mode == 2) { // integer
+			scale = 1;
+		}
+		layer_to_geojson(layer, z, x, y, !pipeline, pipeline, pipeline, false, 0, 0, 0, !force, state, scale);
 
 		if (!pipeline) {
 			if (true) {
@@ -223,7 +230,7 @@ void handle(std::string message, int z, unsigned x, unsigned y, std::set<std::st
 	}
 }
 
-void decode(char *fname, int z, unsigned x, unsigned y, std::set<std::string> const &to_decode, bool pipeline, bool stats, std::set<std::string> const &exclude_meta) {
+void decode(char *fname, int z, unsigned x, unsigned y, std::set<std::string> const &to_decode, bool pipeline, bool stats, std::set<std::string> const &exclude_meta, int coordinate_mode) {
 	sqlite3 *db = NULL;
 	bool isdir = false;
 	int oz = z;
@@ -240,12 +247,12 @@ void decode(char *fname, int z, unsigned x, unsigned y, std::set<std::string> co
 					if (strcmp(map, "SQLite format 3") != 0) {
 						if (z >= 0) {
 							std::string s = std::string(map, st.st_size);
-							handle(s, z, x, y, to_decode, pipeline, stats, state);
+							handle(s, z, x, y, to_decode, pipeline, stats, state, coordinate_mode);
 							munmap(map, st.st_size);
 							return;
 						} else {
 							fprintf(stderr, "Must specify zoom/x/y to decode a single pbf file\n");
-							exit(EXIT_FAILURE);
+							exit(EXIT_ARGS);
 						}
 					}
 				}
@@ -256,7 +263,7 @@ void decode(char *fname, int z, unsigned x, unsigned y, std::set<std::string> co
 		}
 		if (close(fd) != 0) {
 			perror("close");
-			exit(EXIT_FAILURE);
+			exit(EXIT_CLOSE);
 		}
 	} else {
 		perror(fname);
@@ -272,13 +279,13 @@ void decode(char *fname, int z, unsigned x, unsigned y, std::set<std::string> co
 	} else {
 		if (sqlite3_open(fname, &db) != SQLITE_OK) {
 			fprintf(stderr, "%s: %s\n", fname, sqlite3_errmsg(db));
-			exit(EXIT_FAILURE);
+			exit(EXIT_OPEN);
 		}
 
 		char *err = NULL;
 		if (sqlite3_exec(db, "PRAGMA integrity_check;", NULL, NULL, &err) != SQLITE_OK) {
 			fprintf(stderr, "%s: integrity_check: %s\n", fname, err);
-			exit(EXIT_FAILURE);
+			exit(EXIT_SQLITE);
 		}
 	}
 
@@ -299,7 +306,7 @@ void decode(char *fname, int z, unsigned x, unsigned y, std::set<std::string> co
 			sqlite3_stmt *stmt2;
 			if (sqlite3_prepare_v2(db, sql2, -1, &stmt2, NULL) != SQLITE_OK) {
 				fprintf(stderr, "%s: select failed: %s\n", fname, sqlite3_errmsg(db));
-				exit(EXIT_FAILURE);
+				exit(EXIT_SQLITE);
 			}
 
 			while (sqlite3_step(stmt2) == SQLITE_ROW) {
@@ -308,7 +315,7 @@ void decode(char *fname, int z, unsigned x, unsigned y, std::set<std::string> co
 
 				if (name == NULL || value == NULL) {
 					fprintf(stderr, "Corrupt mbtiles file: null metadata\n");
-					exit(EXIT_FAILURE);
+					exit(EXIT_SQLITE);
 				}
 
 				if (exclude_meta.count((char *) name) == 0) {
@@ -361,7 +368,7 @@ void decode(char *fname, int z, unsigned x, unsigned y, std::set<std::string> co
 				FILE *f = fopen(fn.c_str(), "rb");
 				if (f == NULL) {
 					perror(fn.c_str());
-					exit(EXIT_FAILURE);
+					exit(EXIT_OPEN);
 				}
 
 				std::string s;
@@ -372,14 +379,14 @@ void decode(char *fname, int z, unsigned x, unsigned y, std::set<std::string> co
 				}
 				fclose(f);
 
-				handle(s, tiles[i].z, tiles[i].x, tiles[i].y, to_decode, pipeline, stats, state);
+				handle(s, tiles[i].z, tiles[i].x, tiles[i].y, to_decode, pipeline, stats, state, coordinate_mode);
 			}
 		} else {
 			const char *sql = "SELECT tile_data, zoom_level, tile_column, tile_row from tiles where zoom_level between ? and ? order by zoom_level, tile_column, tile_row;";
 			sqlite3_stmt *stmt;
 			if (sqlite3_prepare_v2(db, sql, -1, &stmt, NULL) != SQLITE_OK) {
 				fprintf(stderr, "%s: select failed: %s\n", fname, sqlite3_errmsg(db));
-				exit(EXIT_FAILURE);
+				exit(EXIT_SQLITE);
 			}
 
 			sqlite3_bind_int(stmt, 1, minzoom);
@@ -407,18 +414,18 @@ void decode(char *fname, int z, unsigned x, unsigned y, std::set<std::string> co
 
 				if (tz < 0 || tz >= 32) {
 					fprintf(stderr, "Impossible zoom level %d in mbtiles\n", tz);
-					exit(EXIT_FAILURE);
+					exit(EXIT_IMPOSSIBLE);
 				}
 
 				ty = (1LL << tz) - 1 - ty;
 				const char *s = (const char *) sqlite3_column_blob(stmt, 0);
 
 				if (s == NULL) {
 					fprintf(stderr, "Corrupt mbtiles file: null entry in tiles table\n");
-					exit(EXIT_FAILURE);
+					exit(EXIT_SQLITE);
 				}
 
-				handle(std::string(s, len), tz, tx, ty, to_decode, pipeline, stats, state);
+				handle(std::string(s, len), tz, tx, ty, to_decode, pipeline, stats, state, coordinate_mode);
 			}
 
 			sqlite3_finalize(stmt);
@@ -443,7 +450,7 @@ void decode(char *fname, int z, unsigned x, unsigned y, std::set<std::string> co
 			sqlite3_stmt *stmt;
 			if (sqlite3_prepare_v2(db, sql, -1, &stmt, NULL) != SQLITE_OK) {
 				fprintf(stderr, "%s: select failed: %s\n", fname, sqlite3_errmsg(db));
-				exit(EXIT_FAILURE);
+				exit(EXIT_SQLITE);
 			}
 
 			sqlite3_bind_int(stmt, 1, z);
@@ -456,14 +463,14 @@ void decode(char *fname, int z, unsigned x, unsigned y, std::set<std::string> co
 
 				if (s == NULL) {
 					fprintf(stderr, "Corrupt mbtiles file: null entry in tiles table\n");
-					exit(EXIT_FAILURE);
+					exit(EXIT_SQLITE);
 				}
 
 				if (z != oz) {
 					fprintf(stderr, "%s: Warning: using tile %d/%u/%u instead of %d/%u/%u\n", fname, z, x, y, oz, ox, oy);
 				}
 
-				handle(std::string(s, len), z, x, y, to_decode, pipeline, stats, state);
+				handle(std::string(s, len), z, x, y, to_decode, pipeline, stats, state, coordinate_mode);
 				handled = 1;
 			}
 
@@ -477,13 +484,13 @@ void decode(char *fname, int z, unsigned x, unsigned y, std::set<std::string> co
 
 	if (sqlite3_close(db) != SQLITE_OK) {
 		fprintf(stderr, "%s: could not close database: %s\n", fname, sqlite3_errmsg(db));
-		exit(EXIT_FAILURE);
+		exit(EXIT_CLOSE);
 	}
 }
 
 void usage(char **argv) {
 	fprintf(stderr, "Usage: %s [-s projection] [-Z minzoom] [-z maxzoom] [-l layer ...] file.mbtiles [zoom x y]\n", argv[0]);
-	exit(EXIT_FAILURE);
+	exit(EXIT_ARGS);
 }
 
 int main(int argc, char **argv) {
@@ -494,9 +501,12 @@ int main(int argc, char **argv) {
 	bool pipeline = false;
 	bool stats = false;
 	std::set<std::string> exclude_meta;
+	int coordinate_mode = 0;
 
 	struct option long_options[] = {
 		{"projection", required_argument, 0, 's'},
+		{"fractional-coordinates", no_argument, 0, 'F'},
+		{"integer-coordinates", no_argument, 0, 'I'},
 		{"maximum-zoom", required_argument, 0, 'z'},
 		{"minimum-zoom", required_argument, 0, 'Z'},
 		{"layer", required_argument, 0, 'l'},
@@ -527,6 +537,14 @@ int main(int argc, char **argv) {
 			set_projection_or_exit(optarg);
 			break;
 
+		case 'F':
+			coordinate_mode = 1;
+			break;
+
+		case 'I':
+			coordinate_mode = 2;
+			break;
+
 		case 'z':
 			maxzoom = atoi(optarg);
 			break;
@@ -561,9 +579,9 @@ int main(int argc, char **argv) {
 	}
 
 	if (argc == optind + 4) {
-		decode(argv[optind], atoi(argv[optind + 1]), atoi(argv[optind + 2]), atoi(argv[optind + 3]), to_decode, pipeline, stats, exclude_meta);
+		decode(argv[optind], atoi(argv[optind + 1]), atoi(argv[optind + 2]), atoi(argv[optind + 3]), to_decode, pipeline, stats, exclude_meta, coordinate_mode);
 	} else if (argc == optind + 1) {
-		decode(argv[optind], -1, -1, -1, to_decode, pipeline, stats, exclude_meta);
+		decode(argv[optind], -1, -1, -1, to_decode, pipeline, stats, exclude_meta, coordinate_mode);
 	} else {
 		usage(argv);
 	}