Don't preflight zoom levels for potential as-needed dropping (#40)

* Working on eliminating preflighting * Adjust for the map/images schema change * Avoid generating duplicate tiles with the detail reduction strategy * Do error checking if tiles in a directory can't be written * Update changelog and version * Revert unintentional code reordering * Neglected to add the exit on error here
felt · Nov 29, 2022 · 5c647cd · 5c647cd
1 parent 3095adb
commit 5c647cd
Show file tree

Hide file tree

Showing 7 changed files with 176 additions and 56 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,7 @@
+## 2.14.0
+
+* Don't preflight each zoom level if one of the as-needed options is specified. Instead, go ahead and write out the tiles, and then clear out the zoom level for another try if necessary.
+
 ## 2.13.1
 
 * Simplify geometry earlier when the in-memory representation of a tile gets large, to reduce peak memory usage

diff --git a/dirtiles.cpp b/dirtiles.cpp
@@ -24,13 +24,19 @@ std::string dir_read_tile(std::string base, struct zxy tile) {
 }
 
 void dir_write_tile(const char *outdir, int z, int tx, int ty, std::string const &pbf) {
+	// Don't check mkdir error returns, since most of these calls to
+	// mkdir will be creating directories that already exist.
 	mkdir(outdir, S_IRWXU | S_IRWXG | S_IRWXO);
+
 	std::string curdir(outdir);
 	std::string slash("/");
+
 	std::string newdir = curdir + slash + std::to_string(z);
 	mkdir(newdir.c_str(), S_IRWXU | S_IRWXG | S_IRWXO);
+
 	newdir = newdir + "/" + std::to_string(tx);
 	mkdir(newdir.c_str(), S_IRWXU | S_IRWXG | S_IRWXO);
+
 	newdir = newdir + "/" + std::to_string(ty) + ".pbf";
 
 	struct stat st;
@@ -39,9 +45,21 @@ void dir_write_tile(const char *outdir, int z, int tx, int ty, std::string const
 		exit(EXIT_EXISTS);
 	}
 
-	std::ofstream pbfFile(newdir, std::ios::out | std::ios::binary);
-	pbfFile.write(pbf.data(), pbf.size());
-	pbfFile.close();
+	FILE *fp = fopen(newdir.c_str(), "wb");
+	if (fp == NULL) {
+		fprintf(stderr, "%s: %s\n", newdir.c_str(), strerror(errno));
+		exit(EXIT_WRITE);
+	}
+
+	if (fwrite(pbf.c_str(), sizeof(char), pbf.size(), fp) != pbf.size()) {
+		fprintf(stderr, "%s: %s\n", newdir.c_str(), strerror(errno));
+		exit(EXIT_WRITE);
+	}
+
+	if (fclose(fp) != 0) {
+		fprintf(stderr, "%s: %s\n", newdir.c_str(), strerror(errno));
+		exit(EXIT_CLOSE);
+	}
 }
 
 static bool numeric(const char *s) {
@@ -160,6 +178,54 @@ std::vector<zxy> enumerate_dirtiles(const char *fname, int minzoom, int maxzoom)
 	return tiles;
 }
 
+void dir_erase_zoom(const char *fname, int zoom) {
+	DIR *d1 = opendir(fname);
+	if (d1 != NULL) {
+		struct dirent *dp;
+		while ((dp = readdir(d1)) != NULL) {
+			if (numeric(dp->d_name) && atoi(dp->d_name) == zoom) {
+				std::string z = std::string(fname) + "/" + dp->d_name;
+
+				DIR *d2 = opendir(z.c_str());
+				if (d2 == NULL) {
+					perror(z.c_str());
+					exit(EXIT_OPEN);
+				}
+
+				struct dirent *dp2;
+				while ((dp2 = readdir(d2)) != NULL) {
+					if (numeric(dp2->d_name)) {
+						std::string x = z + "/" + dp2->d_name;
+
+						DIR *d3 = opendir(x.c_str());
+						if (d3 == NULL) {
+							perror(x.c_str());
+							exit(EXIT_OPEN);
+						}
+
+						struct dirent *dp3;
+						while ((dp3 = readdir(d3)) != NULL) {
+							if (pbfname(dp3->d_name)) {
+								std::string y = x + "/" + dp3->d_name;
+								if (unlink(y.c_str()) != 0) {
+									perror(y.c_str());
+									exit(EXIT_UNLINK);
+								}
+							}
+						}
+
+						closedir(d3);
+					}
+				}
+
+				closedir(d2);
+			}
+		}
+
+		closedir(d1);
+	}
+}
+
 sqlite3 *dirmeta2tmp(const char *fname) {
 	sqlite3 *db;
 	char *err = NULL;

diff --git a/dirtiles.hpp b/dirtiles.hpp
@@ -6,6 +6,7 @@
 #define DIRTILES_HPP
 
 void dir_write_tile(const char *outdir, int z, int tx, int ty, std::string const &pbf);
+void dir_erase_zoom(const char *outdir, int z);
 
 void check_dir(const char *d, char **argv, bool force, bool forcetable);
 

diff --git a/mbtiles.cpp b/mbtiles.cpp
@@ -73,14 +73,14 @@ sqlite3 *mbtiles_open(char *dbname, char **argv, int forcetable) {
 		}
 	}
 
-	// "images" maps a content hash to tile contents
-	if (sqlite3_exec(outdb, "CREATE TABLE images (tile_data blob, tile_id text);", NULL, NULL, &err) != SQLITE_OK) {
+	// "images" maps a content hash to tile contents, per zoom level
+	if (sqlite3_exec(outdb, "CREATE TABLE images (zoom_level integer, tile_data blob, tile_id text);", NULL, NULL, &err) != SQLITE_OK) {
 		fprintf(stderr, "%s: create images table: %s\n", argv[0], err);
 		if (!forcetable) {
 			exit(EXIT_EXISTS);
 		}
 	}
-	if (sqlite3_exec(outdb, "CREATE UNIQUE INDEX images_id ON images (tile_id);", NULL, NULL, &err) != SQLITE_OK) {
+	if (sqlite3_exec(outdb, "CREATE UNIQUE INDEX images_id ON images (zoom_level, tile_id);", NULL, NULL, &err) != SQLITE_OK) {
 		fprintf(stderr, "%s: create images index: %s\n", argv[0], err);
 		if (!forcetable) {
 			exit(EXIT_EXISTS);
@@ -89,7 +89,7 @@ sqlite3 *mbtiles_open(char *dbname, char **argv, int forcetable) {
 
 	// "tiles" is a view that retrieves content from "images"
 	// via the content hash looked up from "map".
-	if (sqlite3_exec(outdb, "CREATE VIEW tiles AS SELECT map.zoom_level AS zoom_level, map.tile_column AS tile_column, map.tile_row AS tile_row, images.tile_data AS tile_data FROM map JOIN images ON images.tile_id = map.tile_id;", NULL, NULL, &err) != SQLITE_OK) {
+	if (sqlite3_exec(outdb, "CREATE VIEW tiles AS SELECT map.zoom_level AS zoom_level, map.tile_column AS tile_column, map.tile_row AS tile_row, images.tile_data AS tile_data FROM map JOIN images ON images.tile_id = map.tile_id and images.zoom_level = map.zoom_level;", NULL, NULL, &err) != SQLITE_OK) {
 		fprintf(stderr, "%s: create tiles view: %s\n", argv[0], err);
 		if (!forcetable) {
 			exit(EXIT_EXISTS);
@@ -113,20 +113,23 @@ void mbtiles_write_tile(sqlite3 *outdb, int z, int tx, int ty, const char *data,
 	// following https://github.com/mapbox/node-mbtiles/blob/master/lib/mbtiles.js
 
 	sqlite3_stmt *stmt;
-	const char *images = "replace into images (tile_id, tile_data) values (?, ?)";
+	const char *images = "replace into images (zoom_level, tile_id, tile_data) values (?, ?, ?)";
 	if (sqlite3_prepare_v2(outdb, images, -1, &stmt, NULL) != SQLITE_OK) {
 		fprintf(stderr, "sqlite3 images prep failed\n");
 		exit(EXIT_SQLITE);
 	}
 
-	sqlite3_bind_blob(stmt, 1, hash.c_str(), hash.size(), NULL);
-	sqlite3_bind_blob(stmt, 2, data, size, NULL);
+	sqlite3_bind_int(stmt, 1, z);
+	sqlite3_bind_blob(stmt, 2, hash.c_str(), hash.size(), NULL);
+	sqlite3_bind_blob(stmt, 3, data, size, NULL);
 
 	if (sqlite3_step(stmt) != SQLITE_DONE) {
 		fprintf(stderr, "sqlite3 images insert failed: %s\n", sqlite3_errmsg(outdb));
+		exit(EXIT_SQLITE);
 	}
 	if (sqlite3_finalize(stmt) != SQLITE_OK) {
 		fprintf(stderr, "sqlite3 images finalize failed: %s\n", sqlite3_errmsg(outdb));
+		exit(EXIT_SQLITE);
 	}
 
 	const char *map = "insert into map (zoom_level, tile_column, tile_row, tile_id) values (?, ?, ?, ?)";
@@ -142,9 +145,47 @@ void mbtiles_write_tile(sqlite3 *outdb, int z, int tx, int ty, const char *data,
 
 	if (sqlite3_step(stmt) != SQLITE_DONE) {
 		fprintf(stderr, "sqlite3 map insert failed: %s\n", sqlite3_errmsg(outdb));
+		exit(EXIT_SQLITE);
+	}
+	if (sqlite3_finalize(stmt) != SQLITE_OK) {
+		fprintf(stderr, "sqlite3 finalize failed: %s\n", sqlite3_errmsg(outdb));
+		exit(EXIT_SQLITE);
+	}
+}
+
+void mbtiles_erase_zoom(sqlite3 *outdb, int z) {
+	sqlite3_stmt *stmt;
+
+	const char *query = "delete from map where zoom_level = ?";
+	if (sqlite3_prepare_v2(outdb, query, -1, &stmt, NULL) != SQLITE_OK) {
+		fprintf(stderr, "sqlite3 delete map prep failed\n");
+		exit(EXIT_SQLITE);
+	}
+
+	sqlite3_bind_int(stmt, 1, z);
+	if (sqlite3_step(stmt) != SQLITE_DONE) {
+		fprintf(stderr, "sqlite3 delete map failed: %s\n", sqlite3_errmsg(outdb));
+		exit(EXIT_SQLITE);
 	}
 	if (sqlite3_finalize(stmt) != SQLITE_OK) {
-		fprintf(stderr, "sqlite3 map finalize failed: %s\n", sqlite3_errmsg(outdb));
+		fprintf(stderr, "sqlite3 delete map finalize failed: %s\n", sqlite3_errmsg(outdb));
+		exit(EXIT_SQLITE);
+	}
+
+	query = "delete from images where zoom_level = ?";
+	if (sqlite3_prepare_v2(outdb, query, -1, &stmt, NULL) != SQLITE_OK) {
+		fprintf(stderr, "sqlite3 delete images prep failed\n");
+		exit(EXIT_SQLITE);
+	}
+
+	sqlite3_bind_int(stmt, 1, z);
+	if (sqlite3_step(stmt) != SQLITE_DONE) {
+		fprintf(stderr, "sqlite3 delete images failed: %s\n", sqlite3_errmsg(outdb));
+		exit(EXIT_SQLITE);
+	}
+	if (sqlite3_finalize(stmt) != SQLITE_OK) {
+		fprintf(stderr, "sqlite3 delete images finalize failed: %s\n", sqlite3_errmsg(outdb));
+		exit(EXIT_SQLITE);
 	}
 }
 

diff --git a/mbtiles.hpp b/mbtiles.hpp
@@ -46,6 +46,7 @@ struct layermap_entry {
 sqlite3 *mbtiles_open(char *dbname, char **argv, int forcetable);
 
 void mbtiles_write_tile(sqlite3 *outdb, int z, int tx, int ty, const char *data, int size);
+void mbtiles_erase_zoom(sqlite3 *outdb, int z);
 
 void mbtiles_write_metadata(sqlite3 *outdb, const char *outdir, const char *fname, int minzoom, int maxzoom, double minlat, double minlon, double maxlat, double maxlon, double midlat, double midlon, int forcetable, const char *attribution, std::map<std::string, layermap_entry> const &layermap, bool vector, const char *description, bool do_tilestats, std::map<std::string, std::string> const &attribute_descriptions, std::string const &program, std::string const &commandline, std::vector<strategy> const &strategies);