Reduce maximum memory used for vertex sorting (#170)

* Add logging for out-of-memory debugging

* Use less memory for sub-sorting

* Reduce maximum memory used for vertex sorting

Author: e-n-f

CHANGELOG.md

@@ -1,3 +1,7 @@
+# 2.37.1
+
+* Reduce maximum memory used for vertex sorting
+
 # 2.37.0
 
 * Speed up tile-join overzooming and make it use less memory, by not including empty child tiles in the enumeration

main.cpp

@@ -2041,7 +2041,7 @@ std::pair<int, metadata> read_input(std::vector<source> &sources, char *fname, i
 			vertex_readers.push_back(readers[i].vertexfile);
 			rewind(readers[i].vertexfile);
 		}
-		fqsort(vertex_readers, sizeof(vertex), vertexcmp, vertex_out, memsize / 10);
+		fqsort(vertex_readers, sizeof(vertex), vertexcmp, vertex_out, memsize / 20);
 
 		for (size_t i = 0; i < CPUS; i++) {
 			if (fclose(readers[i].vertexfile) != 0) {
@@ -2107,7 +2107,7 @@ std::pair<int, metadata> read_input(std::vector<source> &sources, char *fname, i
 			rewind(readers[i].nodefile);
 		}
 
-		fqsort(node_readers, sizeof(node), nodecmp, node_out, memsize / 10);
+		fqsort(node_readers, sizeof(node), nodecmp, node_out, memsize / 20);
 
 		for (size_t i = 0; i < CPUS; i++) {
 			if (fclose(readers[i].nodefile) != 0) {

sort.cpp

@@ -4,12 +4,16 @@
 #include <vector>
 #include <string>
 
-#define MAX_MEMORY (10 * 1024 * 1024)
+#define MAX_MEMORY (1024 * 1024 * 1024)	 // 1 GB
 
 void fqsort(std::vector<FILE *> &inputs, size_t width, int (*cmp)(const void *, const void *), FILE *out, size_t mem) {
 	std::string pivot;
 	FILE *fp1, *fp2;
 
+	if (mem > MAX_MEMORY) {
+		mem = MAX_MEMORY;
+	}
+
 	{
 		// read some elements into memory to choose a pivot from
 		//

version.hpp

@@ -1,6 +1,6 @@
 #ifndef VERSION_HPP
 #define VERSION_HPP
 
-#define VERSION "v2.37.0"
+#define VERSION "v2.37.1"
 
 #endif