Merge pull request #871 from lonvia/json-export-opensearch

Implement missing JSON export for OpenSearch version

Author: lonvia

app/opensearch/src/main/java/de/komoot/photon/JsonDumper.java

@@ -1,22 +1,59 @@
 package de.komoot.photon;
 
-import org.apache.commons.lang3.NotImplementedException;
+import com.fasterxml.jackson.core.JsonEncoding;
+import com.fasterxml.jackson.core.JsonGenerator;
+import com.fasterxml.jackson.core.Version;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.module.SimpleModule;
+import de.komoot.photon.opensearch.PhotonDocSerializer;
+import org.slf4j.Logger;
 
-import java.io.FileNotFoundException;
+import java.io.File;
+import java.io.IOException;
 
 public class JsonDumper implements Importer {
+    private static final Logger LOGGER = org.slf4j.LoggerFactory.getLogger(JsonDumper.class);
 
-    public JsonDumper(String filename, String[] languages, String[] extraTags) throws FileNotFoundException {
-        throw new NotImplementedException();
+    final JsonGenerator generator;
+
+    public JsonDumper(String filename, String[] languages, String[] extraTags) throws IOException {
+        final var module = new SimpleModule("PhotonDocSerializer",
+                new Version(1, 0, 0, null, null, null));
+        module.addSerializer(PhotonDoc.class, new PhotonDocSerializer(languages, extraTags));
+
+        final ObjectMapper mapper = new ObjectMapper();
+        mapper.registerModule(module);
+
+        if ("-".equals(filename)) {
+            generator = mapper.getFactory().createGenerator(System.out, JsonEncoding.UTF8);
+        } else {
+            generator = mapper.getFactory().createGenerator(new File(filename), JsonEncoding.UTF8);
+        }
+        generator.writeStartObject();
+        generator.writeObjectField("id", "Photon Dump Header");
+        generator.writeObjectField("version", PhotonDocSerializer.FORMAT_VERSION);
+        generator.writeEndObject();
     }
 
     @Override
     public void add(PhotonDoc doc, int objectId) {
-        throw new NotImplementedException();
+        try {
+            generator.writeStartObject();
+            generator.writeObjectField("id", doc.getUid(objectId));
+            generator.writeObjectField("document", doc);
+            generator.writeEndObject();
+        } catch (IOException e) {
+            LOGGER.error("Error writing json file", e);
+        }
+
     }
 
     @Override
     public void finish() {
-        throw new NotImplementedException();
+        try {
+            generator.close();
+        } catch (IOException e) {
+            LOGGER.warn("Error while closing output file",  e);
+        }
     }
 }

app/opensearch/src/main/java/de/komoot/photon/opensearch/PhotonDocSerializer.java

@@ -15,6 +15,10 @@
 import java.util.Set;
 
 public class PhotonDocSerializer extends StdSerializer<PhotonDoc> {
+    // Versioning of the json output format produced. This version appears
+    // in JSON dumps and allows to track changes.
+    public static final String FORMAT_VERSION = "1.0.0";
+
     private final String[] languages;
     private final String[] extraTags;
 

src/main/java/de/komoot/photon/App.java

@@ -13,7 +13,6 @@
 import spark.Request;
 import spark.Response;
 
-import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.util.*;
 import java.util.concurrent.ConcurrentLinkedQueue;
@@ -112,7 +111,7 @@ private static void startJsonDump(CommandLineArgs args) {
 
             importFromDatabase(args, jsonDumper);
             LOGGER.info("Json dump was created: {}", filename);
-        } catch (FileNotFoundException e) {
+        } catch (IOException e) {
             throw new UsageException("Cannot create dump: " + e.getMessage());
         }
     }

src/main/java/de/komoot/photon/CommandLineArgs.java

@@ -53,7 +53,7 @@ public class CommandLineArgs {
     @Parameter(names = "-query-timeout", description = "Time after which to cancel queries to the ES database (in seconds).")
     private int queryTimeout = 7;
 
-    @Parameter(names = "-json", description = "Read from nominatim database and dump it to the given file in a json-like format (useful for developing).")
+    @Parameter(names = "-json", description = "Read from nominatim database and dump it to the given file in a json-like format (use '-' for dumping to stdout).")
     private String jsonDump = null;
 
     @Parameter(names = "-host", description = "Hostname of the PostgreSQL database.")

src/main/resources/log4j2.xml

@@ -1,17 +1,17 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <Configuration status="error">
-	<Appenders>
-		<Console name="stdout" target="SYSTEM_OUT">
-			<PatternLayout pattern="%d [%t] %-5p %c - %m%n" />
-		</Console>
-		<Async name="ASYNC" bufferSize="500">
-        	<AppenderRef ref="stdout"/>
-    	</Async>   
-	</Appenders>
-	<Loggers>
-	    <Logger name="de.komoot.photon" level="info"/>
-		<Root level="warn">
-			<AppenderRef ref="ASYNC" />
-		</Root>
-	</Loggers>
+    <Appenders>
+        <Console name="stderr" target="SYSTEM_ERR">
+            <PatternLayout pattern="%d [%t] %-5p %c - %m%n" />
+        </Console>
+        <Async name="ASYNC" bufferSize="500">
+            <AppenderRef ref="stderr"/>
+        </Async>
+    </Appenders>
+    <Loggers>
+        <Logger name="de.komoot.photon" level="info"/>
+        <Root level="warn">
+            <AppenderRef ref="ASYNC" />
+        </Root>
+    </Loggers>
 </Configuration>