diff --git a/app/es_embedded/es/mappings.json b/app/es_embedded/es/mappings.json index 7085a49c1..39c4f7563 100644 --- a/app/es_embedded/es/mappings.json +++ b/app/es_embedded/es/mappings.json @@ -49,6 +49,9 @@ "coordinate": { "type": "geo_point" }, + "geometry": { + "type": "geo_shape" + }, "country": { "properties": { "default": { diff --git a/app/es_embedded/src/main/java/de/komoot/photon/Server.java b/app/es_embedded/src/main/java/de/komoot/photon/Server.java index 1e7adebb0..a52907891 100644 --- a/app/es_embedded/src/main/java/de/komoot/photon/Server.java +++ b/app/es_embedded/src/main/java/de/komoot/photon/Server.java @@ -56,6 +56,7 @@ public class Server { private static final String FIELD_VERSION = "database_version"; private static final String FIELD_LANGUAGES = "indexed_languages"; private static final String FIELD_IMPORT_DATE = "import_date"; + private static final String FIELD_SUPPORT_GEOMETRIES = "support_geometries"; private Node esNode; @@ -177,14 +178,18 @@ private void setupDirectories(URL directoryName) throws IOException, URISyntaxEx } - public DatabaseProperties recreateIndex(String[] languages, Date importDate, boolean supportStructuredQueries) throws IOException { + public DatabaseProperties recreateIndex(String[] languages, Date importDate, boolean supportStructuredQueries, boolean supportGeometries) throws IOException { deleteIndex(); loadIndexSettings().createIndex(esClient, PhotonIndex.NAME); createAndPutIndexMapping(languages, supportStructuredQueries); - DatabaseProperties dbProperties = new DatabaseProperties(languages, importDate, false); + DatabaseProperties dbProperties = new DatabaseProperties() + .setLanguages(languages) + .setImportDate(importDate) + .setSupportGeometries(supportGeometries); + saveToDatabase(dbProperties); return dbProperties; @@ -239,6 +244,7 @@ public void saveToDatabase(DatabaseProperties dbProperties) throws IOException .field(FIELD_VERSION, DATABASE_VERSION) .field(FIELD_LANGUAGES, String.join(",", dbProperties.getLanguages())) .field(FIELD_IMPORT_DATE, dbProperties.getImportDate() instanceof Date ? dbProperties.getImportDate().toInstant() : null) + .field(FIELD_SUPPORT_GEOMETRIES, Boolean.toString(dbProperties.getSupportGeometries())) .endObject().endObject(); esClient.prepareIndex(PhotonIndex.NAME, PhotonIndex.TYPE). @@ -276,11 +282,15 @@ public DatabaseProperties loadFromDatabase() { } String langString = properties.get(FIELD_LANGUAGES); + String importDateString = properties.get(FIELD_IMPORT_DATE); + String supportGeometries = properties.get(FIELD_SUPPORT_GEOMETRIES); + return new DatabaseProperties(langString == null ? null : langString.split(","), - importDateString == null ? null : Date.from(Instant.parse(importDateString)), - false); + importDateString == null ? null : Date.from(Instant.parse(importDateString)), + false, + Boolean.parseBoolean(supportGeometries)); } public Importer createImporter(String[] languages, String[] extraTags) { diff --git a/app/es_embedded/src/main/java/de/komoot/photon/elasticsearch/ElasticResult.java b/app/es_embedded/src/main/java/de/komoot/photon/elasticsearch/ElasticResult.java index 55b8ed5cf..d92a38d30 100644 --- a/app/es_embedded/src/main/java/de/komoot/photon/elasticsearch/ElasticResult.java +++ b/app/es_embedded/src/main/java/de/komoot/photon/elasticsearch/ElasticResult.java @@ -66,6 +66,11 @@ public double[] getCoordinates() { return new double[]{coordinate.get(Constants.LON), coordinate.get(Constants.LAT)}; } + @Override + public String getGeometry() { + return (String) result.getSource().get("geometry"); + } + @Override public double[] getExtent() { final Map extent = (Map) result.getSource().get("extent"); diff --git a/app/es_embedded/src/main/java/de/komoot/photon/elasticsearch/PhotonDocConverter.java b/app/es_embedded/src/main/java/de/komoot/photon/elasticsearch/PhotonDocConverter.java index cc2688bd5..f5580d9f9 100644 --- a/app/es_embedded/src/main/java/de/komoot/photon/elasticsearch/PhotonDocConverter.java +++ b/app/es_embedded/src/main/java/de/komoot/photon/elasticsearch/PhotonDocConverter.java @@ -3,9 +3,11 @@ import de.komoot.photon.Constants; import de.komoot.photon.PhotonDoc; import de.komoot.photon.nominatim.model.AddressType; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentFactory; + +import org.elasticsearch.common.xcontent.*; +import org.elasticsearch.common.xcontent.json.JsonXContent; import org.locationtech.jts.geom.Envelope; +import org.locationtech.jts.io.geojson.GeoJsonWriter; import java.io.IOException; import java.util.HashMap; @@ -38,6 +40,17 @@ public static XContentBuilder convert(PhotonDoc doc, String[] languages, String[ .endObject(); } + if (doc.getGeometry() != null) { + GeoJsonWriter g = new GeoJsonWriter(); + + XContentParser parser = JsonXContent + .jsonXContent + .createParser(NamedXContentRegistry.EMPTY, g.write(doc.getGeometry())); + + builder.field("geometry"); + builder.copyCurrentStructure(parser); + } + if (doc.getHouseNumber() != null) { builder.field("housenumber", doc.getHouseNumber()); } diff --git a/app/es_embedded/src/test/java/de/komoot/photon/ESBaseTester.java b/app/es_embedded/src/test/java/de/komoot/photon/ESBaseTester.java index fc850397d..1ea9bd62a 100644 --- a/app/es_embedded/src/test/java/de/komoot/photon/ESBaseTester.java +++ b/app/es_embedded/src/test/java/de/komoot/photon/ESBaseTester.java @@ -7,6 +7,8 @@ import de.komoot.photon.searcher.PhotonResult; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.io.TempDir; +import org.locationtech.jts.io.ParseException; +import org.locationtech.jts.io.WKTReader; import java.io.IOException; import java.nio.file.Path; @@ -25,9 +27,9 @@ public class ESBaseTester { private ElasticTestServer server; - protected PhotonDoc createDoc(double lon, double lat, int id, int osmId, String key, String value) { + protected PhotonDoc createDoc(double lon, double lat, int id, int osmId, String key, String value) throws ParseException { Point location = FACTORY.createPoint(new Coordinate(lon, lat)); - return new PhotonDoc(id, "W", osmId, key, value).names(Collections.singletonMap("name", "berlin")).centroid(location); + return new PhotonDoc(id, "W", osmId, key, value).names(Collections.singletonMap("name", "berlin")).centroid(location).geometry(new WKTReader().read("POLYGON ((6.4440619 52.1969454, 6.4441094 52.1969158, 6.4441408 52.1969347, 6.4441138 52.1969516, 6.4440933 52.1969643, 6.4440619 52.1969454))")); } protected PhotonResult getById(int id) { @@ -45,17 +47,21 @@ public void tearDown() throws IOException { } public void setUpES() throws IOException { - setUpES(dataDirectory, "en"); + setUpES(dataDirectory, false,"en"); + } + + public void setUpESWithGeometry() throws IOException { + setUpES(dataDirectory, true,"en"); } /** * Setup the ES server * * @throws IOException */ - public void setUpES(Path testDirectory, String... languages) throws IOException { + public void setUpES(Path testDirectory, boolean supportGeometries, String... languages) throws IOException { server = new ElasticTestServer(testDirectory.toString()); server.start(TEST_CLUSTER_NAME, new String[]{}); - server.recreateIndex(languages, new Date(), false); + server.recreateIndex(languages, new Date(), false, supportGeometries); refresh(); } diff --git a/app/es_embedded/src/test/java/de/komoot/photon/elasticsearch/ElasticGetIdResult.java b/app/es_embedded/src/test/java/de/komoot/photon/elasticsearch/ElasticGetIdResult.java index a7579cd40..6a4fb487c 100644 --- a/app/es_embedded/src/test/java/de/komoot/photon/elasticsearch/ElasticGetIdResult.java +++ b/app/es_embedded/src/test/java/de/komoot/photon/elasticsearch/ElasticGetIdResult.java @@ -34,6 +34,10 @@ public double[] getCoordinates() { throw new NotImplementedException(); } + public String getGeometry() { + throw new NotImplementedException(); + } + @Override public double[] getExtent() { throw new NotImplementedException(); diff --git a/app/es_embedded/src/test/java/de/komoot/photon/elasticsearch/ElasticResultTest.java b/app/es_embedded/src/test/java/de/komoot/photon/elasticsearch/ElasticResultTest.java index db51d985e..58e5ce73d 100644 --- a/app/es_embedded/src/test/java/de/komoot/photon/elasticsearch/ElasticResultTest.java +++ b/app/es_embedded/src/test/java/de/komoot/photon/elasticsearch/ElasticResultTest.java @@ -44,7 +44,7 @@ protected PhotonDoc createDoc(double lon, double lat, int id, int osmId, String @BeforeAll void setUp() throws Exception { - setUpES(instanceTestDirectory, "en", "de", "fr", "it"); + setUpES(instanceTestDirectory, false, "en", "de", "fr", "it"); Importer instance = getServer().createImporter(new String[]{"en", "de", "fr", "it"}, new String[]{"population", "capital"}); diff --git a/app/es_embedded/src/test/java/de/komoot/photon/elasticsearch/ImporterTest.java b/app/es_embedded/src/test/java/de/komoot/photon/elasticsearch/ImporterTest.java index c5fafc91e..5fc6801d2 100644 --- a/app/es_embedded/src/test/java/de/komoot/photon/elasticsearch/ImporterTest.java +++ b/app/es_embedded/src/test/java/de/komoot/photon/elasticsearch/ImporterTest.java @@ -6,6 +6,8 @@ import de.komoot.photon.searcher.PhotonResult; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.locationtech.jts.io.ParseException; +import org.locationtech.jts.io.WKTReader; import java.io.IOException; import java.util.Collections; @@ -22,10 +24,11 @@ public void setUp() throws IOException { } @Test - void testAddSimpleDoc() { + void testAddSimpleDoc() throws ParseException { Importer instance = makeImporterWithExtra(""); instance.add(new PhotonDoc(1234, "N", 1000, "place", "city") + .geometry(new WKTReader().read("MULTIPOLYGON (((6.111933 51.2659309, 6.1119417 51.2659247, 6.1119554 51.2659249, 6.1119868 51.2659432, 6.111964 51.2659591, 6.1119333 51.2659391, 6.111933 51.2659309)))")) .extraTags(Collections.singletonMap("maxspeed", "100")), 0); instance.finish(); refresh(); diff --git a/app/es_embedded/src/test/java/de/komoot/photon/elasticsearch/ServerTest.java b/app/es_embedded/src/test/java/de/komoot/photon/elasticsearch/ServerTest.java index 7553dbafd..4dc09d4d9 100644 --- a/app/es_embedded/src/test/java/de/komoot/photon/elasticsearch/ServerTest.java +++ b/app/es_embedded/src/test/java/de/komoot/photon/elasticsearch/ServerTest.java @@ -16,7 +16,7 @@ void testSaveAndLoadFromDatabase() throws IOException { setUpES(); Date now = new Date(); - DatabaseProperties prop = new DatabaseProperties(new String[]{"en", "de", "fr"}, now, false); + DatabaseProperties prop = new DatabaseProperties(new String[]{"en", "de", "fr"}, now, false, false); getServer().saveToDatabase(prop); prop = getServer().loadFromDatabase(); diff --git a/app/opensearch/build.gradle b/app/opensearch/build.gradle index 344457b36..2be69292f 100644 --- a/app/opensearch/build.gradle +++ b/app/opensearch/build.gradle @@ -22,7 +22,7 @@ dependencies { implementation 'org.apache.httpcomponents.client5:httpclient5:5.4.1' implementation 'com.fasterxml.jackson.core:jackson-databind:2.18.2' - implementation('org.codelibs.opensearch:opensearch-runner:2.18.0.0') { + implementation('org.codelibs.opensearch:opensearch-runner:2.18.0.1') { exclude(module: 'repository-url') exclude(module: 'reindex-client') exclude(module: 'rank-eval-client') diff --git a/app/opensearch/src/main/java/de/komoot/photon/Server.java b/app/opensearch/src/main/java/de/komoot/photon/Server.java index 35ea6257d..fe555dd35 100644 --- a/app/opensearch/src/main/java/de/komoot/photon/Server.java +++ b/app/opensearch/src/main/java/de/komoot/photon/Server.java @@ -33,9 +33,11 @@ public class Server { private static final Logger LOGGER = org.slf4j.LoggerFactory.getLogger(Server.class); - public static final String OPENSEARCH_MODULES = - "org.opensearch.transport.Netty4Plugin," - + "org.opensearch.analysis.common.CommonAnalysisPlugin"; +// public static final String OPENSEARCH_MODULES = +// "org.opensearch.transport.Netty4Plugin," +// + "org.opensearch.analysis.common.CommonAnalysisPlugin," +// + "org.opensearch.geo.GeoModulePlugin," +// + "org.opensearch.geospatial.plugin.GeospatialPlugin"; protected OpenSearchClient client; private OpenSearchRunner runner = null; @@ -86,7 +88,6 @@ private HttpHost[] startInternal(String clusterName) { .basePath(dataDirectory) .clusterName(clusterName) .numOfNode(1) - .moduleTypes(OPENSEARCH_MODULES) ); runner.ensureYellow(); @@ -119,7 +120,7 @@ public void shutdown() { } } - public DatabaseProperties recreateIndex(String[] languages, Date importDate, boolean supportStructuredQueries) throws IOException { + public DatabaseProperties recreateIndex(String[] languages, Date importDate, boolean supportStructuredQueries, boolean supportGeometries) throws IOException { // delete any existing data if (client.indices().exists(e -> e.index(PhotonIndex.NAME)).value()) { client.indices().delete(d -> d.index(PhotonIndex.NAME)); @@ -129,7 +130,7 @@ public DatabaseProperties recreateIndex(String[] languages, Date importDate, boo (new IndexMapping(supportStructuredQueries)).addLanguages(languages).putMapping(client, PhotonIndex.NAME); - var dbProperties = new DatabaseProperties(languages, importDate, supportStructuredQueries); + var dbProperties = new DatabaseProperties(languages, importDate, supportStructuredQueries, supportGeometries); saveToDatabase(dbProperties); return dbProperties; @@ -180,7 +181,8 @@ public DatabaseProperties loadFromDatabase() throws IOException { return new DatabaseProperties(dbEntry.source().languages, dbEntry.source().importDate, - dbEntry.source().supportStructuredQueries); + dbEntry.source().supportStructuredQueries, + dbEntry.source().supportGeometries); } public Importer createImporter(String[] languages, String[] extraTags) { diff --git a/app/opensearch/src/main/java/de/komoot/photon/opensearch/DBPropertyEntry.java b/app/opensearch/src/main/java/de/komoot/photon/opensearch/DBPropertyEntry.java index b9b88dc01..21679cab3 100644 --- a/app/opensearch/src/main/java/de/komoot/photon/opensearch/DBPropertyEntry.java +++ b/app/opensearch/src/main/java/de/komoot/photon/opensearch/DBPropertyEntry.java @@ -9,6 +9,7 @@ public class DBPropertyEntry { public Date importDate; public String[] languages; public boolean supportStructuredQueries; + public boolean supportGeometries; public DBPropertyEntry() {} @@ -17,5 +18,6 @@ public DBPropertyEntry(DatabaseProperties props, String databaseVersion) { importDate = props.getImportDate(); languages = props.getLanguages(); supportStructuredQueries = props.getSupportStructuredQueries(); + supportGeometries = props.getSupportGeometries(); } } diff --git a/app/opensearch/src/main/java/de/komoot/photon/opensearch/Importer.java b/app/opensearch/src/main/java/de/komoot/photon/opensearch/Importer.java index c3e7b31bf..dc488cbc2 100644 --- a/app/opensearch/src/main/java/de/komoot/photon/opensearch/Importer.java +++ b/app/opensearch/src/main/java/de/komoot/photon/opensearch/Importer.java @@ -4,6 +4,8 @@ import org.opensearch.client.opensearch.OpenSearchClient; import org.opensearch.client.opensearch._types.Time; import org.opensearch.client.opensearch.core.BulkRequest; +import org.opensearch.client.opensearch.core.bulk.BulkOperation; +import org.opensearch.client.opensearch.core.bulk.BulkResponseItem; import org.slf4j.Logger; import java.io.IOException; @@ -54,7 +56,14 @@ private void saveDocuments() { var response = client.bulk(bulkRequest.build()); if (response.errors()) { - LOGGER.error("Error during bulk import."); + for (BulkResponseItem bri: response.items()) { + LOGGER.error("Error during bulk import."); + if (bri.error() != null) { + LOGGER.error(bri.error().reason()); + LOGGER.error(bri.error().type()); + LOGGER.error(bri.error().stackTrace()); + } + } } } catch (IOException e) { LOGGER.error("Error during bulk import", e); diff --git a/app/opensearch/src/main/java/de/komoot/photon/opensearch/IndexMapping.java b/app/opensearch/src/main/java/de/komoot/photon/opensearch/IndexMapping.java index a88c190ab..32840c6bc 100644 --- a/app/opensearch/src/main/java/de/komoot/photon/opensearch/IndexMapping.java +++ b/app/opensearch/src/main/java/de/komoot/photon/opensearch/IndexMapping.java @@ -78,6 +78,7 @@ private void setupBaseMappings() { } mappings.properties("coordinate", b -> b.geoPoint(p -> p)); + mappings.properties("geometry", b -> b.geoShape(p -> p)); mappings.properties("countrycode", b -> b.keyword(p -> p.index(true))); mappings.properties("importance", b -> b.float_(p -> p.index(false))); diff --git a/app/opensearch/src/main/java/de/komoot/photon/opensearch/OpenSearchResult.java b/app/opensearch/src/main/java/de/komoot/photon/opensearch/OpenSearchResult.java index 3cbf95069..0bfeba686 100644 --- a/app/opensearch/src/main/java/de/komoot/photon/opensearch/OpenSearchResult.java +++ b/app/opensearch/src/main/java/de/komoot/photon/opensearch/OpenSearchResult.java @@ -11,14 +11,16 @@ public class OpenSearchResult implements PhotonResult { private double score = 0.0; private final double[] extent; private final double[] coordinates; + private final String geometry; private final Map infos; private final Map> localeTags; - OpenSearchResult(double[] extent, double[] coordinates, Map infos, Map> localeTags) { + OpenSearchResult(double[] extent, double[] coordinates, Map infos, Map> localeTags, String geometry) { this.extent = extent; this.coordinates = coordinates; this.infos = infos; this.localeTags = localeTags; + this.geometry = geometry; } public OpenSearchResult setScore(double score) { @@ -61,6 +63,10 @@ public double[] getCoordinates() { return coordinates; } + public String getGeometry() { + return geometry; + } + @Override public double[] getExtent() { return extent; diff --git a/app/opensearch/src/main/java/de/komoot/photon/opensearch/OpenSearchResultDeserializer.java b/app/opensearch/src/main/java/de/komoot/photon/opensearch/OpenSearchResultDeserializer.java index 1573aa393..7f822f365 100644 --- a/app/opensearch/src/main/java/de/komoot/photon/opensearch/OpenSearchResultDeserializer.java +++ b/app/opensearch/src/main/java/de/komoot/photon/opensearch/OpenSearchResultDeserializer.java @@ -29,6 +29,11 @@ public OpenSearchResult deserialize(JsonParser p, DeserializationContext ctxt) t final Map tags = new HashMap<>(); final Map> localeTags = new HashMap<>(); + String geometry = null; + if (node.get("geometry") != null) { + geometry = node.get("geometry").toString(); + } + var fields = node.fields(); while (fields.hasNext()) { final var entry = fields.next(); @@ -55,7 +60,7 @@ public OpenSearchResult deserialize(JsonParser p, DeserializationContext ctxt) t } } - return new OpenSearchResult(extent, coordinates, tags, localeTags); + return new OpenSearchResult(extent, coordinates, tags, localeTags, geometry); } private double[] extractExtent(ObjectNode node) { @@ -78,5 +83,4 @@ private double[] extractCoordinate(ObjectNode node) { return new double[]{node.get(Constants.LON).doubleValue(), node.get(Constants.LAT).doubleValue()}; } - } diff --git a/app/opensearch/src/main/java/de/komoot/photon/opensearch/PhotonDocSerializer.java b/app/opensearch/src/main/java/de/komoot/photon/opensearch/PhotonDocSerializer.java index 5071cda34..4d3adc899 100644 --- a/app/opensearch/src/main/java/de/komoot/photon/opensearch/PhotonDocSerializer.java +++ b/app/opensearch/src/main/java/de/komoot/photon/opensearch/PhotonDocSerializer.java @@ -7,6 +7,7 @@ import de.komoot.photon.PhotonDoc; import de.komoot.photon.Utils; import org.locationtech.jts.geom.Envelope; +import org.locationtech.jts.io.geojson.GeoJsonWriter; import java.io.IOException; import java.util.HashMap; @@ -52,6 +53,15 @@ public void serialize(PhotonDoc value, JsonGenerator gen, SerializerProvider pro gen.writeEndObject(); } + if (value.getGeometry() != null && !value.getGeometry().getGeometryType().equals("Point")) { + // Convert JTS Geometry to GeoJSON + GeoJsonWriter geoJsonWriter = new GeoJsonWriter(); + String geoJson = geoJsonWriter.write(value.getGeometry()); + + gen.writeFieldName("geometry"); + gen.writeRawValue(geoJson); + } + if (value.getHouseNumber() != null) { gen.writeStringField("housenumber", value.getHouseNumber()); } @@ -86,7 +96,7 @@ public void serialize(PhotonDoc value, JsonGenerator gen, SerializerProvider pro gen.writeEndObject(); } - private void writeName(JsonGenerator gen, PhotonDoc doc, String[] languages) throws IOException { + private void writeName(JsonGenerator gen, PhotonDoc doc, String[] languages) throws IOException { Map fNames = new HashMap<>(); doc.copyName(fNames, "default", "name"); diff --git a/app/opensearch/src/test/java/de/komoot/photon/ESBaseTester.java b/app/opensearch/src/test/java/de/komoot/photon/ESBaseTester.java index 45280e237..98bf0979d 100644 --- a/app/opensearch/src/test/java/de/komoot/photon/ESBaseTester.java +++ b/app/opensearch/src/test/java/de/komoot/photon/ESBaseTester.java @@ -1,5 +1,6 @@ package de.komoot.photon; +import de.komoot.photon.opensearch.Importer; import de.komoot.photon.opensearch.OpenSearchTestServer; import de.komoot.photon.searcher.PhotonResult; import org.junit.jupiter.api.AfterEach; @@ -7,12 +8,15 @@ import org.locationtech.jts.geom.Coordinate; import org.locationtech.jts.geom.GeometryFactory; import org.locationtech.jts.geom.PrecisionModel; +import org.locationtech.jts.io.ParseException; +import org.locationtech.jts.io.WKTReader; import java.io.IOException; import java.nio.file.Path; import java.util.Collections; import java.util.Date; + public class ESBaseTester { public static final String TEST_CLUSTER_NAME = "photon-test"; protected static final GeometryFactory FACTORY = new GeometryFactory(new PrecisionModel(), 4326); @@ -22,11 +26,12 @@ public class ESBaseTester { private OpenSearchTestServer server; - protected PhotonDoc createDoc(double lon, double lat, int id, int osmId, String key, String value) { + protected PhotonDoc createDoc(double lon, double lat, int id, int osmId, String key, String value) throws ParseException { final var location = FACTORY.createPoint(new Coordinate(lon, lat)); return new PhotonDoc(id, "W", osmId, key, value) .names(Collections.singletonMap("name", "berlin")) - .centroid(location); + .centroid(location) + .geometry(new WKTReader().read("POLYGON ((6.4440619 52.1969454, 6.4441094 52.1969158, 6.4441408 52.1969347, 6.4441138 52.1969516, 6.4440933 52.1969643, 6.4440619 52.1969454))")); } @AfterEach @@ -43,34 +48,38 @@ protected PhotonResult getById(String id) { } public void setUpES() throws IOException { - setUpES(dataDirectory, "en"); + setUpES(dataDirectory, false, "en"); + } + + public void setUpESWithGeometry() throws IOException { + setUpES(dataDirectory, true, "en"); } - public void setUpES(Path testDirectory, String... languages) throws IOException { + public void setUpES(Path testDirectory, boolean supportGeometries, String... languages) throws IOException { server = new OpenSearchTestServer(testDirectory.toString()); server.startTestServer(TEST_CLUSTER_NAME); - server.recreateIndex(languages, new Date(), true); + server.recreateIndex(languages, new Date(), true, supportGeometries); server.refreshIndexes(); } protected Importer makeImporter() { - return server.createImporter(new String[]{"en"}, new String[]{}); + return (Importer) server.createImporter(new String[]{"en"}, new String[]{}); } protected Importer makeImporterWithExtra(String... extraTags) { - return server.createImporter(new String[]{"en"}, extraTags); + return (Importer) server.createImporter(new String[]{"en"}, extraTags); } protected Importer makeImporterWithLanguages(String... languages) { - return server.createImporter(languages, new String[]{}); + return (Importer) server.createImporter(languages, new String[]{}); } protected Updater makeUpdater() { - return server.createUpdater(new String[]{"en"}, new String[]{}); + return (Updater) server.createUpdater(new String[]{"en"}, new String[]{}); } protected Updater makeUpdaterWithExtra(String... extraTags) { - return server.createUpdater(new String[]{"en"}, extraTags); + return (Updater) server.createUpdater(new String[]{"en"}, extraTags); } protected Server getServer() { diff --git a/app/opensearch/src/test/java/de/komoot/photon/ServerTest.java b/app/opensearch/src/test/java/de/komoot/photon/ServerTest.java index ac14173f0..abace3cf6 100644 --- a/app/opensearch/src/test/java/de/komoot/photon/ServerTest.java +++ b/app/opensearch/src/test/java/de/komoot/photon/ServerTest.java @@ -16,7 +16,8 @@ void testSaveAndLoadFromDatabase() throws IOException { Date now = new Date(); DatabaseProperties prop = new DatabaseProperties(new String[]{"en", "de", "fr"}, now, - false); + false, + false); getServer().saveToDatabase(prop); prop = getServer().loadFromDatabase(); diff --git a/app/opensearch/src/test/java/de/komoot/photon/opensearch/ImporterTest.java b/app/opensearch/src/test/java/de/komoot/photon/opensearch/ImporterTest.java index f1b2a8687..1fd9d1448 100644 --- a/app/opensearch/src/test/java/de/komoot/photon/opensearch/ImporterTest.java +++ b/app/opensearch/src/test/java/de/komoot/photon/opensearch/ImporterTest.java @@ -6,6 +6,8 @@ import de.komoot.photon.searcher.PhotonResult; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.locationtech.jts.io.ParseException; +import org.locationtech.jts.io.WKTReader; import java.io.IOException; import java.util.Collections; @@ -18,14 +20,15 @@ class ImporterTest extends ESBaseTester { @BeforeEach public void setUp() throws IOException { - setUpES(); + setUpESWithGeometry(); } @Test - void testAddSimpleDoc() { + void testAddSimpleDoc() throws ParseException { Importer instance = makeImporterWithExtra(""); instance.add(new PhotonDoc(1234, "N", 1000, "place", "city") + .geometry(new WKTReader().read("MULTIPOLYGON (((6.111933 51.2659309, 6.1119417 51.2659247, 6.1119554 51.2659249, 6.1119868 51.2659432, 6.111964 51.2659591, 6.1119333 51.2659391, 6.111933 51.2659309)))")) .extraTags(Collections.singletonMap("maxspeed", "100")), 0); instance.finish(); diff --git a/app/opensearch/src/test/java/de/komoot/photon/opensearch/OpenSearchTestServer.java b/app/opensearch/src/test/java/de/komoot/photon/opensearch/OpenSearchTestServer.java index 161f60715..6c65537a8 100644 --- a/app/opensearch/src/test/java/de/komoot/photon/opensearch/OpenSearchTestServer.java +++ b/app/opensearch/src/test/java/de/komoot/photon/opensearch/OpenSearchTestServer.java @@ -38,8 +38,7 @@ public void build(final int number, final Settings.Builder settingsBuilder) { .basePath(instanceDir) .clusterName(clusterName) .numOfNode(1) - .baseHttpPort(9200) - .moduleTypes(OPENSEARCH_MODULES)); + .baseHttpPort(9200)); // wait for yellow status runner.ensureYellow(); diff --git a/app/opensearch/src/test/java/de/komoot/photon/opensearch/StructuredQueryTest.java b/app/opensearch/src/test/java/de/komoot/photon/opensearch/StructuredQueryTest.java index b4c96cafc..335e7eb46 100644 --- a/app/opensearch/src/test/java/de/komoot/photon/opensearch/StructuredQueryTest.java +++ b/app/opensearch/src/test/java/de/komoot/photon/opensearch/StructuredQueryTest.java @@ -45,7 +45,7 @@ private static int getRank(AddressType type) { @BeforeEach void setUp() throws Exception { - setUpES(instanceTestDirectory, LANGUAGE, "de", "fr"); + setUpES(instanceTestDirectory, false, LANGUAGE, "de", "fr"); Importer instance = makeImporter(); var country = new PhotonDoc(0, "R", 0, "place", "country") diff --git a/buildSrc/shared.gradle b/buildSrc/shared.gradle index d9a6817d7..13b653b3b 100644 --- a/buildSrc/shared.gradle +++ b/buildSrc/shared.gradle @@ -13,6 +13,8 @@ application { java { sourceCompatibility = JavaVersion.VERSION_11 targetCompatibility = JavaVersion.VERSION_11 + compileJava.options.encoding = "UTF-8" + compileTestJava.options.encoding = "UTF-8" } repositories { @@ -52,6 +54,7 @@ dependencies { exclude(module: 'commons-logging') } implementation 'org.locationtech.jts:jts-core:1.20.0' + implementation 'org.locationtech.jts.io:jts-io-common:1.20.0' implementation 'com.sparkjava:spark-core:2.9.4' implementation 'net.postgis:postgis-jdbc:2024.1.0' implementation 'org.json:json:20240303' diff --git a/src/main/java/de/komoot/photon/App.java b/src/main/java/de/komoot/photon/App.java index 93a6479bc..e75e99197 100644 --- a/src/main/java/de/komoot/photon/App.java +++ b/src/main/java/de/komoot/photon/App.java @@ -130,12 +130,12 @@ private static void startNominatimImport(CommandLineArgs args, Server esServer) } private static String[] initDatabase(CommandLineArgs args, Server esServer) { - final var nominatimConnector = new NominatimImporter(args.getHost(), args.getPort(), args.getDatabase(), args.getUser(), args.getPassword()); + final var nominatimConnector = new NominatimImporter(args.getHost(), args.getPort(), args.getDatabase(), args.getUser(), args.getPassword(), args.getImportGeometryColumn()); final Date importDate = nominatimConnector.getLastImportDate(); try { // Clear out previous data. - var dbProperties = esServer.recreateIndex(args.getLanguages(), importDate, args.getSupportStructuredQueries()); + var dbProperties = esServer.recreateIndex(args.getLanguages(), importDate, args.getSupportStructuredQueries(), args.getImportGeometryColumn()); // clear out previous data return dbProperties.getLanguages(); } catch (IOException e) { throw new UsageException("Cannot setup index, elastic search config files not readable"); @@ -143,7 +143,7 @@ private static String[] initDatabase(CommandLineArgs args, Server esServer) { } private static void importFromDatabase(CommandLineArgs args, Importer importer) { - final var connector = new NominatimImporter(args.getHost(), args.getPort(), args.getDatabase(), args.getUser(), args.getPassword()); + final var connector = new NominatimImporter(args.getHost(), args.getPort(), args.getDatabase(), args.getUser(), args.getPassword(), args.getImportGeometryColumn()); connector.prepareDatabase(); connector.loadCountryNames(); @@ -172,7 +172,7 @@ private static void importFromDatabase(CommandLineArgs args, Importer importer) for (int i = 0; i < numThreads; ++i) { final NominatimImporter threadConnector; if (i > 0) { - threadConnector = new NominatimImporter(args.getHost(), args.getPort(), args.getDatabase(), args.getUser(), args.getPassword()); + threadConnector = new NominatimImporter(args.getHost(), args.getPort(), args.getDatabase(), args.getUser(), args.getPassword(), args.getImportGeometryColumn()); threadConnector.loadCountryNames(); } else { threadConnector = connector; @@ -211,7 +211,7 @@ private static void importFromDatabase(CommandLineArgs args, Importer importer) private static void startNominatimUpdateInit(CommandLineArgs args) { - NominatimUpdater nominatimUpdater = new NominatimUpdater(args.getHost(), args.getPort(), args.getDatabase(), args.getUser(), args.getPassword()); + NominatimUpdater nominatimUpdater = new NominatimUpdater(args.getHost(), args.getPort(), args.getDatabase(), args.getUser(), args.getPassword(), args.getImportGeometryColumn()); nominatimUpdater.initUpdates(args.getNominatimUpdateInit()); } @@ -236,7 +236,7 @@ private static NominatimUpdater setupNominatimUpdater(CommandLineArgs args, Serv // Get database properties and ensure that the version is compatible. DatabaseProperties dbProperties = server.loadFromDatabase(); - NominatimUpdater nominatimUpdater = new NominatimUpdater(args.getHost(), args.getPort(), args.getDatabase(), args.getUser(), args.getPassword()); + NominatimUpdater nominatimUpdater = new NominatimUpdater(args.getHost(), args.getPort(), args.getDatabase(), args.getUser(), args.getPassword(), args.getImportGeometryColumn()); nominatimUpdater.setUpdater(server.createUpdater(dbProperties.getLanguages(), args.getExtraTags())); return nominatimUpdater; } @@ -251,6 +251,10 @@ private static void startApi(CommandLineArgs args, Server server) throws IOExcep dbProperties.restrictLanguages(args.getLanguages()); } + LOGGER.info("Starting API with the following settings: " + + "\n Languages: {} \n Import Date: {} \n Support Structured Queries: {} \n Support Geometries: {}", + dbProperties.getLanguages(), dbProperties.getImportDate(), dbProperties.getSupportStructuredQueries(), dbProperties.getSupportGeometries()); + port(args.getListenPort()); ipAddress(args.getListenIp()); @@ -266,13 +270,13 @@ private static void startApi(CommandLineArgs args, Server server) throws IOExcep String[] langs = dbProperties.getLanguages(); SearchHandler searchHandler = server.createSearchHandler(langs, args.getQueryTimeout()); - get("api", new SearchRequestHandler("api", searchHandler, langs, args.getDefaultLanguage(), args.getMaxResults())); - get("api/", new SearchRequestHandler("api/", searchHandler, langs, args.getDefaultLanguage(), args.getMaxResults())); + get("api", new SearchRequestHandler("api", searchHandler, langs, args.getDefaultLanguage(), args.getMaxResults(), dbProperties.getSupportGeometries())); + get("api/", new SearchRequestHandler("api/", searchHandler, langs, args.getDefaultLanguage(), args.getMaxResults(), dbProperties.getSupportGeometries())); if (dbProperties.getSupportStructuredQueries()) { StructuredSearchHandler structured = server.createStructuredSearchHandler(langs, args.getQueryTimeout()); - get("structured", new StructuredSearchRequestHandler("structured", structured, langs, args.getDefaultLanguage(), args.getMaxResults())); - get("structured/", new StructuredSearchRequestHandler("structured/", structured, langs, args.getDefaultLanguage(), args.getMaxResults())); + get("structured", new StructuredSearchRequestHandler("structured", structured, langs, args.getDefaultLanguage(), args.getMaxResults(), dbProperties.getSupportGeometries())); + get("structured/", new StructuredSearchRequestHandler("structured/", structured, langs, args.getDefaultLanguage(), args.getMaxResults(), dbProperties.getSupportGeometries())); } ReverseHandler reverseHandler = server.createReverseHandler(args.getQueryTimeout()); diff --git a/src/main/java/de/komoot/photon/CommandLineArgs.java b/src/main/java/de/komoot/photon/CommandLineArgs.java index 4a9b1c03b..638e0482b 100644 --- a/src/main/java/de/komoot/photon/CommandLineArgs.java +++ b/src/main/java/de/komoot/photon/CommandLineArgs.java @@ -98,6 +98,9 @@ public class CommandLineArgs { @Parameter(names = "-max-reverse-results", description = "The maximum possible 'limit' parameter for reverse geocoding searches") private int maxReverseResults = 50; + @Parameter(names = "-import-geometry-column", description = "[import-only] Add the 'geometry' column from Nominatim on import (i.e. add Polygons/Linestrings/Multipolygons etc. for cities, countries etc.). WARNING: This will increase the Elasticsearch Index size! (~575GB for Planet)") + private boolean importGeometryColumn = false; + public String[] getLanguages(boolean useDefaultIfEmpty) { if (useDefaultIfEmpty && languages.isEmpty()) { return new String[]{"en", "de", "fr", "it"}; @@ -215,5 +218,9 @@ public int getMaxReverseResults() { public int getMaxResults() { return maxResults; } + + public boolean getImportGeometryColumn() { + return importGeometryColumn; + } } diff --git a/src/main/java/de/komoot/photon/Constants.java b/src/main/java/de/komoot/photon/Constants.java index 50957da07..fc3a9f268 100644 --- a/src/main/java/de/komoot/photon/Constants.java +++ b/src/main/java/de/komoot/photon/Constants.java @@ -24,4 +24,5 @@ public class Constants { public static final String OSM_VALUE = "osm_value"; public static final String OBJECT_TYPE = "type"; public static final String CLASSIFICATION = "classification"; + public static final String GEOMETRY = "geometry"; } diff --git a/src/main/java/de/komoot/photon/DatabaseProperties.java b/src/main/java/de/komoot/photon/DatabaseProperties.java index 49ce21f2b..875cd1917 100644 --- a/src/main/java/de/komoot/photon/DatabaseProperties.java +++ b/src/main/java/de/komoot/photon/DatabaseProperties.java @@ -10,14 +10,21 @@ public class DatabaseProperties { private String[] languages; private Date importDate; - private final boolean supportStructuredQueries; + private boolean supportStructuredQueries; + private boolean supportGeometries; - public DatabaseProperties(String[] languages, Date importDate, boolean supportStructuredQueries) { + public DatabaseProperties(String[] languages, Date importDate, boolean supportStructuredQueries, boolean supportGeometries) { this.languages = languages; this.importDate = importDate; this.supportStructuredQueries = supportStructuredQueries; + this.supportGeometries = supportGeometries; } + public DatabaseProperties() { + + } + + /** * Return the list of languages for which the database is configured. * If no list was set, then the default is returned. @@ -32,6 +39,18 @@ public String[] getLanguages() { return languages; } + /** + * Replace the language list with the given list. + * + * @param languages Array of two-letter language codes. + * + * @return This object for function chaining. + */ + public DatabaseProperties setLanguages(String[] languages) { + this.languages = languages; + return this; + } + /** * Set language list to the intersection between the existing list and the given list. * @@ -63,16 +82,40 @@ public void restrictLanguages(String[] languageList) { } } - public void setImportDate(Date importDate) { - this.importDate = importDate; - } - - public Date getImportDate() { return this.importDate; } + public DatabaseProperties setImportDate(Date importDate) { + this.importDate = importDate; + return this; + } + public boolean getSupportStructuredQueries() { return supportStructuredQueries; } + + public DatabaseProperties setSupportStructuredQueries(boolean supportStructuredQueries) { + this.supportStructuredQueries = supportStructuredQueries; + return this; + } + + public boolean getSupportGeometries() { + return supportGeometries; + } + + public DatabaseProperties setSupportGeometries(boolean supportGeometries) { + this.supportGeometries = supportGeometries; + return this; + } + + @Override + public String toString() { + return "DatabaseProperties{" + + "languages=" + Arrays.toString(languages) + + ", importDate=" + importDate + + ", supportStructuredQueries=" + supportStructuredQueries + + ", supportGeometries=" + supportGeometries + + '}'; + } } diff --git a/src/main/java/de/komoot/photon/PhotonDoc.java b/src/main/java/de/komoot/photon/PhotonDoc.java index f25683941..cb3e21614 100644 --- a/src/main/java/de/komoot/photon/PhotonDoc.java +++ b/src/main/java/de/komoot/photon/PhotonDoc.java @@ -1,4 +1,4 @@ -package de.komoot.photon; + package de.komoot.photon; import de.komoot.photon.nominatim.model.AddressRow; import org.locationtech.jts.geom.Envelope; @@ -35,6 +35,7 @@ public class PhotonDoc { private Set> context = new HashSet<>(); private String houseNumber = null; private Point centroid = null; + private Geometry geometry = null; public PhotonDoc(long placeId, String osmType, long osmId, String tagKey, String tagValue) { this.placeId = placeId; @@ -63,6 +64,7 @@ public PhotonDoc(PhotonDoc other) { this.rankAddress = other.rankAddress; this.addressParts = other.addressParts; this.context = other.context; + this.geometry = other.geometry; } public PhotonDoc names(Map names) { @@ -87,6 +89,11 @@ public PhotonDoc centroid(Geometry centroid) { return this; } + public PhotonDoc geometry(Geometry geometry) { + this.geometry = (Geometry) geometry; + return this; + } + public PhotonDoc countryCode(String countryCode) { if (countryCode != null) { this.countryCode = countryCode.toUpperCase(); @@ -337,4 +344,33 @@ public String getHouseNumber() { public Point getCentroid() { return this.centroid; } + + public Geometry getGeometry() { + return this.geometry; + } + + @Override + public String toString() { + return "PhotonDoc{" + + "placeId=" + placeId + + ", osmType='" + osmType + '\'' + + ", osmId=" + osmId + + ", tagKey='" + tagKey + '\'' + + ", tagValue='" + tagValue + '\'' + + ", name=" + name + + ", postcode='" + postcode + '\'' + + ", extratags=" + extratags + + ", bbox=" + bbox + + ", parentPlaceId=" + parentPlaceId + + ", importance=" + importance + + ", countryCode='" + countryCode + '\'' + + ", linkedPlaceId=" + linkedPlaceId + + ", rankAddress=" + rankAddress + + ", addressParts=" + addressParts + + ", context=" + context + + ", houseNumber='" + houseNumber + '\'' + + ", centroid=" + centroid + + ", geometry=" + geometry + + '}'; + } } diff --git a/src/main/java/de/komoot/photon/ReverseSearchRequestHandler.java b/src/main/java/de/komoot/photon/ReverseSearchRequestHandler.java index e34bb61fe..e6cda6d63 100644 --- a/src/main/java/de/komoot/photon/ReverseSearchRequestHandler.java +++ b/src/main/java/de/komoot/photon/ReverseSearchRequestHandler.java @@ -53,6 +53,6 @@ public String handle(Request request, Response response) { debugInfo = requestHandler.dumpQuery(photonRequest); } - return new GeocodeJsonFormatter(false, photonRequest.getLanguage()).convert(results, debugInfo); + return new GeocodeJsonFormatter(false, photonRequest.getLanguage(), photonRequest.getGeometry()).convert(results, debugInfo); } } diff --git a/src/main/java/de/komoot/photon/SearchRequestHandler.java b/src/main/java/de/komoot/photon/SearchRequestHandler.java index e1b8575ed..5a5d07bb1 100644 --- a/src/main/java/de/komoot/photon/SearchRequestHandler.java +++ b/src/main/java/de/komoot/photon/SearchRequestHandler.java @@ -20,16 +20,18 @@ public class SearchRequestHandler extends RouteImpl { private final PhotonRequestFactory photonRequestFactory; private final SearchHandler requestHandler; + private final boolean supportGeometries; - SearchRequestHandler(String path, SearchHandler dbHandler, String[] languages, String defaultLanguage, int maxResults) { + SearchRequestHandler(String path, SearchHandler dbHandler, String[] languages, String defaultLanguage, int maxResults, boolean supportGeometries) { super(path); List supportedLanguages = Arrays.asList(languages); - this.photonRequestFactory = new PhotonRequestFactory(supportedLanguages, defaultLanguage, maxResults); + this.photonRequestFactory = new PhotonRequestFactory(supportedLanguages, defaultLanguage, maxResults, supportGeometries); this.requestHandler = dbHandler; + this.supportGeometries = supportGeometries; } @Override - public String handle(Request request, Response response) { + public String handle(Request request, Response response) throws BadRequestException { PhotonRequest photonRequest = null; try { photonRequest = photonRequestFactory.create(request); @@ -39,6 +41,12 @@ public String handle(Request request, Response response) { throw halt(e.getHttpStatus(), json.toString()); } + if (!supportGeometries && photonRequest.getReturnGeometry()) { + JSONObject json = new JSONObject(); + json.put("message", "You're explicitly requesting a geometry, but geometries are not imported!"); + throw halt(400, json.toString()); + } + List results = requestHandler.search(photonRequest); // Further filtering @@ -54,6 +62,6 @@ public String handle(Request request, Response response) { debugInfo = requestHandler.dumpQuery(photonRequest); } - return new GeocodeJsonFormatter(photonRequest.getDebug(), photonRequest.getLanguage()).convert(results, debugInfo); + return new GeocodeJsonFormatter(photonRequest.getDebug(), photonRequest.getLanguage(), photonRequest.getReturnGeometry()).convert(results, debugInfo); } } \ No newline at end of file diff --git a/src/main/java/de/komoot/photon/StructuredSearchRequestHandler.java b/src/main/java/de/komoot/photon/StructuredSearchRequestHandler.java index cc098084b..2c3a63853 100644 --- a/src/main/java/de/komoot/photon/StructuredSearchRequestHandler.java +++ b/src/main/java/de/komoot/photon/StructuredSearchRequestHandler.java @@ -17,12 +17,14 @@ public class StructuredSearchRequestHandler extends RouteImpl { private final PhotonRequestFactory photonRequestFactory; private final StructuredSearchHandler requestHandler; + private final boolean supportGeometries; - StructuredSearchRequestHandler(String path, StructuredSearchHandler dbHandler, String[] languages, String defaultLanguage, int maxResults) { + StructuredSearchRequestHandler(String path, StructuredSearchHandler dbHandler, String[] languages, String defaultLanguage, int maxResults, boolean supportGeometries) { super(path); List supportedLanguages = Arrays.asList(languages); - this.photonRequestFactory = new PhotonRequestFactory(supportedLanguages, defaultLanguage, maxResults); + this.photonRequestFactory = new PhotonRequestFactory(supportedLanguages, defaultLanguage, maxResults, supportGeometries); this.requestHandler = dbHandler; + this.supportGeometries = supportGeometries; } @Override @@ -36,6 +38,12 @@ public String handle(Request request, Response response) { throw halt(e.getHttpStatus(), json.toString()); } + if (!supportGeometries && photonRequest.getReturnGeometry()) { + JSONObject json = new JSONObject(); + json.put("message", "You're requesting a Geometry, but Geometries are not imported!"); + throw halt(400, json.toString()); + } + List results = requestHandler.search(photonRequest); // Further filtering @@ -46,7 +54,12 @@ public String handle(Request request, Response response) { results = results.subList(0, photonRequest.getLimit()); } + String debugInfo = null; - return new GeocodeJsonFormatter(photonRequest.getDebug(), photonRequest.getLanguage()).convert(results, debugInfo); + /* if (photonRequest.getDebug()) { + debugInfo = requestHandler.dumpQuery(photonRequest); + } + */ + return new GeocodeJsonFormatter(photonRequest.getDebug(), photonRequest.getLanguage(), photonRequest.getReturnGeometry()).convert(results, debugInfo); } } \ No newline at end of file diff --git a/src/main/java/de/komoot/photon/nominatim/NominatimConnector.java b/src/main/java/de/komoot/photon/nominatim/NominatimConnector.java index e5b28d6ec..9f0876aad 100644 --- a/src/main/java/de/komoot/photon/nominatim/NominatimConnector.java +++ b/src/main/java/de/komoot/photon/nominatim/NominatimConnector.java @@ -22,8 +22,9 @@ public class NominatimConnector { protected final TransactionTemplate txTemplate; protected Map> countryNames; protected final boolean hasNewStyleInterpolation; + protected boolean useGeometryColumn; - protected NominatimConnector(String host, int port, String database, String username, String password, DBDataAdapter dataAdapter) { + protected NominatimConnector(String host, int port, String database, String username, String password, DBDataAdapter dataAdapter, boolean useGeometryColumn) { BasicDataSource dataSource = new BasicDataSource(); dataSource.setUrl(String.format("jdbc:postgresql://%s:%d/%s", host, port, database)); @@ -42,6 +43,7 @@ protected NominatimConnector(String host, int port, String database, String user dbutils = dataAdapter; hasNewStyleInterpolation = dbutils.hasColumn(template, "location_property_osmline", "step"); + this.useGeometryColumn = useGeometryColumn; } public Date getLastImportDate() { diff --git a/src/main/java/de/komoot/photon/nominatim/NominatimImporter.java b/src/main/java/de/komoot/photon/nominatim/NominatimImporter.java index 8c5ba6864..1d48b2034 100644 --- a/src/main/java/de/komoot/photon/nominatim/NominatimImporter.java +++ b/src/main/java/de/komoot/photon/nominatim/NominatimImporter.java @@ -17,12 +17,12 @@ public class NominatimImporter extends NominatimConnector { private static final Logger LOGGER = org.slf4j.LoggerFactory.getLogger(NominatimImporter.class); - public NominatimImporter(String host, int port, String database, String username, String password) { - this(host, port, database, username, password, new PostgisDataAdapter()); + public NominatimImporter(String host, int port, String database, String username, String password, boolean useGeometryColumn) { + this(host, port, database, username, password, new PostgisDataAdapter(), useGeometryColumn); } - public NominatimImporter(String host, int port, String database, String username, String password, DBDataAdapter dataAdapter) { - super(host, port, database, username, password, dataAdapter); + public NominatimImporter(String host, int port, String database, String username, String password, DBDataAdapter dataAdapter, boolean useGeometryColumn) { + super(host, port, database, username, password, dataAdapter, useGeometryColumn); } @@ -55,12 +55,18 @@ public void readCountry(String countryCode, ImportThread importThread) { NominatimAddressCache addressCache = new NominatimAddressCache(); addressCache.loadCountryAddresses(template, dbutils, countryCode); - final PlaceRowMapper placeRowMapper = new PlaceRowMapper(dbutils); + final PlaceRowMapper placeRowMapper = new PlaceRowMapper(dbutils, useGeometryColumn); + String query = "SELECT place_id, osm_type, osm_id, class, type, name, postcode," + + " address, extratags, ST_Envelope(geometry) AS bbox, parent_place_id," + + " linked_place_id, rank_address, rank_search, importance, country_code, centroid, "; + + if (useGeometryColumn) { + query += "geometry,"; + } + // First read ranks below 30, independent places template.query( - "SELECT place_id, osm_type, osm_id, class, type, name, postcode," + - " address, extratags, ST_Envelope(geometry) AS bbox, parent_place_id," + - " linked_place_id, rank_address, rank_search, importance, country_code, centroid," + + query + dbutils.jsonArrayFromSelect( "address_place_id", "FROM place_addressline pa " + @@ -88,12 +94,17 @@ public void readCountry(String countryCode, ImportThread importThread) { }); // Next get all POIs/housenumbers. + query = "SELECT p.place_id, p.osm_type, p.osm_id, p.class, p.type, p.name, p.postcode," + + " p.address, p.extratags, ST_Envelope(p.geometry) AS bbox, p.parent_place_id," + + " p.linked_place_id, p.rank_address, p.rank_search, p.importance, p.country_code, p.centroid, " + + " parent.class as parent_class, parent.type as parent_type," + + " parent.rank_address as parent_rank_address, parent.name as parent_name, "; + + if (useGeometryColumn) { + query += "p.geometry as geometry, "; + } template.query( - "SELECT p.place_id, p.osm_type, p.osm_id, p.class, p.type, p.name, p.postcode," + - " p.address, p.extratags, ST_Envelope(p.geometry) AS bbox, p.parent_place_id," + - " p.linked_place_id, p.rank_address, p.rank_search, p.importance, p.country_code, p.centroid," + - " parent.class as parent_class, parent.type as parent_type," + - " parent.rank_address as parent_rank_address, parent.name as parent_name, " + + query + dbutils.jsonArrayFromSelect( "address_place_id", "FROM place_addressline pa " + diff --git a/src/main/java/de/komoot/photon/nominatim/NominatimUpdater.java b/src/main/java/de/komoot/photon/nominatim/NominatimUpdater.java index d1afabfc1..9e278a416 100644 --- a/src/main/java/de/komoot/photon/nominatim/NominatimUpdater.java +++ b/src/main/java/de/komoot/photon/nominatim/NominatimUpdater.java @@ -75,16 +75,18 @@ public class NominatimUpdater extends NominatimConnector { // One-item cache for address terms. Speeds up processing of rank 30 objects. private long parentPlaceId = -1; private List parentTerms = null; + private boolean useGeometryColumn; - public NominatimUpdater(String host, int port, String database, String username, String password) { - this(host, port, database, username, password, new PostgisDataAdapter()); + public NominatimUpdater(String host, int port, String database, String username, String password, boolean useGeometryColumn) { + this(host, port, database, username, password, new PostgisDataAdapter(), useGeometryColumn); } - public NominatimUpdater(String host, int port, String database, String username, String password, DBDataAdapter dataAdapter) { - super(host, port, database, username, password, dataAdapter); + public NominatimUpdater(String host, int port, String database, String username, String password, DBDataAdapter dataAdapter, boolean useGeometryColumn) { + super(host, port, database, username, password, dataAdapter, useGeometryColumn); + + final var placeRowMapper = new PlaceRowMapper(dbutils, useGeometryColumn); - final var placeRowMapper = new PlaceRowMapper(dbutils); placeToNominatimResult = (rs, rowNum) -> { PhotonDoc doc = placeRowMapper.mapRow(rs, rowNum); assert (doc != null); @@ -265,8 +267,16 @@ private List getPlaces(String table) { public List getByPlaceId(long placeId) { + String query = SELECT_COLS_PLACEX; + + if (useGeometryColumn) { + query += ", geometry"; + } + + query += " FROM placex WHERE place_id = ? and indexed_status = 0"; + List result = template.query( - SELECT_COLS_PLACEX + " FROM placex WHERE place_id = ? and indexed_status = 0", + query, placeToNominatimResult, placeId); return result.isEmpty() ? null : result.get(0).getDocsWithHousenumber(); diff --git a/src/main/java/de/komoot/photon/nominatim/PostgisDataAdapter.java b/src/main/java/de/komoot/photon/nominatim/PostgisDataAdapter.java index 37e6718fc..bb3a24708 100644 --- a/src/main/java/de/komoot/photon/nominatim/PostgisDataAdapter.java +++ b/src/main/java/de/komoot/photon/nominatim/PostgisDataAdapter.java @@ -34,9 +34,13 @@ public Geometry extractGeometry(ResultSet rs, String columnName) throws SQLExcep try { StringBuffer sb = new StringBuffer(); wkt.getGeometry().outerWKT(sb); - return new WKTReader().read(sb.toString()); + + Geometry geometry = new WKTReader().read(sb.toString()); + geometry.setSRID(4326); + return geometry; } catch (ParseException e) { // ignore + System.out.println(e); } } diff --git a/src/main/java/de/komoot/photon/nominatim/model/PlaceRowMapper.java b/src/main/java/de/komoot/photon/nominatim/model/PlaceRowMapper.java index 4944f00a6..543b3cef7 100644 --- a/src/main/java/de/komoot/photon/nominatim/model/PlaceRowMapper.java +++ b/src/main/java/de/komoot/photon/nominatim/model/PlaceRowMapper.java @@ -16,11 +16,17 @@ public class PlaceRowMapper implements RowMapper { private final DBDataAdapter dbutils; + private boolean useGeometryColumn; public PlaceRowMapper(DBDataAdapter dbutils) { this.dbutils = dbutils; } + public PlaceRowMapper(DBDataAdapter dbutils, boolean useGeometryColumn) { + this.dbutils = dbutils; + this.useGeometryColumn = useGeometryColumn; + } + @Override public PhotonDoc mapRow(ResultSet rs, int rowNum) throws SQLException { PhotonDoc doc = new PhotonDoc(rs.getLong("place_id"), @@ -36,6 +42,14 @@ public PhotonDoc mapRow(ResultSet rs, int rowNum) throws SQLException { .rankAddress(rs.getInt("rank_address")) .postcode(rs.getString("postcode")); + if (useGeometryColumn) { + try { + doc.geometry(dbutils.extractGeometry(rs, "geometry")); + } catch (IllegalArgumentException e) { + System.out.println("Could not get Geometry: " + e); + } + } + double importance = rs.getDouble("importance"); doc.importance(rs.wasNull() ? (0.75 - rs.getInt("rank_search") / 40d) : importance); diff --git a/src/main/java/de/komoot/photon/query/PhotonRequestBase.java b/src/main/java/de/komoot/photon/query/PhotonRequestBase.java index 720686e49..9b7803dee 100644 --- a/src/main/java/de/komoot/photon/query/PhotonRequestBase.java +++ b/src/main/java/de/komoot/photon/query/PhotonRequestBase.java @@ -18,6 +18,7 @@ public class PhotonRequestBase private int zoom = 14; private Envelope bbox = null; private boolean debug = false; + private boolean returnGeometry = false; private final List osmTagFilters = new ArrayList<>(1); private Set layerFilters = new HashSet<>(1); @@ -53,6 +54,8 @@ public String getLanguage() { public boolean getDebug() { return debug; } + public boolean getReturnGeometry() { return returnGeometry; } + public List getOsmTagFilters() { return osmTagFilters; } @@ -100,4 +103,9 @@ void setBbox(Envelope bbox) { void enableDebug() { this.debug = true; } + + void setReturnGeometry(boolean returnGeometry) { + this.returnGeometry = returnGeometry; + } + } diff --git a/src/main/java/de/komoot/photon/query/PhotonRequestFactory.java b/src/main/java/de/komoot/photon/query/PhotonRequestFactory.java index 081244d94..cb369384b 100644 --- a/src/main/java/de/komoot/photon/query/PhotonRequestFactory.java +++ b/src/main/java/de/komoot/photon/query/PhotonRequestFactory.java @@ -17,9 +17,10 @@ public class PhotonRequestFactory { private final BoundingBoxParamConverter bboxParamConverter; private final LayerParamValidator layerParamValidator; private final int maxResults; + private final boolean supportGeometries; private static final HashSet REQUEST_QUERY_PARAMS = new HashSet<>(Arrays.asList("lang", "q", "lon", "lat", - "limit", "osm_tag", "location_bias_scale", "bbox", "debug", "zoom", "layer")); + "limit", "osm_tag", "location_bias_scale", "bbox", "debug", "zoom", "layer", "geometry")); private static final HashSet STRUCTURED_ADDRESS_FIELDS = new HashSet<>(Arrays.asList("countrycode", "state", "county", "city", "postcode", "district", "housenumber", "street")); @@ -28,11 +29,12 @@ public class PhotonRequestFactory { "countrycode", "state", "county", "city", "postcode", "district", "housenumber", "street")); - public PhotonRequestFactory(List supportedLanguages, String defaultLanguage, int maxResults) { + public PhotonRequestFactory(List supportedLanguages, String defaultLanguage, int maxResults, boolean supportGeometries) { this.languageResolver = new RequestLanguageResolver(supportedLanguages, defaultLanguage); this.bboxParamConverter = new BoundingBoxParamConverter(); this.layerParamValidator = new LayerParamValidator(); this.maxResults = maxResults; + this.supportGeometries = supportGeometries; } public StructuredPhotonRequest createStructured(Request webRequest) throws BadRequestException { @@ -59,6 +61,7 @@ public StructuredPhotonRequest createStructured(Request webRequest) throws BadRe result.setStreet(webRequest.queryParams("street")); result.setHouseNumber(webRequest.queryParams("housenumber")); + addCommonParameters(webRequest, result); return result; @@ -110,6 +113,14 @@ private void addCommonParameters(Request webRequest, PhotonRequestBase request) if (layerFiltersQueryMap.hasValue()) { request.setLayerFilter(layerParamValidator.validate(layerFiltersQueryMap.values())); } + + // If the database supports geometries, return them by default. + request.setReturnGeometry(supportGeometries); + + // Check if the user explicitly doesn't want a geometry. + if (webRequest.queryParams("geometry") != null) { + request.setReturnGeometry(parseBoolean(webRequest, "geometry")); + } } private Integer parseInt(Request webRequest, String param) throws BadRequestException { @@ -145,4 +156,15 @@ private Double parseDouble(Request webRequest, String param) throws BadRequestEx return outVal; } + + private Boolean parseBoolean(Request webRequest, String param) { + boolean booleanVal = false; + String value = webRequest.queryParams(param); + + if (value != null && !value.isEmpty()) { + booleanVal = Boolean.parseBoolean(value); + } + + return booleanVal; + } } diff --git a/src/main/java/de/komoot/photon/query/ReverseRequest.java b/src/main/java/de/komoot/photon/query/ReverseRequest.java index cb8c78cde..217b63dfd 100644 --- a/src/main/java/de/komoot/photon/query/ReverseRequest.java +++ b/src/main/java/de/komoot/photon/query/ReverseRequest.java @@ -20,9 +20,10 @@ public class ReverseRequest implements Serializable { private final Set layerFilters; private final List osmTagFilters = new ArrayList<>(1); private final boolean debug; + private final boolean geometry; public ReverseRequest(Point location, String language, double radius, String queryStringFilter, int limit, - boolean locationDistanceSort, Set layerFilter, boolean debug) { + boolean locationDistanceSort, Set layerFilter, boolean debug, boolean geometry) { this.location = location; this.language = language; this.radius = radius; @@ -31,6 +32,7 @@ public ReverseRequest(Point location, String language, double radius, String que this.locationDistanceSort = locationDistanceSort; this.layerFilters = layerFilter; this.debug = debug; + this.geometry = geometry; } public Point getLocation() { @@ -69,6 +71,10 @@ public boolean getDebug() { return debug; } + public boolean getGeometry() { + return geometry; + } + ReverseRequest addOsmTagFilter(TagFilter filter) { osmTagFilters.add(filter); return this; diff --git a/src/main/java/de/komoot/photon/query/ReverseRequestFactory.java b/src/main/java/de/komoot/photon/query/ReverseRequestFactory.java index e539fe127..70a5c2fb4 100644 --- a/src/main/java/de/komoot/photon/query/ReverseRequestFactory.java +++ b/src/main/java/de/komoot/photon/query/ReverseRequestFactory.java @@ -16,7 +16,7 @@ public class ReverseRequestFactory { private static final HashSet REQUEST_QUERY_PARAMS = new HashSet<>(Arrays.asList("lang", "lon", "lat", "radius", - "query_string_filter", "distance_sort", "limit", "layer", "osm_tag", "debug")); + "query_string_filter", "distance_sort", "limit", "layer", "osm_tag", "debug", "geometry")); private static final LocationParamConverter mandatoryLocationParamConverter = new LocationParamConverter(true); private final RequestLanguageResolver languageResolver; @@ -86,7 +86,7 @@ public ReverseRequest create(Request webRequest) throws BadRequestException { } String queryStringFilter = webRequest.queryParams("query_string_filter"); - ReverseRequest request = new ReverseRequest(location, language, radius, queryStringFilter, limit, locationDistanceSort, layerFilter, enableDebug); + ReverseRequest request = new ReverseRequest(location, language, radius, queryStringFilter, limit, locationDistanceSort, layerFilter, enableDebug, Boolean.parseBoolean(webRequest.queryParams("geometry"))); QueryParamsMap tagFiltersQueryMap = webRequest.queryMap("osm_tag"); if (tagFiltersQueryMap.hasValue()) { diff --git a/src/main/java/de/komoot/photon/searcher/GeocodeJsonFormatter.java b/src/main/java/de/komoot/photon/searcher/GeocodeJsonFormatter.java index 4abb437e7..cce1e9178 100644 --- a/src/main/java/de/komoot/photon/searcher/GeocodeJsonFormatter.java +++ b/src/main/java/de/komoot/photon/searcher/GeocodeJsonFormatter.java @@ -15,10 +15,12 @@ public class GeocodeJsonFormatter implements ResultFormatter { private final boolean addDebugInfo; private final String language; + private final boolean useGeometryColumn; - public GeocodeJsonFormatter(boolean addDebugInfo, String language) { + public GeocodeJsonFormatter(boolean addDebugInfo, String language, boolean useGeometryColumn) { this.addDebugInfo = addDebugInfo; this.language = language; + this.useGeometryColumn = useGeometryColumn; } @Override @@ -26,14 +28,32 @@ public String convert(List results, String debugInfo) { final JSONArray features = new JSONArray(results.size()); for (PhotonResult result : results) { - final double[] coordinates = result.getCoordinates(); - - features.put(new JSONObject() + if (useGeometryColumn && (result.get("geometry") != null || result.getGeometry() != null)) { + if (result.get("geometry") != null) { + features.put(new JSONObject() + .put("type", "Feature") + .put("properties", getResultProperties(result)) + .put("geometry", result.get("geometry"))); + } + else { + // We need to un-escape the JSON String first + JSONObject jsonObject = new JSONObject(result.getGeometry()); + + features.put(new JSONObject() + .put("type", "Feature") + .put("properties", getResultProperties(result)) + .put("geometry", jsonObject)); + } + } else { + final double[] coordinates = result.getCoordinates(); + + features.put(new JSONObject() .put("type", "Feature") .put("properties", getResultProperties(result)) .put("geometry", new JSONObject() .put("type", "Point") .put("coordinates", coordinates))); + } } final JSONObject out = new JSONObject(); diff --git a/src/main/java/de/komoot/photon/searcher/PhotonResult.java b/src/main/java/de/komoot/photon/searcher/PhotonResult.java index c667066ba..ffbcefd26 100644 --- a/src/main/java/de/komoot/photon/searcher/PhotonResult.java +++ b/src/main/java/de/komoot/photon/searcher/PhotonResult.java @@ -24,6 +24,9 @@ public interface PhotonResult { Map getMap(String key); double[] getCoordinates(); + + String getGeometry(); + double[] getExtent(); double getScore(); diff --git a/src/test/java/de/komoot/photon/ApiIntegrationTest.java b/src/test/java/de/komoot/photon/ApiIntegrationTest.java index a36e00eb2..589dc9f8d 100644 --- a/src/test/java/de/komoot/photon/ApiIntegrationTest.java +++ b/src/test/java/de/komoot/photon/ApiIntegrationTest.java @@ -7,12 +7,14 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.CsvSource; +import org.locationtech.jts.io.WKTReader; import java.io.BufferedReader; import java.io.InputStreamReader; import java.net.HttpURLConnection; import java.net.URL; import java.net.URLConnection; +import java.util.Collections; import java.util.stream.Collectors; import static org.junit.jupiter.api.Assertions.*; @@ -26,10 +28,11 @@ class ApiIntegrationTest extends ESBaseTester { @BeforeEach void setUp() throws Exception { - setUpES(); + setUpESWithGeometry(); Importer instance = makeImporter(); instance.add(createDoc(13.38886, 52.51704, 1000, 1000, "place", "city").importance(0.6), 0); instance.add(createDoc(13.39026, 52.54714, 1001, 1001, "place", "town").importance(0.3), 0); + instance.add(createDoc(13.39026, 52.54714, 1002, 1002, "place", "city").importance(0.3).names(Collections.singletonMap("name", "linestring")).geometry(new WKTReader().read("LINESTRING (30 10, 10 30, 40 40)")), 0); instance.finish(); refresh(); } @@ -146,4 +149,47 @@ void testApiStatus() throws Exception { assertEquals("Ok", json.getString("status")); assertEquals(prop.getImportDate().toInstant().toString(), json.getString("import_date")); } + + @Test + void testSearchAndGetGeometry() throws Exception { + App.main(new String[]{"-cluster", TEST_CLUSTER_NAME, "-listen-port", Integer.toString(LISTEN_PORT), "-transport-addresses", "127.0.0.1"}); + awaitInitialization(); + HttpURLConnection connection = (HttpURLConnection) new URL("http://127.0.0.1:" + port() + "/api?q=berlin&limit=1").openConnection(); + JSONObject json = new JSONObject( + new BufferedReader(new InputStreamReader(connection.getInputStream())).lines().collect(Collectors.joining("\n"))); + JSONArray features = json.getJSONArray("features"); + assertEquals(1, features.length()); + JSONObject feature = features.getJSONObject(0); + + JSONObject geometry = feature.getJSONObject("geometry"); + assertEquals("Polygon", geometry.getString("type")); + + JSONObject properties = feature.getJSONObject("properties"); + assertEquals("W", properties.getString("osm_type")); + assertEquals("place", properties.getString("osm_key")); + assertEquals("city", properties.getString("osm_value")); + assertEquals("berlin", properties.getString("name")); + } + + @Test + void testSearchAndGetLineString() throws Exception { + App.main(new String[]{"-cluster", TEST_CLUSTER_NAME, "-listen-port", Integer.toString(LISTEN_PORT), "-transport-addresses", "127.0.0.1"}); + awaitInitialization(); + HttpURLConnection connection = (HttpURLConnection) new URL("http://127.0.0.1:" + port() + "/api?q=linestring&limit=1").openConnection(); + JSONObject json = new JSONObject( + new BufferedReader(new InputStreamReader(connection.getInputStream())).lines().collect(Collectors.joining("\n"))); + JSONArray features = json.getJSONArray("features"); + assertEquals(1, features.length()); + JSONObject feature = features.getJSONObject(0); + + JSONObject geometry = feature.getJSONObject("geometry"); + assertEquals("LineString", geometry.getString("type")); + assertEquals("[[30,10],[10,30],[40,40]]", geometry.getJSONArray("coordinates").toString()); + + JSONObject properties = feature.getJSONObject("properties"); + assertEquals("W", properties.getString("osm_type")); + assertEquals("place", properties.getString("osm_key")); + assertEquals("city", properties.getString("osm_value")); + assertEquals("linestring", properties.getString("name")); + } } diff --git a/src/test/java/de/komoot/photon/DatabasePropertiesTest.java b/src/test/java/de/komoot/photon/DatabasePropertiesTest.java index 1ada97a82..1efd650cd 100644 --- a/src/test/java/de/komoot/photon/DatabasePropertiesTest.java +++ b/src/test/java/de/komoot/photon/DatabasePropertiesTest.java @@ -17,7 +17,7 @@ class DatabasePropertiesTest extends ESBaseTester { @Test void testSetLanguages() { var now = new Date(); - DatabaseProperties prop = new DatabaseProperties(new String[]{"en", "bg", "de"}, now, false); + DatabaseProperties prop = new DatabaseProperties(new String[]{"en", "bg", "de"}, now, false, false); assertArrayEquals(new String[]{"en", "bg", "de"}, prop.getLanguages()); assertEquals(now, prop.getImportDate()); @@ -28,7 +28,7 @@ void testSetLanguages() { */ @Test void testRestrictLanguagesUnsetLanguages() { - DatabaseProperties prop = new DatabaseProperties(null, null, false); + DatabaseProperties prop = new DatabaseProperties(null, null, false, false); prop.restrictLanguages(new String[]{"en", "bg", "de"}); assertArrayEquals(new String[]{"en", "bg", "de"}, prop.getLanguages()); @@ -40,7 +40,7 @@ void testRestrictLanguagesUnsetLanguages() { */ @Test void testRestrictLanguagesAlreadySet() { - DatabaseProperties prop = new DatabaseProperties(new String[]{"en", "de", "fr"}, null, false); + DatabaseProperties prop = new DatabaseProperties(new String[]{"en", "de", "fr"}, null, false, false); prop.restrictLanguages(new String[]{"cn", "de", "en", "es"}); diff --git a/src/test/java/de/komoot/photon/api/ApiLanguagesTest.java b/src/test/java/de/komoot/photon/api/ApiLanguagesTest.java index 9e821fe9f..91242b92d 100644 --- a/src/test/java/de/komoot/photon/api/ApiLanguagesTest.java +++ b/src/test/java/de/komoot/photon/api/ApiLanguagesTest.java @@ -48,7 +48,7 @@ protected PhotonDoc createDoc(int id, String key, String value, String... names) } private void importPlaces(String... languages) throws Exception { - setUpES(dataDirectory, languages); + setUpES(dataDirectory, false, languages); Importer instance = makeImporterWithLanguages(languages); instance.add(createDoc(1000, "place", "city", "name:en", "thething", "name:fr", "letruc", "name:ch", "dasding"), 0); diff --git a/src/test/java/de/komoot/photon/nominatim/NominatimConnectorDBTest.java b/src/test/java/de/komoot/photon/nominatim/NominatimConnectorDBTest.java index 6b32e449c..82fa15f55 100644 --- a/src/test/java/de/komoot/photon/nominatim/NominatimConnectorDBTest.java +++ b/src/test/java/de/komoot/photon/nominatim/NominatimConnectorDBTest.java @@ -38,7 +38,7 @@ void setup() { .build(); - connector = new NominatimImporter(null, 0, null, null, null, new H2DataAdapter()); + connector = new NominatimImporter(null, 0, null, null, null, new H2DataAdapter(), true); importer = new CollectingImporter(); jdbc = new JdbcTemplate(db); @@ -361,6 +361,13 @@ void testNoCountry() { } @Test + void testGeometry() { + PlacexTestRow place = new PlacexTestRow("building", "yes").name("Oosterbroek Zuivel").country("de").add(jdbc); + readEntireDatabase(); + assertEquals(1, importer.size()); + assertNotNull(importer.get(place).getGeometry()); + } + void testUsePostcodeFromPlacex() { PlacexTestRow parent = PlacexTestRow.make_street("Main St").add(jdbc); PlacexTestRow place = new PlacexTestRow("building", "yes") diff --git a/src/test/java/de/komoot/photon/nominatim/NominatimUpdaterDBTest.java b/src/test/java/de/komoot/photon/nominatim/NominatimUpdaterDBTest.java index 6c30218f5..228fb476f 100644 --- a/src/test/java/de/komoot/photon/nominatim/NominatimUpdaterDBTest.java +++ b/src/test/java/de/komoot/photon/nominatim/NominatimUpdaterDBTest.java @@ -29,7 +29,7 @@ void setup() { .build(); - connector = new NominatimUpdater(null, 0, null, null, null, new H2DataAdapter()); + connector = new NominatimUpdater(null, 0, null, null, null, new H2DataAdapter(), false); updater = new CollectingUpdater(); connector.setUpdater(updater); diff --git a/src/test/java/de/komoot/photon/nominatim/testdb/PlacexTestRow.java b/src/test/java/de/komoot/photon/nominatim/testdb/PlacexTestRow.java index a505d1901..a847d1709 100644 --- a/src/test/java/de/komoot/photon/nominatim/testdb/PlacexTestRow.java +++ b/src/test/java/de/komoot/photon/nominatim/testdb/PlacexTestRow.java @@ -23,6 +23,7 @@ public class PlacexTestRow { private Integer rankAddress = 30; private Integer rankSearch = 30; private String centroid; + private String geometry; private String postcode; private String countryCode = "us"; private Double importance = null; @@ -33,6 +34,7 @@ public PlacexTestRow(String key, String value) { this.value = value; osmId = placeId; centroid = "POINT (1.0 34.0)"; + geometry = "POLYGON ((6.4440619 52.1969454, 6.4441094 52.1969158, 6.4441408 52.1969347, 6.4441138 52.1969516, 6.4440933 52.1969643, 6.4440619 52.1969454))"; } public static PlacexTestRow make_street(String name) { @@ -123,11 +125,10 @@ public PlacexTestRow postcode(String postcode) { public PlacexTestRow add(JdbcTemplate jdbc) { jdbc.update("INSERT INTO placex (place_id, parent_place_id, osm_type, osm_id, class, type, rank_search, rank_address," - + " centroid, name, country_code, importance, address, postcode, indexed_status)" - + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ? FORMAT JSON, ?, ?, ? FORMAT JSON, ?, 0)", - placeId, parentPlaceId, osmType, osmId, key, value, rankSearch, rankAddress, centroid, + + " centroid, geometry, name, country_code, importance, address, postcode, indexed_status)" + + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? FORMAT JSON, ?, ?, ? FORMAT JSON, ?, 0)", + placeId, parentPlaceId, osmType, osmId, key, value, rankSearch, rankAddress, centroid, geometry, asJson(names), countryCode, importance, asJson(address), postcode); - return this; } diff --git a/src/test/java/de/komoot/photon/query/PhotonRequestFactoryTest.java b/src/test/java/de/komoot/photon/query/PhotonRequestFactoryTest.java index 1139f387a..8eff8e9aa 100644 --- a/src/test/java/de/komoot/photon/query/PhotonRequestFactoryTest.java +++ b/src/test/java/de/komoot/photon/query/PhotonRequestFactoryTest.java @@ -56,7 +56,7 @@ private void requestWithLayers(Request mockRequest, String... layers) { } private PhotonRequest createPhotonRequest(Request mockRequest) throws BadRequestException { - PhotonRequestFactory factory = new PhotonRequestFactory(Collections.singletonList("en"), "en", 10); + PhotonRequestFactory factory = new PhotonRequestFactory(Collections.singletonList("en"), "en", 10, true); return factory.create(mockRequest); } @@ -210,4 +210,20 @@ void testWithBadLayerFilters() { assertTrue(exception.getMessage().contains(expectedMessageFragment), String.format("Error message doesn not contain '%s': %s", expectedMessageFragment, exception.getMessage())); } + + @Test + void testWithGeometry() throws Exception { + Request mockRequest = createRequestWithQueryParams("q", "berlin"); + PhotonRequest photonRequest = createPhotonRequest(mockRequest); + + assertTrue(photonRequest.getReturnGeometry()); + } + + @Test + void testWithoutGeometry() throws Exception { + Request mockRequest = createRequestWithQueryParams("q", "berlin", "geometry", "false"); + PhotonRequest photonRequest = createPhotonRequest(mockRequest); + + assertFalse(photonRequest.getReturnGeometry()); + } } \ No newline at end of file diff --git a/src/test/java/de/komoot/photon/query/QueryByLanguageTest.java b/src/test/java/de/komoot/photon/query/QueryByLanguageTest.java index cead641c4..16ba85ba9 100644 --- a/src/test/java/de/komoot/photon/query/QueryByLanguageTest.java +++ b/src/test/java/de/komoot/photon/query/QueryByLanguageTest.java @@ -25,7 +25,7 @@ class QueryByLanguageTest extends ESBaseTester { private Importer setup(String... languages) throws IOException { languageList = languages; - setUpES(dataDirectory, languages); + setUpES(dataDirectory, false, languages); return makeImporterWithLanguages(languages); } diff --git a/src/test/java/de/komoot/photon/query/QueryFilterLayerTest.java b/src/test/java/de/komoot/photon/query/QueryFilterLayerTest.java index bff0a3373..5ad8d9380 100644 --- a/src/test/java/de/komoot/photon/query/QueryFilterLayerTest.java +++ b/src/test/java/de/komoot/photon/query/QueryFilterLayerTest.java @@ -26,7 +26,7 @@ class QueryFilterLayerTest extends ESBaseTester { @BeforeAll void setUp() throws Exception { - setUpES(instanceTestDirectory, "en", "de", "fr"); + setUpES(instanceTestDirectory, false, "en", "de", "fr"); Importer instance = makeImporter(); int id = 0; diff --git a/src/test/java/de/komoot/photon/query/QueryFilterTagValueTest.java b/src/test/java/de/komoot/photon/query/QueryFilterTagValueTest.java index ea3859d4e..1bd2641bd 100644 --- a/src/test/java/de/komoot/photon/query/QueryFilterTagValueTest.java +++ b/src/test/java/de/komoot/photon/query/QueryFilterTagValueTest.java @@ -36,7 +36,7 @@ class QueryFilterTagValueTest extends ESBaseTester { @BeforeAll void setUp() throws Exception { - setUpES(instanceTestDirectory, "en", "de", "fr"); + setUpES(instanceTestDirectory, false, "en", "de", "fr"); Importer instance = makeImporter(); double lon = 13.38886; double lat = 52.51704; diff --git a/src/test/java/de/komoot/photon/query/QueryGeometryTest.java b/src/test/java/de/komoot/photon/query/QueryGeometryTest.java new file mode 100644 index 000000000..cff061e6a --- /dev/null +++ b/src/test/java/de/komoot/photon/query/QueryGeometryTest.java @@ -0,0 +1,82 @@ +package de.komoot.photon.query; + +import de.komoot.photon.ESBaseTester; +import de.komoot.photon.Importer; +import de.komoot.photon.PhotonDoc; +import de.komoot.photon.searcher.PhotonResult; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.locationtech.jts.geom.Coordinate; +import org.locationtech.jts.geom.Point; +import org.locationtech.jts.io.ParseException; +import org.locationtech.jts.io.WKTReader; + +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.assertNotNull; + +/** + * Tests that the database backend produces queries which can find all + * expected results. These tests do not check relevance. + */ +class QueryGeometryTest extends ESBaseTester { + private int testDocId = 10000; + + @BeforeEach + void setup() throws IOException { + setUpESWithGeometry(); + } + + private PhotonDoc createDoc(String... names) throws ParseException { + Map nameMap = new HashMap<>(); + + for (int i = 0; i < names.length - 1; i += 2) { + nameMap.put(names[i], names[i+1]); + } + + ++testDocId; + return new PhotonDoc(testDocId, "N", testDocId, "place", "city").names(nameMap); + } + + private List search(String query) { + return getServer().createSearchHandler(new String[]{"en"}, 1).search(new PhotonRequest(query, "en")); + } + + + @Test + void testSearchGetPolygon() throws IOException, ParseException { + Importer instance = makeImporter(); + Point location = FACTORY.createPoint(new Coordinate(1.0, 2.34)); + PhotonDoc doc = createDoc("name", "Muffle Flu").geometry(new WKTReader().read("POLYGON ((30 10, 40 40, 20 40, 10 20, 30 10))")).centroid(location); + instance.add(doc, 0); + instance.finish(); + refresh(); + List s = search("muffle flu"); + + if (s.get(0).getClass().getName().equals("de.komoot.photon.opensearch.OpenSearchResult")) { + assertNotNull(s.get(0).getGeometry()); + } else { + assertNotNull(s.get(0).get("geometry")); + } + } + + @Test + void testSearchGetLineString() throws IOException, ParseException { + Importer instance = makeImporter(); + Point location = FACTORY.createPoint(new Coordinate(1.0, 2.34)); + PhotonDoc doc = createDoc("name", "Muffle Flu").geometry(new WKTReader().read("LINESTRING (30 10, 10 30, 40 40)")).centroid(location); + instance.add(doc, 0); + instance.finish(); + refresh(); + List s = search("muffle flu"); + + if (s.get(0).getClass().getName().equals("de.komoot.photon.opensearch.OpenSearchResult")) { + assertNotNull(s.get(0).getGeometry()); + } else { + assertNotNull(s.get(0).get("geometry")); + } + } +} diff --git a/src/test/java/de/komoot/photon/query/QueryReverseFilterLayerTest.java b/src/test/java/de/komoot/photon/query/QueryReverseFilterLayerTest.java index 5efac0bbc..883dc8a3b 100644 --- a/src/test/java/de/komoot/photon/query/QueryReverseFilterLayerTest.java +++ b/src/test/java/de/komoot/photon/query/QueryReverseFilterLayerTest.java @@ -28,7 +28,7 @@ class QueryReverseFilterLayerTest extends ESBaseTester { @BeforeAll void setup() throws IOException { - setUpES(instanceTestDirectory, "en"); + setUpES(instanceTestDirectory, false,"en"); Importer instance = makeImporter(); @@ -52,7 +52,7 @@ private List reverse(String... layers) { Point pt = FACTORY.createPoint(new Coordinate(10, 10)); Set layerSet = Arrays.stream(layers).collect(Collectors.toSet()); - ReverseRequest request = new ReverseRequest(pt, "en", 1.0, "", 10, true, layerSet, false); + ReverseRequest request = new ReverseRequest(pt, "en", 1.0, "", 10, true, layerSet, false, false); return getServer().createReverseHandler(1).reverse(request); } diff --git a/src/test/java/de/komoot/photon/query/QueryReverseFilterTagValueTest.java b/src/test/java/de/komoot/photon/query/QueryReverseFilterTagValueTest.java index 4f4a46fe0..0dd82932f 100644 --- a/src/test/java/de/komoot/photon/query/QueryReverseFilterTagValueTest.java +++ b/src/test/java/de/komoot/photon/query/QueryReverseFilterTagValueTest.java @@ -22,6 +22,7 @@ import de.komoot.photon.PhotonDoc; import de.komoot.photon.searcher.PhotonResult; import de.komoot.photon.searcher.TagFilter; +import org.locationtech.jts.io.ParseException; @TestInstance(TestInstance.Lifecycle.PER_CLASS) class QueryReverseFilterTagValueTest extends ESBaseTester { @@ -39,8 +40,8 @@ class QueryReverseFilterTagValueTest extends ESBaseTester { "railway", "station"}; @BeforeAll - void setup() throws IOException { - setUpES(instanceTestDirectory, "en", "de", "fr"); + void setup() throws IOException, ParseException { + setUpES(instanceTestDirectory, false, "en", "de", "fr"); Importer instance = makeImporter(); double lon = 13.38886; double lat = 52.51704; @@ -68,7 +69,7 @@ public void tearDown() throws IOException { private List reverseWithTags(String[] params) { Point pt = FACTORY.createPoint(new Coordinate(13.38886, 52.51704)); - ReverseRequest request = new ReverseRequest(pt, "en", 1.0, "", 50, true, new HashSet<>(), false); + ReverseRequest request = new ReverseRequest(pt, "en", 1.0, "", 50, true, new HashSet<>(), false, false); for (String param : params) { request.addOsmTagFilter(TagFilter.buildOsmTagFilter(param)); } diff --git a/src/test/java/de/komoot/photon/query/QueryReverseTest.java b/src/test/java/de/komoot/photon/query/QueryReverseTest.java index 7d96ff781..0c841bdcf 100644 --- a/src/test/java/de/komoot/photon/query/QueryReverseTest.java +++ b/src/test/java/de/komoot/photon/query/QueryReverseTest.java @@ -12,6 +12,7 @@ import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; +import org.locationtech.jts.io.ParseException; import java.io.IOException; import java.nio.file.Path; @@ -26,8 +27,8 @@ class QueryReverseTest extends ESBaseTester { private static Path instanceTestDirectory; @BeforeAll - void setup() throws IOException { - setUpES(instanceTestDirectory, "en"); + void setup() throws IOException, ParseException { + setUpES(instanceTestDirectory, false, "en"); Importer instance = makeImporter(); instance.add(createDoc(10,10, 100, 100, "place", "house"), 0); @@ -48,7 +49,7 @@ private List reverse(double lon, double lat, double radius, int li Point pt = FACTORY.createPoint(new Coordinate(lon, lat)); return getServer().createReverseHandler(1).reverse( - new ReverseRequest(pt, "en", radius, "", limit, true, new HashSet<>(), false) + new ReverseRequest(pt, "en", radius, "", limit, true, new HashSet<>(), false, false) ); } diff --git a/src/test/java/de/komoot/photon/searcher/GeocodeJsonFormatterTest.java b/src/test/java/de/komoot/photon/searcher/GeocodeJsonFormatterTest.java index eaa8e6333..71851aa17 100644 --- a/src/test/java/de/komoot/photon/searcher/GeocodeJsonFormatterTest.java +++ b/src/test/java/de/komoot/photon/searcher/GeocodeJsonFormatterTest.java @@ -13,12 +13,36 @@ class GeocodeJsonFormatterTest { @Test - void testConvertToGeojson() { - GeocodeJsonFormatter formatter = new GeocodeJsonFormatter(false, "en"); + void testConvertPointToGeojson() { + GeocodeJsonFormatter formatter = new GeocodeJsonFormatter(false, "en", false); + List allPointResults = new ArrayList<>(); + allPointResults.add(createDummyPointResult("99999", "Park Foo", "leisure", "park")); + allPointResults.add(createDummyPointResult("88888", "Bar Park", "leisure", "park")); + + // Test Points + String geojsonString = formatter.convert(allPointResults, null); + JSONObject jsonObj = new JSONObject(geojsonString); + assertEquals("FeatureCollection", jsonObj.getString("type")); + JSONArray features = jsonObj.getJSONArray("features"); + assertEquals(2, features.length()); + for (int i = 0; i < features.length(); i++) { + JSONObject feature = features.getJSONObject(i); + assertEquals("Feature", feature.getString("type")); + assertEquals("Point", feature.getJSONObject("geometry").getString("type")); + assertEquals("leisure", feature.getJSONObject("properties").getString(Constants.OSM_KEY)); + assertEquals("park", feature.getJSONObject("properties").getString(Constants.OSM_VALUE)); + } + } + + @Test + void testConvertGeometryToGeojson() { + GeocodeJsonFormatter formatter = new GeocodeJsonFormatter(false, "en", true); + List allResults = new ArrayList<>(); - allResults.add(createDummyResult("99999", "Park Foo", "leisure", "park")); - allResults.add(createDummyResult("88888", "Bar Park", "leisure", "park")); + allResults.add(createDummyGeometryResult("99999", "Park Foo", "leisure", "park")); + allResults.add(createDummyGeometryResult("88888", "Bar Park", "leisure", "park")); + // Test Geometry String geojsonString = formatter.convert(allResults, null); JSONObject jsonObj = new JSONObject(geojsonString); assertEquals("FeatureCollection", jsonObj.getString("type")); @@ -27,19 +51,32 @@ void testConvertToGeojson() { for (int i = 0; i < features.length(); i++) { JSONObject feature = features.getJSONObject(i); assertEquals("Feature", feature.getString("type")); - assertEquals("Point", feature.getJSONObject("geometry").getString("type")); + assertEquals("MultiPolygon", feature.getJSONObject("geometry").getString("type")); assertEquals("leisure", feature.getJSONObject("properties").getString(Constants.OSM_KEY)); assertEquals("park", feature.getJSONObject("properties").getString(Constants.OSM_VALUE)); } } - private PhotonResult createDummyResult(String postCode, String name, String osmKey, - String osmValue) { + private PhotonResult createDummyPointResult(String postCode, String name, String osmKey, + String osmValue) { + return new MockPhotonResult() + .put(Constants.POSTCODE, postCode) + .putLocalized(Constants.NAME, "en", name) + .put(Constants.OSM_KEY, osmKey) + .put(Constants.OSM_VALUE, osmValue) + .put("geometry", new JSONObject() + .put("type", "Point") + .put("coordinates", new double[]{42, 21})); + } + + private PhotonResult createDummyGeometryResult(String postCode, String name, String osmKey, + String osmValue) { return new MockPhotonResult() .put(Constants.POSTCODE, postCode) .putLocalized(Constants.NAME, "en", name) .put(Constants.OSM_KEY, osmKey) - .put(Constants.OSM_VALUE, osmValue); + .put(Constants.OSM_VALUE, osmValue) + .put(Constants.GEOMETRY, new JSONObject("{\"type\":\"MultiPolygon\",\"coordinates\":[[[[-100.0,40.0],[-100.0,45.0],[-90.0,45.0],[-90.0,40.0],[-100.0,40.0]]],[[[-80.0,35.0],[-80.0,40.0],[-70.0,40.0],[-70.0,35.0],[-80.0,35.0]]]]}")); } } diff --git a/src/test/java/de/komoot/photon/searcher/MockPhotonResult.java b/src/test/java/de/komoot/photon/searcher/MockPhotonResult.java index 02efcc581..613f92947 100644 --- a/src/test/java/de/komoot/photon/searcher/MockPhotonResult.java +++ b/src/test/java/de/komoot/photon/searcher/MockPhotonResult.java @@ -9,6 +9,7 @@ public class MockPhotonResult implements PhotonResult { final Map data = new HashMap<>(); final double[] coordinates = new double[]{42, 21}; + final String geometry = "{\"type\":\"MultiPolygon\",\"coordinates\":[[[[-100.0,40.0],[-100.0,45.0],[-90.0,45.0],[-90.0,40.0],[-100.0,40.0]]],[[[-80.0,35.0],[-80.0,40.0],[-70.0,40.0],[-70.0,35.0],[-80.0,35.0]]]]}"; final double[] extent = new double[]{0, 1, 2, 3}; final Map localized = new HashMap<>(); @@ -32,6 +33,11 @@ public double[] getCoordinates() { return coordinates; } + @Override + public String getGeometry() { + return geometry; + } + @Override public double[] getExtent() { return extent;