Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add option for polygon/linestring in results #823

Open
wants to merge 24 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
24 commits
Select commit Hold shift + click to select a range
9ec20d4
feat: initial support for geometry column
Jun 18, 2024
5a7fb1a
feat: add unit test
Jul 9, 2024
31e2111
feat: add geometry column as an commanline argument
Jul 11, 2024
97899e8
Merge branch 'master' into feature/geometry_support
Jul 11, 2024
031d962
bugfix: undo intellij magic
Jul 11, 2024
152c2d1
bugfix: minor build bugs
Jul 11, 2024
5d1b854
feat: add polygon as an api request instead of a default
Jul 11, 2024
d544bb3
feat: add unit tests and process remarks of @lonvia
Sep 5, 2024
41c20c4
bugfix: minor fixes
Sep 5, 2024
cef09d7
bugfix: typo
Sep 5, 2024
053c3fb
Merge remote-tracking branch 'origin' into feature/geometry_support
Nov 16, 2024
161c967
fix: OpenSearch implementation of Geometry column
Nov 16, 2024
583d575
fix: remarks of lonvia
Nov 16, 2024
0abb0b4
fix: unit tests
Nov 16, 2024
6f53266
Merge remote-tracking branch 'origin' into feature/geometry_support
Dec 5, 2024
db58e9a
fix: opensearch implementation
Dec 14, 2024
17c2c0d
fix: latest remarks of lonvia
Jan 12, 2025
3918258
Merge branch 'komoot:master' into feature/geometry_support
red-fenix Jan 25, 2025
34cd622
Merge remote-tracking branch 'origin/master' into feature/geometry_su…
Feb 22, 2025
eff3b06
feat: add support for all polygon types
Feb 23, 2025
a5ac9b4
bugfix: rename all "polygon" references to "geometry"
Mar 11, 2025
e6570c5
bugfix: set correct crs/srid on geometry import
Mar 12, 2025
9aa1117
bugfix: fix Photondoc constructors
Mar 12, 2025
216e7e6
Merge branch 'master' into feature/geometry_support
Mar 12, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions app/es_embedded/es/mappings.json
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,9 @@
"coordinate": {
"type": "geo_point"
},
"geometry": {
"type": "geo_shape"
},
"country": {
"properties": {
"default": {
Expand Down
18 changes: 14 additions & 4 deletions app/es_embedded/src/main/java/de/komoot/photon/Server.java
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@ public class Server {
private static final String FIELD_VERSION = "database_version";
private static final String FIELD_LANGUAGES = "indexed_languages";
private static final String FIELD_IMPORT_DATE = "import_date";
private static final String FIELD_SUPPORT_GEOMETRIES = "support_geometries";

private Node esNode;

Expand Down Expand Up @@ -177,14 +178,18 @@ private void setupDirectories(URL directoryName) throws IOException, URISyntaxEx

}

public DatabaseProperties recreateIndex(String[] languages, Date importDate, boolean supportStructuredQueries) throws IOException {
public DatabaseProperties recreateIndex(String[] languages, Date importDate, boolean supportStructuredQueries, boolean supportGeometries) throws IOException {
deleteIndex();

loadIndexSettings().createIndex(esClient, PhotonIndex.NAME);

createAndPutIndexMapping(languages, supportStructuredQueries);

DatabaseProperties dbProperties = new DatabaseProperties(languages, importDate, false);
DatabaseProperties dbProperties = new DatabaseProperties()
.setLanguages(languages)
.setImportDate(importDate)
.setSupportGeometries(supportGeometries);

saveToDatabase(dbProperties);

return dbProperties;
Expand Down Expand Up @@ -239,6 +244,7 @@ public void saveToDatabase(DatabaseProperties dbProperties) throws IOException
.field(FIELD_VERSION, DATABASE_VERSION)
.field(FIELD_LANGUAGES, String.join(",", dbProperties.getLanguages()))
.field(FIELD_IMPORT_DATE, dbProperties.getImportDate() instanceof Date ? dbProperties.getImportDate().toInstant() : null)
.field(FIELD_SUPPORT_GEOMETRIES, Boolean.toString(dbProperties.getSupportGeometries()))
.endObject().endObject();

esClient.prepareIndex(PhotonIndex.NAME, PhotonIndex.TYPE).
Expand Down Expand Up @@ -276,11 +282,15 @@ public DatabaseProperties loadFromDatabase() {
}

String langString = properties.get(FIELD_LANGUAGES);

String importDateString = properties.get(FIELD_IMPORT_DATE);

String supportGeometries = properties.get(FIELD_SUPPORT_GEOMETRIES);

return new DatabaseProperties(langString == null ? null : langString.split(","),
importDateString == null ? null : Date.from(Instant.parse(importDateString)),
false);
importDateString == null ? null : Date.from(Instant.parse(importDateString)),
false,
Boolean.parseBoolean(supportGeometries));
}

public Importer createImporter(String[] languages, String[] extraTags) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,11 @@ public double[] getCoordinates() {
return new double[]{coordinate.get(Constants.LON), coordinate.get(Constants.LAT)};
}

@Override
public String getGeometry() {
return (String) result.getSource().get("geometry");
}

@Override
public double[] getExtent() {
final Map<String, Object> extent = (Map<String, Object>) result.getSource().get("extent");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,11 @@
import de.komoot.photon.Constants;
import de.komoot.photon.PhotonDoc;
import de.komoot.photon.nominatim.model.AddressType;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;

import org.elasticsearch.common.xcontent.*;
import org.elasticsearch.common.xcontent.json.JsonXContent;
import org.locationtech.jts.geom.Envelope;
import org.locationtech.jts.io.geojson.GeoJsonWriter;

import java.io.IOException;
import java.util.HashMap;
Expand Down Expand Up @@ -38,6 +40,17 @@ public static XContentBuilder convert(PhotonDoc doc, String[] languages, String[
.endObject();
}

if (doc.getGeometry() != null) {
GeoJsonWriter g = new GeoJsonWriter();

XContentParser parser = JsonXContent
.jsonXContent
.createParser(NamedXContentRegistry.EMPTY, g.write(doc.getGeometry()));

builder.field("geometry");
builder.copyCurrentStructure(parser);
}

if (doc.getHouseNumber() != null) {
builder.field("housenumber", doc.getHouseNumber());
}
Expand Down
16 changes: 11 additions & 5 deletions app/es_embedded/src/test/java/de/komoot/photon/ESBaseTester.java
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@
import de.komoot.photon.searcher.PhotonResult;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.io.TempDir;
import org.locationtech.jts.io.ParseException;
import org.locationtech.jts.io.WKTReader;

import java.io.IOException;
import java.nio.file.Path;
Expand All @@ -25,9 +27,9 @@ public class ESBaseTester {

private ElasticTestServer server;

protected PhotonDoc createDoc(double lon, double lat, int id, int osmId, String key, String value) {
protected PhotonDoc createDoc(double lon, double lat, int id, int osmId, String key, String value) throws ParseException {
Point location = FACTORY.createPoint(new Coordinate(lon, lat));
return new PhotonDoc(id, "W", osmId, key, value).names(Collections.singletonMap("name", "berlin")).centroid(location);
return new PhotonDoc(id, "W", osmId, key, value).names(Collections.singletonMap("name", "berlin")).centroid(location).geometry(new WKTReader().read("POLYGON ((6.4440619 52.1969454, 6.4441094 52.1969158, 6.4441408 52.1969347, 6.4441138 52.1969516, 6.4440933 52.1969643, 6.4440619 52.1969454))"));
}

protected PhotonResult getById(int id) {
Expand All @@ -45,17 +47,21 @@ public void tearDown() throws IOException {
}

public void setUpES() throws IOException {
setUpES(dataDirectory, "en");
setUpES(dataDirectory, false,"en");
}

public void setUpESWithGeometry() throws IOException {
setUpES(dataDirectory, true,"en");
}
/**
* Setup the ES server
*
* @throws IOException
*/
public void setUpES(Path testDirectory, String... languages) throws IOException {
public void setUpES(Path testDirectory, boolean supportGeometries, String... languages) throws IOException {
server = new ElasticTestServer(testDirectory.toString());
server.start(TEST_CLUSTER_NAME, new String[]{});
server.recreateIndex(languages, new Date(), false);
server.recreateIndex(languages, new Date(), false, supportGeometries);
refresh();
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,10 @@ public double[] getCoordinates() {
throw new NotImplementedException();
}

public String getGeometry() {
throw new NotImplementedException();
}

@Override
public double[] getExtent() {
throw new NotImplementedException();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ protected PhotonDoc createDoc(double lon, double lat, int id, int osmId, String

@BeforeAll
void setUp() throws Exception {
setUpES(instanceTestDirectory, "en", "de", "fr", "it");
setUpES(instanceTestDirectory, false, "en", "de", "fr", "it");
Importer instance = getServer().createImporter(new String[]{"en", "de", "fr", "it"},
new String[]{"population", "capital"});

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@
import de.komoot.photon.searcher.PhotonResult;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.locationtech.jts.io.ParseException;
import org.locationtech.jts.io.WKTReader;

import java.io.IOException;
import java.util.Collections;
Expand All @@ -22,10 +24,11 @@ public void setUp() throws IOException {
}

@Test
void testAddSimpleDoc() {
void testAddSimpleDoc() throws ParseException {
Importer instance = makeImporterWithExtra("");

instance.add(new PhotonDoc(1234, "N", 1000, "place", "city")
.geometry(new WKTReader().read("MULTIPOLYGON (((6.111933 51.2659309, 6.1119417 51.2659247, 6.1119554 51.2659249, 6.1119868 51.2659432, 6.111964 51.2659591, 6.1119333 51.2659391, 6.111933 51.2659309)))"))
.extraTags(Collections.singletonMap("maxspeed", "100")), 0);
instance.finish();
refresh();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ void testSaveAndLoadFromDatabase() throws IOException {
setUpES();

Date now = new Date();
DatabaseProperties prop = new DatabaseProperties(new String[]{"en", "de", "fr"}, now, false);
DatabaseProperties prop = new DatabaseProperties(new String[]{"en", "de", "fr"}, now, false, false);
getServer().saveToDatabase(prop);

prop = getServer().loadFromDatabase();
Expand Down
2 changes: 1 addition & 1 deletion app/opensearch/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ dependencies {
implementation 'org.apache.httpcomponents.client5:httpclient5:5.4.1'
implementation 'com.fasterxml.jackson.core:jackson-databind:2.18.2'

implementation('org.codelibs.opensearch:opensearch-runner:2.18.0.0') {
implementation('org.codelibs.opensearch:opensearch-runner:2.18.0.1') {
exclude(module: 'repository-url')
exclude(module: 'reindex-client')
exclude(module: 'rank-eval-client')
Expand Down
16 changes: 9 additions & 7 deletions app/opensearch/src/main/java/de/komoot/photon/Server.java
Original file line number Diff line number Diff line change
Expand Up @@ -33,9 +33,11 @@ public class Server {

private static final Logger LOGGER = org.slf4j.LoggerFactory.getLogger(Server.class);

public static final String OPENSEARCH_MODULES =
"org.opensearch.transport.Netty4Plugin,"
+ "org.opensearch.analysis.common.CommonAnalysisPlugin";
// public static final String OPENSEARCH_MODULES =
// "org.opensearch.transport.Netty4Plugin,"
// + "org.opensearch.analysis.common.CommonAnalysisPlugin,"
// + "org.opensearch.geo.GeoModulePlugin,"
// + "org.opensearch.geospatial.plugin.GeospatialPlugin";

protected OpenSearchClient client;
private OpenSearchRunner runner = null;
Expand Down Expand Up @@ -86,7 +88,6 @@ private HttpHost[] startInternal(String clusterName) {
.basePath(dataDirectory)
.clusterName(clusterName)
.numOfNode(1)
.moduleTypes(OPENSEARCH_MODULES)
);

runner.ensureYellow();
Expand Down Expand Up @@ -119,7 +120,7 @@ public void shutdown() {
}
}

public DatabaseProperties recreateIndex(String[] languages, Date importDate, boolean supportStructuredQueries) throws IOException {
public DatabaseProperties recreateIndex(String[] languages, Date importDate, boolean supportStructuredQueries, boolean supportGeometries) throws IOException {
// delete any existing data
if (client.indices().exists(e -> e.index(PhotonIndex.NAME)).value()) {
client.indices().delete(d -> d.index(PhotonIndex.NAME));
Expand All @@ -129,7 +130,7 @@ public DatabaseProperties recreateIndex(String[] languages, Date importDate, boo

(new IndexMapping(supportStructuredQueries)).addLanguages(languages).putMapping(client, PhotonIndex.NAME);

var dbProperties = new DatabaseProperties(languages, importDate, supportStructuredQueries);
var dbProperties = new DatabaseProperties(languages, importDate, supportStructuredQueries, supportGeometries);
saveToDatabase(dbProperties);

return dbProperties;
Expand Down Expand Up @@ -180,7 +181,8 @@ public DatabaseProperties loadFromDatabase() throws IOException {

return new DatabaseProperties(dbEntry.source().languages,
dbEntry.source().importDate,
dbEntry.source().supportStructuredQueries);
dbEntry.source().supportStructuredQueries,
dbEntry.source().supportGeometries);
}

public Importer createImporter(String[] languages, String[] extraTags) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ public class DBPropertyEntry {
public Date importDate;
public String[] languages;
public boolean supportStructuredQueries;
public boolean supportGeometries;

public DBPropertyEntry() {}

Expand All @@ -17,5 +18,6 @@ public DBPropertyEntry(DatabaseProperties props, String databaseVersion) {
importDate = props.getImportDate();
languages = props.getLanguages();
supportStructuredQueries = props.getSupportStructuredQueries();
supportGeometries = props.getSupportGeometries();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@
import org.opensearch.client.opensearch.OpenSearchClient;
import org.opensearch.client.opensearch._types.Time;
import org.opensearch.client.opensearch.core.BulkRequest;
import org.opensearch.client.opensearch.core.bulk.BulkOperation;
import org.opensearch.client.opensearch.core.bulk.BulkResponseItem;
import org.slf4j.Logger;

import java.io.IOException;
Expand Down Expand Up @@ -54,7 +56,14 @@ private void saveDocuments() {
var response = client.bulk(bulkRequest.build());

if (response.errors()) {
LOGGER.error("Error during bulk import.");
for (BulkResponseItem bri: response.items()) {
LOGGER.error("Error during bulk import.");
if (bri.error() != null) {
LOGGER.error(bri.error().reason());
LOGGER.error(bri.error().type());
LOGGER.error(bri.error().stackTrace());
}
}
}
} catch (IOException e) {
LOGGER.error("Error during bulk import", e);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,7 @@ private void setupBaseMappings() {
}

mappings.properties("coordinate", b -> b.geoPoint(p -> p));
mappings.properties("geometry", b -> b.geoShape(p -> p));
mappings.properties("countrycode", b -> b.keyword(p -> p.index(true)));
mappings.properties("importance", b -> b.float_(p -> p.index(false)));

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,14 +11,16 @@ public class OpenSearchResult implements PhotonResult {
private double score = 0.0;
private final double[] extent;
private final double[] coordinates;
private final String geometry;
private final Map<String, Object> infos;
private final Map<String, Map<String, String>> localeTags;

OpenSearchResult(double[] extent, double[] coordinates, Map<String, Object> infos, Map<String, Map<String, String>> localeTags) {
OpenSearchResult(double[] extent, double[] coordinates, Map<String, Object> infos, Map<String, Map<String, String>> localeTags, String geometry) {
this.extent = extent;
this.coordinates = coordinates;
this.infos = infos;
this.localeTags = localeTags;
this.geometry = geometry;
}

public OpenSearchResult setScore(double score) {
Expand Down Expand Up @@ -61,6 +63,10 @@ public double[] getCoordinates() {
return coordinates;
}

public String getGeometry() {
return geometry;
}

@Override
public double[] getExtent() {
return extent;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,11 @@ public OpenSearchResult deserialize(JsonParser p, DeserializationContext ctxt) t
final Map<String, Object> tags = new HashMap<>();
final Map<String, Map<String, String>> localeTags = new HashMap<>();

String geometry = null;
if (node.get("geometry") != null) {
geometry = node.get("geometry").toString();
}

var fields = node.fields();
while (fields.hasNext()) {
final var entry = fields.next();
Expand All @@ -55,7 +60,7 @@ public OpenSearchResult deserialize(JsonParser p, DeserializationContext ctxt) t
}
}

return new OpenSearchResult(extent, coordinates, tags, localeTags);
return new OpenSearchResult(extent, coordinates, tags, localeTags, geometry);
}

private double[] extractExtent(ObjectNode node) {
Expand All @@ -78,5 +83,4 @@ private double[] extractCoordinate(ObjectNode node) {

return new double[]{node.get(Constants.LON).doubleValue(), node.get(Constants.LAT).doubleValue()};
}

}
Loading
Loading