From 6b9a1286c247484b3d94a1e1116abc3c324c98a9 Mon Sep 17 00:00:00 2001 From: Erica Fischer Date: Tue, 7 Jan 2025 16:38:07 -0800 Subject: [PATCH] Get the attributes back in the right order --- tests/join-sql/countries.pmtiles.json | 6 +- tile-join.cpp | 198 +++++++++++++++++--------- 2 files changed, 134 insertions(+), 70 deletions(-) diff --git a/tests/join-sql/countries.pmtiles.json b/tests/join-sql/countries.pmtiles.json index f2cce016..de215f8d 100644 --- a/tests/join-sql/countries.pmtiles.json +++ b/tests/join-sql/countries.pmtiles.json @@ -15,11 +15,11 @@ }, "features": [ { "type": "FeatureCollection", "properties": { "zoom": 0, "x": 0, "y": 0 }, "features": [ { "type": "FeatureCollection", "properties": { "layer": "parsed-bboxes", "version": 2, "extent": 1073741824 }, "features": [ -{ "type": "Feature", "id": 322, "properties": { "another": "yes", "country": "Germany", "fid": 1, "something": "blah", "ne10-admin0:name_en": "germany", "ne10-admin0:iso_a2_eh": "de", "ne10-admin0:iso_a3_eh": "deu", "felt:cluster_size": 1 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 15.022060, 47.271121 ], [ 5.852490, 47.271121 ], [ 5.852490, 55.065334 ], [ 15.022060, 55.065334 ], [ 15.022060, 47.271121 ] ] ] } } +{ "type": "Feature", "id": 322, "properties": { "ne10-admin0:name_en": "germany", "ne10-admin0:iso_a2_eh": "de", "ne10-admin0:iso_a3_eh": "deu", "felt:cluster_size": 1, "another": "yes", "country": "Germany", "fid": 1, "something": "blah" }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 15.022060, 47.271121 ], [ 5.852490, 47.271121 ], [ 5.852490, 55.065334 ], [ 15.022060, 55.065334 ], [ 15.022060, 47.271121 ] ] ] } } , -{ "type": "Feature", "id": 435, "properties": { "another": "why", "country": "France", "fid": 2, "something": "what", "ne10-admin0:name_en": "france", "ne10-admin0:iso_a2_eh": "fr", "ne10-admin0:iso_a3_eh": "fra", "felt:cluster_size": 1 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ -61.797841, -21.370782 ], [ -61.797841, 51.087541 ], [ 55.854503, 51.087541 ], [ 55.854503, -21.370782 ], [ -61.797841, -21.370782 ] ] ] } } +{ "type": "Feature", "id": 435, "properties": { "ne10-admin0:name_en": "france", "ne10-admin0:iso_a2_eh": "fr", "ne10-admin0:iso_a3_eh": "fra", "felt:cluster_size": 1, "another": "why", "country": "France", "fid": 2, "something": "what" }, "geometry": { "type": "Polygon", "coordinates": [ [ [ -61.797841, -21.370782 ], [ -61.797841, 51.087541 ], [ 55.854503, 51.087541 ], [ 55.854503, -21.370782 ], [ -61.797841, -21.370782 ] ] ] } } , -{ "type": "Feature", "id": 650, "properties": { "another": "bar", "country": "Italy", "fid": 3, "something": "foo", "ne10-admin0:name_en": "italy", "ne10-admin0:iso_a2_eh": "it", "ne10-admin0:iso_a3_eh": "ita", "felt:cluster_size": 1 }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 18.517426, 35.489244 ], [ 6.602728, 35.489244 ], [ 6.602728, 47.085215 ], [ 18.517426, 47.085215 ], [ 18.517426, 35.489244 ] ] ] } } +{ "type": "Feature", "id": 650, "properties": { "ne10-admin0:name_en": "italy", "ne10-admin0:iso_a2_eh": "it", "ne10-admin0:iso_a3_eh": "ita", "felt:cluster_size": 1, "another": "bar", "country": "Italy", "fid": 3, "something": "foo" }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 18.517426, 35.489244 ], [ 6.602728, 35.489244 ], [ 6.602728, 47.085215 ], [ 18.517426, 47.085215 ], [ 18.517426, 35.489244 ] ] ] } } ] } ] } ] } diff --git a/tile-join.cpp b/tile-join.cpp index 14c95e8d..a0c75faf 100644 --- a/tile-join.cpp +++ b/tile-join.cpp @@ -294,16 +294,17 @@ void append_tile(std::string message, int z, unsigned x, unsigned y, std::map> attributes; + std::vector key_order; + }; - if (feat.has_id) { - outfeature.has_id = true; - outfeature.id = feat.id; - } + std::vector matches; + bool matched = false; - std::map> attributes; - std::vector key_order; + // start filling out sql matches if (f < joined.size()) { if (joined[f].size() > 0) { @@ -311,90 +312,149 @@ void append_tile(std::string message, int z, unsigned x, unsigned y, std::map>(key, std::pair(val, sv))); + m.key_order.push_back(key); + } + } + } + for (auto const &kv : joined_feature) { if (kv.first == attribute_for_id) { - outfeature.has_id = true; - outfeature.id = mvt_value_to_long_long(kv.second); + m.has_id = true; + m.id = mvt_value_to_long_long(kv.second); } else if (include.count(kv.first) || (!exclude_all && exclude.count(kv.first) == 0 && exclude_attributes.count(kv.first) == 0)) { if (kv.second.type != mvt_null) { - attributes.insert(std::pair>(kv.first, std::pair(kv.second, mvt_value_to_serial_val(kv.second)))); - key_order.push_back(kv.first); + m.attributes.insert(std::pair>(kv.first, std::pair(kv.second, mvt_value_to_serial_val(kv.second)))); + m.key_order.push_back(kv.first); } } } + + matches.push_back(m); } } - for (size_t t = 0; t + 1 < feat.tags.size(); t += 2) { - const std::string &key = layer.keys[feat.tags[t]]; - mvt_value &val = layer.values[feat.tags[t + 1]]; - serial_val sv = mvt_value_to_serial_val(val); + // look for csv matches and start filling them out - if (sv.type == mvt_null) { - continue; - } + if (!matched) { + match m; + m.id = feat.id; + m.has_id = feat.has_id; + // populate attributes and key_order as we look for matches, + // because apparently at some point i thought it was important + // to insert the joined attributes at the point in the sequence + // where the join key had been - if (!exclude_all_tile_attributes) { - if (include.count(key) || (!exclude_all && exclude.count(key) == 0 && exclude_attributes.count(key) == 0)) { - attributes.insert(std::pair>(key, std::pair(val, sv))); - key_order.push_back(key); + for (size_t t = 0; t + 1 < feat.tags.size(); t += 2) { + const std::string &key = layer.keys[feat.tags[t]]; + mvt_value &val = layer.values[feat.tags[t + 1]]; + serial_val sv = mvt_value_to_serial_val(val); + + if (val.type == mvt_null) { + continue; + } + + if (!exclude_all_tile_attributes) { + if (include.count(std::string(key)) || (!exclude_all && exclude.count(std::string(key)) == 0 && exclude_attributes.count(std::string(key)) == 0)) { + m.attributes.insert(std::pair>(key, std::pair(val, sv))); + m.key_order.push_back(key); + } } - } - if (header.size() > 0 && key == header[0]) { - std::map>::iterator ii = mapping.find(sv.s); + if (header.size() > 0 && key == header[0]) { + std::map>::iterator ii = mapping.find(sv.s); + + if (ii != mapping.end()) { + std::vector fields = ii->second; + matched = true; + + for (size_t i = 1; i < fields.size(); i++) { + std::string joinkey = header[i]; + std::string joinval = fields[i]; + int attr_type = mvt_string; + + if (joinval.size() > 0) { + if (joinval[0] == '"') { + joinval = csv_dequote(joinval); + } else if (is_number(joinval)) { + attr_type = mvt_double; + } + } else if (pe) { + attr_type = mvt_null; + } - if (ii != mapping.end()) { - std::vector fields = ii->second; - matched = 1; + const char *sjoinkey = joinkey.c_str(); - for (size_t i = 1; i < fields.size(); i++) { - std::string joinkey = header[i]; - std::string joinval = fields[i]; - int attr_type = mvt_string; + if (include.count(joinkey) || (!exclude_all && exclude.count(joinkey) == 0 && exclude_attributes.count(joinkey) == 0 && attr_type != mvt_null)) { + mvt_value outval; + if (attr_type == mvt_string) { + outval.type = mvt_string; + outval.set_string_value(joinval); + } else { + outval.type = mvt_double; + outval.numeric_value.double_value = atof(joinval.c_str()); + } - if (joinval.size() > 0) { - if (joinval[0] == '"') { - joinval = csv_dequote(joinval); - } else if (is_number(joinval)) { - attr_type = mvt_double; - } - } else if (pe) { - attr_type = mvt_null; - } + auto fa = m.attributes.find(sjoinkey); + if (fa != m.attributes.end()) { + m.attributes.erase(fa); + } - const char *sjoinkey = joinkey.c_str(); + serial_val outsv; + outsv.type = outval.type; + outsv.s = joinval; - if (include.count(joinkey) || (!exclude_all && exclude.count(joinkey) == 0 && exclude_attributes.count(joinkey) == 0 && attr_type != mvt_null)) { - mvt_value outval; - if (attr_type == mvt_string) { - outval.type = mvt_string; - outval.set_string_value(joinval); - } else { - outval.type = mvt_double; - outval.numeric_value.double_value = atof(joinval.c_str()); - } + outval = stringified_to_mvt_value(outval.type, joinval.c_str(), tile_stringpool); - auto fa = attributes.find(sjoinkey); - if (fa != attributes.end()) { - attributes.erase(fa); + m.attributes.insert(std::pair>(joinkey, std::pair(outval, outsv))); + m.key_order.push_back(joinkey); } + } + } + } + } - serial_val outsv; - outsv.type = outval.type; - outsv.s = joinval; + if (matched) { + matches.push_back(m); + } + } - outval = stringified_to_mvt_value(outval.type, joinval.c_str(), tile_stringpool); + if (!matched && !ifmatched) { + // no matches, but they said to keep even unmatched tile features, + // so make one that is just the original feature - attributes.insert(std::pair>(joinkey, std::pair(outval, outsv))); - key_order.push_back(joinkey); - } + match m; + m.id = feat.id; + m.has_id = feat.has_id; + + if (!exclude_all_tile_attributes) { + for (size_t t = 0; t + 1 < feat.tags.size(); t += 2) { + const std::string &key = layer.keys[feat.tags[t]]; + mvt_value &val = layer.values[feat.tags[t + 1]]; + serial_val sv = mvt_value_to_serial_val(val); + + if (include.count(key) || (!exclude_all && exclude.count(key) == 0 && exclude_attributes.count(key) == 0)) { + m.attributes.insert(std::pair>(key, std::pair(val, sv))); + m.key_order.push_back(key); } } } + + matches.push_back(m); } - if (matched || !ifmatched) { + for (auto &m : matches) { if (tilestats == layermap.end()) { layermap.insert(std::pair(layer.name, layermap_entry(layermap.size()))); tilestats = layermap.find(layer.name); @@ -402,14 +462,18 @@ void append_tile(std::string message, int z, unsigned x, unsigned y, std::mapsecond.maxzoom = z; } + mvt_feature outfeature; + outfeature.id = m.id; + outfeature.has_id = m.has_id; + // To keep attributes in their original order instead of alphabetical - for (auto k : key_order) { - auto fa = attributes.find(k); + for (auto k : m.key_order) { + auto fa = m.attributes.find(k); - if (fa != attributes.end()) { + if (fa != m.attributes.end()) { outlayer.tag(outfeature, k, fa->second.first); add_to_tilestats(tilestats->second.tilestats, k, fa->second.second); - attributes.erase(fa); + m.attributes.erase(fa); } }