From 8d1a55e5e6dd9fe5465fee78c87d41bda4ce8446 Mon Sep 17 00:00:00 2001 From: Jochen Topf Date: Tue, 28 Jan 2025 17:59:11 +0100 Subject: [PATCH] Fix problem in slim mode with two-state processing of nodes In two-stage processing when getting nodes from the middle we are running the same SQL query as when getting the node locations. But that doesn't work. We need more information, the tags (and possibly the attributes). This fixes this by running a different prepared query in this case. Fixes #2298 --- src/middle-pgsql.cpp | 18 ++- tests/bdd/flex/node-add.feature | 164 +++++++++++++++++++++++++++ tests/data/test_output_flex_node.lua | 86 ++++++++++++++ 3 files changed, 264 insertions(+), 4 deletions(-) create mode 100644 tests/bdd/flex/node-add.feature create mode 100644 tests/data/test_output_flex_node.lua diff --git a/src/middle-pgsql.cpp b/src/middle-pgsql.cpp index 2e0eb995b..c0fd0c36f 100644 --- a/src/middle-pgsql.cpp +++ b/src/middle-pgsql.cpp @@ -522,7 +522,7 @@ std::size_t middle_query_pgsql_t::get_way_node_locations_flatnodes( osmium::Location middle_query_pgsql_t::get_node_location_db(osmid_t id) const { - auto const res = m_db_connection.exec_prepared("get_node", id); + auto const res = m_db_connection.exec_prepared("get_node_location", id); if (res.num_tuples() == 0) { return osmium::Location{}; } @@ -729,11 +729,14 @@ void build_node(osmid_t id, pg_result_t const &res, int res_num, int offset, { osmium::builder::NodeBuilder builder{*buffer}; builder.set_id(id); + builder.set_location(osmium::Location{ + (int)std::strtol(res.get_value(res_num, offset + 0), nullptr, 10), + (int)std::strtol(res.get_value(res_num, offset + 1), nullptr, 10)}); if (with_attributes) { - set_attributes_on_builder(&builder, res, res_num, offset); + set_attributes_on_builder(&builder, res, res_num, offset + 3); } - pgsql_parse_json_tags(res.get_value(res_num, offset + 1), buffer, &builder); + pgsql_parse_json_tags(res.get_value(res_num, offset + 2), buffer, &builder); } /** @@ -1261,7 +1264,7 @@ middle_pgsql_t::get_query_instance() m_store_options); if (m_store_options.nodes) { - mid->prepare("get_node", + mid->prepare("get_node_location", render_template( "SELECT id, lon, lat FROM {schema}\"{prefix}_nodes\"" " WHERE id = $1::int8")); @@ -1270,6 +1273,13 @@ middle_pgsql_t::get_query_instance() render_template( "SELECT id, lon, lat FROM {schema}\"{prefix}_nodes\"" " WHERE id = ANY($1::int8[])")); + + mid->prepare( + "get_node", + render_template("SELECT lon, lat, tags{attribute_columns_use}" + " FROM {schema}\"{prefix}_nodes\" o" + " {users_table_access}" + " WHERE o.id = $1::int8")); } mid->prepare("get_way", diff --git a/tests/bdd/flex/node-add.feature b/tests/bdd/flex/node-add.feature new file mode 100644 index 000000000..178ba81b1 --- /dev/null +++ b/tests/bdd/flex/node-add.feature @@ -0,0 +1,164 @@ +Feature: Adding nodes to a flex database + + Background: + Given the style file 'test_output_flex_node.lua' + + And the OSM data + """ + n11 v1 dV Tt1=yes x1 y1 + n12 v1 dV Tt2=yes x2 y2 + n13 v1 dV Ttboth=yes x3 y3 + n14 v1 dV Ttboth=yes x4 y4 + r30 v1 dV Tt=ag Mn11@,n12@mark,n13@,n14@mark + """ + When running osm2pgsql flex with parameters + | --slim | + + Then table osm2pgsql_test_t1 contains exactly + | node_id | + | 11 | + Then table osm2pgsql_test_t2 contains exactly + | node_id | + | 12 | + Then table osm2pgsql_test_tboth contains exactly + | node_id | + | 13 | + | 14 | + + + Scenario: node is not relevant + Given an empty grid + And the OSM data + """ + n10 v1 dV Tt=ag x0 y0 + r30 v2 dV Tt=ag Mn10@,n11@,n12@mark,n13@,n14@mark + """ + When running osm2pgsql flex with parameters + | --slim | -a | + + Then table osm2pgsql_test_t1 contains exactly + | node_id | + | 11 | + Then table osm2pgsql_test_t2 contains exactly + | node_id | + | 12 | + Then table osm2pgsql_test_tboth contains exactly + | node_id | + | 13 | + | 14 | + + + Scenario: add to t1 + Given an empty grid + And the OSM data + """ + n10 v1 dV Tt1=yes x0 y0 + r30 v2 dV Tt=ag Mn10@,n11@,n12@mark,n13@,n14@mark + """ + When running osm2pgsql flex with parameters + | --slim | -a | + + Then table osm2pgsql_test_t1 contains exactly + | node_id | + | 10 | + | 11 | + Then table osm2pgsql_test_t2 contains exactly + | node_id | + | 12 | + Then table osm2pgsql_test_tboth contains exactly + | node_id | + | 13 | + | 14 | + + + Scenario: add to t2 + Given an empty grid + And the OSM data + """ + n10 v1 dV Tt2=yes x0 y0 + r30 v2 dV Tt=ag Mn10@mark,n11@,n12@mark,n13@,n14@mark + """ + When running osm2pgsql flex with parameters + | --slim | -a | + + Then table osm2pgsql_test_t1 contains exactly + | node_id | + | 11 | + Then table osm2pgsql_test_t2 contains exactly + | node_id | rel_ids | + | 10 | {30} | + | 12 | {30} | + Then table osm2pgsql_test_tboth contains exactly + | node_id | + | 13 | + | 14 | + + + Scenario: add to t1 and t2 + Given an empty grid + And the OSM data + """ + n10 v1 dV Tt1=yes,t2=yes x0 y0 + r30 v2 dV Tt=ag Mn10@mark,n11@,n12@mark,n13@,n14@mark + """ + When running osm2pgsql flex with parameters + | --slim | -a | + + Then table osm2pgsql_test_t1 contains exactly + | node_id | + | 10 | + | 11 | + Then table osm2pgsql_test_t2 contains exactly + | node_id | rel_ids | + | 10 | {30} | + | 12 | {30} | + Then table osm2pgsql_test_tboth contains exactly + | node_id | + | 13 | + | 14 | + + + Scenario: add to tboth (only stage1) + Given an empty grid + And the OSM data + """ + n10 v1 dV Ttboth=yes x0 y0 + r30 v2 dV Tt=ag Mn10@,n11@,n12@mark,n13@,n14@mark + """ + When running osm2pgsql flex with parameters + | --slim | -a | + + Then table osm2pgsql_test_t1 contains exactly + | node_id | + | 11 | + Then table osm2pgsql_test_t2 contains exactly + | node_id | + | 12 | + Then table osm2pgsql_test_tboth contains exactly + | node_id | rel_ids | + | 10 | NULL | + | 13 | NULL | + | 14 | {30} | + + + Scenario: add to tboth (stage1 and stage2) + Given an empty grid + And the OSM data + """ + n10 v1 dV Ttboth=yes x0 y0 + r30 v2 dV Tt=ag Mn10@mark,n11@,n12@mark,n13@,n14@mark + """ + When running osm2pgsql flex with parameters + | --slim | -a | + + Then table osm2pgsql_test_t1 contains exactly + | node_id | + | 11 | + Then table osm2pgsql_test_t2 contains exactly + | node_id | + | 12 | + Then table osm2pgsql_test_tboth contains exactly + | node_id | rel_ids | + | 10 | {30} | + | 13 | NULL | + | 14 | {30} | diff --git a/tests/data/test_output_flex_node.lua b/tests/data/test_output_flex_node.lua new file mode 100644 index 000000000..bf5cc3a57 --- /dev/null +++ b/tests/data/test_output_flex_node.lua @@ -0,0 +1,86 @@ + +local tables = {} + +tables.t1 = osm2pgsql.define_node_table('osm2pgsql_test_t1', { + { column = 'tags', type = 'hstore' }, + { column = 'geom', type = 'point', not_null = true }, +}) + +tables.t2 = osm2pgsql.define_node_table('osm2pgsql_test_t2', { + { column = 'tags', type = 'hstore' }, + { column = 'rel_ids', type = 'text' }, + { column = 'geom', type = 'point', not_null = true }, +}) + +tables.tboth = osm2pgsql.define_node_table('osm2pgsql_test_tboth', { + { column = 'tags', type = 'hstore' }, + { column = 'rel_ids', type = 'text' }, + { column = 'geom', type = 'point', not_null = true }, +}) + +local n2r = {} + +local function get_ids(data) + if data then + local ids = {} + for rel_id, _ in pairs(data) do + ids[#ids + 1] = rel_id + end + table.sort(ids) + return '{' .. table.concat(ids, ',') .. '}' + end +end + +function osm2pgsql.process_node(object) + if object.tags.t1 then + tables.t1:insert{ + tags = object.tags, + geom = object:as_point() + } + end + + if osm2pgsql.stage == 2 and object.tags.t2 then + local ids = get_ids(n2r[object.id]) + if ids then + tables.t2:insert{ + rel_ids = ids, + geom = object:as_point() + } + end + end + + if object.tags.tboth then + local ids = get_ids(n2r[object.id]) + tables.tboth:insert{ + tags = object.tags, + rel_ids = ids, + geom = object:as_point() + } + end +end + +local function node_member_ids(relation) + local ids = {} + for _, member in ipairs(relation.members) do + if member.type == 'n' and member.role == 'mark' then + ids[#ids + 1] = member.ref + end + end + return ids +end + +function osm2pgsql.select_relation_members(relation) + return { nodes = node_member_ids(relation) } +end + +function osm2pgsql.process_relation(object) + for _, member in ipairs(object.members) do + if member.type == 'n' and member.role == 'mark' then + if not n2r[member.ref] then + n2r[member.ref] = {} + end + n2r[member.ref][object.id] = true + end + end +end +