diff --git a/.gitignore b/.gitignore index 5931277..e601402 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,8 @@ site data/* !data/Makefile *.whl +cache + +/*.json +/*.pkl +/*.svg diff --git a/data/Makefile b/data/Makefile index bab47c3..6ddd88c 100644 --- a/data/Makefile +++ b/data/Makefile @@ -14,7 +14,13 @@ clean: # https://github.com/cubao/fmm/wiki/Sample-data pull: network.json -network.json: | network.zip - unzip network.zip +network.json: network.zip + unzip $< network.zip: curl -LO https://github.com/cubao/fmm/files/11698908/network.zip + +pull: suzhoubeizhan.json +suzhoubeizhan.json: suzhoubeizhan.zip + unzip $< +suzhoubeizhan.zip: + curl -LO https://github.com/cubao/fmm/files/12568463/suzhoubeizhan.zip diff --git a/nano_fmm/converter.py b/nano_fmm/converter.py index e5ed2b7..a8214a6 100755 --- a/nano_fmm/converter.py +++ b/nano_fmm/converter.py @@ -54,4 +54,46 @@ def remap_network_with_string_id( os.makedirs(os.path.dirname(export), exist_ok=True) network.dump(export, indent=True) logger.info(f"wrote to {export}") - return network, indexer + return export + else: + return network, indexer + + +def remap_network_to_string_id( + network: Union[str, rapidjson], + *, + export: str = None, +): + if isinstance(network, str): + path = network + network = rapidjson() + network.load(path) + features = network["features"] + for i in range(len(features)): + f = features[i] + props = f["properties"] + if "id" not in props or "nexts" not in props and "prevs" not in props: + continue + props["id"] = str(props["id"]()) + props["nexts"] = [str(n) for n in props["nexts"]()] + props["prevs"] = [str(n) for n in props["prevs"]()] + if export: + export = os.path.abspath(export) + os.makedirs(os.path.dirname(export), exist_ok=True) + network.dump(export, indent=True) + logger.info(f"wrote to {export}") + return export + else: + return network + + +if __name__ == "__main__": + import fire + + fire.core.Display = lambda lines, out: print(*lines, file=out) + fire.Fire( + { + "remap_network_str2int": remap_network_with_string_id, + "remap_network_int2str": remap_network_to_string_id, + } + ) diff --git a/nano_fmm/osmnx_utils.py b/nano_fmm/osmnx_utils.py new file mode 100755 index 0000000..88474bc --- /dev/null +++ b/nano_fmm/osmnx_utils.py @@ -0,0 +1,127 @@ +import json +import os +from collections import defaultdict +from typing import List + +import numpy as np +import osmnx as ox + + +def topological_sort(nodes, nexts): + def toposort_util(node, visited, stack, scope): + visited.add(node) + for neighbor in nexts.get(node, []): + if neighbor not in scope: + continue + if neighbor not in visited: + toposort_util(neighbor, visited, stack, scope) + stack.insert(0, node) + + visited = set() + stack = [] + scope = set(nodes) + for node in nodes: + if node not in visited: + toposort_util(node, visited, stack, scope) + return tuple(stack) + + +def deduplicate_points(coords: np.ndarray) -> np.ndarray: + coords = np.asarray(coords) + deltas = np.sum(np.fabs(coords[:-1, :2] - coords[1:, :2]), axis=1) + if np.count_nonzero(deltas) == len(coords) - 1: + return coords + indices = np.r_[0, np.where(deltas != 0)[0] + 1] + return coords[indices] + + +def pull_map( + output: str, + *, + bbox: List[float] = None, + center_dist: List[float] = None, + network_type: str = "drive", +): + if bbox is not None: + west, south, east, north = bbox + G = ox.graph_from_bbox( + north, + south, + east, + west, + network_type=network_type, + simplify=False, + ) + elif center_dist is not None: + lon, lat = center_dist[:2] + dist = center_dist[2] if len(center_dist) > 2 else 500.0 + G = ox.graph_from_point( + (lat, lon), + dist=dist, + network_type=network_type, + simplify=False, + ) + else: + raise Exception( + "should specify --bbox=LEFT,BOTTOM,RIGHT,TOP or --center_dist=LON,LAT,DIST" + ) + # G = ox.graph_from_address("350 5th Ave, New York, New York", network_type="drive") + # G = ox.graph_from_place("Los Angeles, California", network_type="drive") + + nodes, edges = ox.graph_to_gdfs(G) + # nodes = ox.io._stringify_nonnumeric_cols(nodes) + # edges = ox.io._stringify_nonnumeric_cols(edges) + + # G = ox.project_graph(G) + # G = ox.consolidate_intersections(G, tolerance=10 / 1e5, rebuild_graph=True, dead_ends=True) + + edge2llas = {} + for k, edge in edges.iterrows(): + edge2llas[k[:2]] = np.array(edge["geometry"].coords) + ways = dict(zip(edge2llas.keys(), range(len(edge2llas)))) + heads, tails = defaultdict(set), defaultdict(set) + for s, e in edge2llas: + wid = ways[(s, e)] + heads[s].add(wid) + tails[e].add(wid) + features = [] + for (s, e), geom in edge2llas.items(): + f = { + "type": "Feature", + "geometry": { + "type": "LineString", + "coordinates": geom.tolist(), + }, + "properties": { + "type": "road", + "id": ways[(s, e)], + "nexts": sorted(heads[e]), + "prevs": sorted(tails[s]), + "nodes": [int(s), int(e)], + }, + } + features.append(f) + geojson = { + "type": "FeatureCollection", + "features": features, + } + + if not output: + return geojson + + output = os.path.abspath(output) + os.makedirs(os.path.dirname(output), exist_ok=True) + with open(output, "w") as f: + json.dump(geojson, f, indent=4) + return output + + +if __name__ == "__main__": + import fire + + fire.core.Display = lambda lines, out: print(*lines, file=out) + fire.Fire( + { + "pull_map": pull_map, + } + ) diff --git a/src/nano_fmm/network.cpp b/src/nano_fmm/network.cpp index 1edd422..b732d01 100644 --- a/src/nano_fmm/network.cpp +++ b/src/nano_fmm/network.cpp @@ -9,7 +9,8 @@ namespace nano_fmm { -bool Network::add_road(const Eigen::Ref &geom, int64_t road_id) +bool Network::add_road(const Eigen::Ref &geom, + int64_t road_id) { if (roads_.find(road_id) != roads_.end()) { spdlog::error("duplicate road, id={}, should remove_road first", @@ -226,34 +227,44 @@ void Network::reset() const { rtree_.reset(); } std::unique_ptr Network::load(const std::string &path) { - auto json = load_json(path); + RapidjsonValue json; + try { + json = load_json(path); + } catch (const std::exception &e) { + SPDLOG_ERROR("failed to load json from {}, error: {}", path, e.what()); + return {}; + } if (!json.IsObject()) { SPDLOG_ERROR("invalid network file: {}", path); return {}; } + const auto type = json.FindMember("type"); + if (type == json.MemberEnd() || !type->value.IsString()) { + SPDLOG_WARN("{} has no 'type', should be 'FeatureCollection' (geojson) " + "or 'RoadNetwork' (json)", + path); + return {}; + } bool is_wgs84 = true; auto itr = json.FindMember("is_wgs84"); if (itr != json.MemberEnd() && itr->value.IsBool()) { is_wgs84 = itr->value.GetBool(); } - auto type = json.FindMember("type"); - if (type != json.MemberEnd() && type->value.IsString() && - std::string(type->value.GetString(), type->value.GetStringLength()) == - "FeatureCollection") { - SPDLOG_CRITICAL("loading geojson {}", path); - auto ret = std::make_unique(is_wgs84); + auto ret = std::make_unique(is_wgs84); + const auto type_ = + std::string(type->value.GetString(), type->value.GetStringLength()); + if (type_ == "FeatureCollection") { + SPDLOG_INFO("loading geojson {}", path); ret->from_geojson(json); - return ret; - } - - if (!json.HasMember("roads") || !json.HasMember("nexts") || - !json.HasMember("prevs")) { - SPDLOG_ERROR("network file should at least have roads/nexts/prevs"); - return {}; + } else if (type_ == "RoadNetwork") { + SPDLOG_INFO("loading json {}", path); + ret->from_rapidjson(json); + } else { + SPDLOG_WARN("{} has invalid type:{}, should be 'FeatureCollection' " + "(geojson) or 'RoadNetwork' (json)", + path, type_); + ret.reset(); } - - auto ret = std::make_unique(is_wgs84); - ret->from_rapidjson(json); return ret; } diff --git a/src/nano_fmm/network.hpp b/src/nano_fmm/network.hpp index eea39b4..2a10ec4 100644 --- a/src/nano_fmm/network.hpp +++ b/src/nano_fmm/network.hpp @@ -24,7 +24,7 @@ struct Network Network(bool is_wgs84) : is_wgs84_(is_wgs84) {} // road network - bool add_road(const Eigen::Ref &geom, int64_t road_id); + bool add_road(const Eigen::Ref &geom, int64_t road_id); bool add_link(int64_t source_road, int64_t target_road, bool check_road = false); bool remove_road(int64_t road_id); diff --git a/src/nano_fmm/rapidjson.cpp b/src/nano_fmm/rapidjson.cpp index 2b53f16..f5ea7a0 100644 --- a/src/nano_fmm/rapidjson.cpp +++ b/src/nano_fmm/rapidjson.cpp @@ -197,11 +197,13 @@ RapidjsonValue UbodtRecord::to_rapidjson(RapidjsonAllocator &allocator) const Config &Config::from_rapidjson(const RapidjsonValue &json) { auto json_end = json.MemberEnd(); + FROM_RAPIDJSON((*this), json, json_end, ubodt_thresh) return *this; } RapidjsonValue Config::to_rapidjson(RapidjsonAllocator &allocator) const { RapidjsonValue json(rapidjson::kObjectType); + TO_RAPIDJSON((*this), json, allocator, ubodt_thresh) return json; } @@ -257,13 +259,13 @@ RapidjsonValue Network::to_geojson(RapidjsonAllocator &allocator) const RapidjsonValue features(rapidjson::kArrayType); features.Reserve(roads_.size(), allocator); - std::vector road_ids; + auto roads = std::map(); for (auto &pair : roads_) { - road_ids.push_back(pair.first); + roads.emplace(pair.first, &pair.second); } - std::sort(road_ids.begin(), road_ids.end()); - for (auto id : road_ids) { - auto &ruler = roads_.at(id); + for (auto &pair : roads) { + auto id = pair.first; + auto &ruler = *pair.second; RapidjsonValue geometry(rapidjson::kObjectType); geometry.AddMember("type", "LineString", allocator); geometry.AddMember("coordinates", @@ -307,24 +309,101 @@ RapidjsonValue Network::to_geojson(RapidjsonAllocator &allocator) const RapidjsonValue geojson(rapidjson::kObjectType); geojson.AddMember("type", "FeatureCollection", allocator); + if (!is_wgs84_) { + geojson.AddMember("is_wgs84", RapidjsonValue(is_wgs84_), allocator); + } geojson.AddMember("features", features, allocator); geojson.AddMember("config", config_.to_rapidjson(allocator), allocator); - geojson.AddMember("is_wgs84", RapidjsonValue(is_wgs84_), allocator); return geojson; } Network &Network::from_rapidjson(const RapidjsonValue &json) { - auto json_end = json.MemberEnd(); + for (auto &m : json["roads"].GetObject()) { + add_road(nano_fmm::from_rapidjson(m.value), + std::stoll(std::string(m.name.GetString(), + m.name.GetStringLength()))); + } + for (auto &m : json["nexts"].GetObject()) { + auto curr = std::stoll( + std::string(m.name.GetString(), m.name.GetStringLength())); + auto nexts = nano_fmm::from_rapidjson>(m.value); + for (auto next : nexts) { + add_link(curr, next); + } + } + for (auto &m : json["prevs"].GetObject()) { + auto curr = std::stoll( + std::string(m.name.GetString(), m.name.GetStringLength())); + auto prevs = nano_fmm::from_rapidjson>(m.value); + for (auto prev : prevs) { + add_link(prev, curr); + } + } auto config_itr = json.FindMember("config"); - if (config_itr == json_end) { + if (config_itr == json.MemberEnd()) { config_ = nano_fmm::from_rapidjson(config_itr->value); } return *this; } + RapidjsonValue Network::to_rapidjson(RapidjsonAllocator &allocator) const { RapidjsonValue json(rapidjson::kObjectType); + json.AddMember("type", "RoadNetwork", allocator); + json.AddMember("is_wgs84", RapidjsonValue(is_wgs84_), allocator); + // roads + { + auto roads = std::map(); + for (auto &pair : roads_) { + roads.emplace(pair.first, &pair.second); + } + RapidjsonValue _roads(rapidjson::kObjectType); + for (auto &pair : roads) { + auto rid = std::to_string(pair.first); + _roads.AddMember( + RapidjsonValue(rid.data(), rid.size(), allocator), + nano_fmm::to_rapidjson(pair.second->polyline(), allocator), + allocator); + } + json.AddMember("roads", _roads, allocator); + } + // nexts + { + auto nexts = std::map *>(); + for (auto &pair : nexts_) { + nexts.emplace(pair.first, &pair.second); + } + RapidjsonValue _nexts(rapidjson::kObjectType); + for (auto &pair : nexts) { + auto roads = + std::vector(pair.second->begin(), pair.second->end()); + std::sort(roads.begin(), roads.end()); + auto rid = std::to_string(pair.first); + _nexts.AddMember(RapidjsonValue(rid.data(), rid.size(), allocator), + nano_fmm::to_rapidjson(roads, allocator), + allocator); + } + json.AddMember("nexts", _nexts, allocator); + } + { + auto prevs = std::map *>(); + for (auto &pair : prevs_) { + prevs.emplace(pair.first, &pair.second); + } + RapidjsonValue _prevs(rapidjson::kObjectType); + for (auto &pair : prevs) { + auto roads = + std::vector(pair.second->begin(), pair.second->end()); + std::sort(roads.begin(), roads.end()); + auto rid = std::to_string(pair.first); + _prevs.AddMember(RapidjsonValue(rid.data(), rid.size(), allocator), + nano_fmm::to_rapidjson(roads, allocator), + allocator); + } + json.AddMember("prevs", _prevs, allocator); + } + json.AddMember("config", config_.to_rapidjson(allocator), allocator); return json; } diff --git a/tests/test_basic.py b/tests/test_basic.py index b1c29d6..268f6f2 100644 --- a/tests/test_basic.py +++ b/tests/test_basic.py @@ -570,7 +570,34 @@ def test_indexer(): assert indexer2.to_rapidjson() != indexer.to_rapidjson() +# fmm.utils.get_logging_level() # fmm.utils.set_logging_level(0) # trace # # fmm.utils.set_logging_level(6) # off -# network = Network.load("build/remapped.geojson") -# assert network.to_geojson() + + +def test_network(): + network = Network.load("README.md") + assert network is None + network = Network.load("missing_file") + assert network is None + + from nano_fmm.converter import remap_network_with_string_id + + geojson, _ = remap_network_with_string_id("data/suzhoubeizhan.json") + network = Network(is_wgs84=True) + network.from_geojson(geojson) + assert len(network.roads()) == 1016 + assert network.to_geojson().dump("build/network.geojson", indent=True) + assert network.to_rapidjson().dump("build/network.json", indent=True) + + network = Network.load("build/network.geojson") + network.to_geojson().dump("build/network2.geojson", indent=True) + network.to_rapidjson().dump("build/network2.json", indent=True) + + network = Network.load("build/network.json") + network.to_geojson().dump("build/network3.geojson", indent=True) + network.to_rapidjson().dump("build/network3.json", indent=True) + + rows = network.build_ubodt() + rows = sorted(rows) + print(rows[:5])