From 81ae5f14bf4bae4139ccec7487084cbc3256d838 Mon Sep 17 00:00:00 2001 From: program-- Date: Mon, 1 May 2023 09:05:33 -0700 Subject: [PATCH 01/62] initial geopackage support impl --- include/geopackage/GeoPackage.hpp | 113 ++++++++++ include/geopackage/Geometry.hpp | 31 +++ include/geopackage/SQLite.hpp | 360 ++++++++++++++++++++++++++++++ 3 files changed, 504 insertions(+) create mode 100644 include/geopackage/GeoPackage.hpp create mode 100644 include/geopackage/Geometry.hpp create mode 100644 include/geopackage/SQLite.hpp diff --git a/include/geopackage/GeoPackage.hpp b/include/geopackage/GeoPackage.hpp new file mode 100644 index 0000000000..b1b901244c --- /dev/null +++ b/include/geopackage/GeoPackage.hpp @@ -0,0 +1,113 @@ +#ifndef NGEN_GEOPACKAGE_H +#define NGEN_GEOPACKAGE_H + +#include +#include +#include +#include +#include +#include + +#include "SQLite.hpp" +#include "FeatureCollection.hpp" + +namespace geopackage { + +inline const geojson::FeatureType feature_type_map(const std::string& g) +{ + if (g == "POINT") return geojson::FeatureType::Point; + if (g == "LINESTRING") return geojson::FeatureType::LineString; + if (g == "POLYGON") return geojson::FeatureType::Polygon; + if (g == "MULTIPOINT") return geojson::FeatureType::MultiPoint; + if (g == "MULTILINESTRING") return geojson::FeatureType::MultiLineString; + if (g == "MULTIPOLYGON") return geojson::FeatureType::MultiPolygon; + return geojson::FeatureType::GeometryCollection; +} + +inline geojson::coordinate_t build_point(); +inline geojson::linestring_t build_linestring(); +inline geojson::polygon_t build_polygon(); +inline geojson::multipoint_t build_multipoint(); +inline geojson::multilinestring_t build_multilinestring(); +inline geojson::multipolygon_t build_multipolygon(); + +inline geojson::geometry build_geometry(); + +inline geojson::Feature build_feature( + const sqlite_iter& row, + const std::string& geom_type, + const std::string& geom_col +); + +inline std::shared_ptr read(const std::string& gpkg_path, const std::string& layer = "", const std::vector& ids = {}) +{ + sqlite db(gpkg_path); + // Check if layer exists + if (!db.has_table(layer)) { + throw std::runtime_error(""); + } + + // Layer exists, getting statement for it + std::string joined_ids = ""; + if (!ids.empty()) { + std::accumulate( + ids.begin(), + ids.end(), + joined_ids, + [](const std::string& origin, const std::string& append) { + return origin.empty() ? append : origin + "," + append; + } + ); + + joined_ids = " WHERE id (" + joined_ids + ")"; + } + + // Get layer bounding box + sqlite_iter query_get_layer_bbox = db.query("SELECT min_x, min_y, max_x, max_y FROM gpkg_contents WHERE table_name = ?", layer); + query_get_layer_bbox.next(); + const double min_x = query_get_layer_bbox.get(0); + const double min_y = query_get_layer_bbox.get(1); + const double max_x = query_get_layer_bbox.get(2); + const double max_y = query_get_layer_bbox.get(3); + + // Get number of features + sqlite_iter query_get_layer_count = db.query("SELECT COUNT(*) FROM " + layer); + query_get_layer_count.next(); + const int layer_feature_count = query_get_layer_count.get(0); + + // Get layer feature metadata (geometry column name + type) + sqlite_iter query_get_layer_geom_meta = db.query("SELECT column_name, geometry_type_name FROM gpkg_geometry_columns WHERE table_name = ?", layer); + const std::string layer_geometry_column = query_get_layer_geom_meta.get(0); + const std::string layer_geometry_type = query_get_layer_geom_meta.get(1); + + sqlite_iter query_get_layer_data_meta = db.query("SELECT column_name FROM gpkg_data_columns WHERE table_name = ?", layer); + std::vector layer_data_columns; + query_get_layer_data_meta.next(); + while (!query_get_layer_data_meta.done()) { + layer_data_columns.push_back(query_get_layer_data_meta.get(0)); + query_get_layer_data_meta.next(); + } + + // Get layer + sqlite_iter query_get_layer = db.query("SELECT * FROM " + layer + joined_ids + " ORDER BY id"); + std::vector features; + features.reserve(layer_feature_count); + query_get_layer.next(); + while(!query_get_layer.done()) { + features.push_back(build_feature( + query_get_layer, + layer_geometry_type, + layer_geometry_column + )); + } + + const auto fc = geojson::FeatureCollection( + std::move(features), + {min_x, min_y, max_x, max_y} + ); + + return std::make_shared(fc); +} + +} // namespace geopackage +#endif // NGEN_GEOPACKAGE_H \ No newline at end of file diff --git a/include/geopackage/Geometry.hpp b/include/geopackage/Geometry.hpp new file mode 100644 index 0000000000..33e0465275 --- /dev/null +++ b/include/geopackage/Geometry.hpp @@ -0,0 +1,31 @@ +#ifndef NGEN_GEOPACKAGE_GEOMETRY_H +#define NGEN_GEOPACKAGE_GEOMETRY_H + +#include +#include +#include + +namespace geopackage { + +struct GeoPackageGeometryHeader +{ + uint8_t version; + struct { + unsigned reserved : 2; + unsigned type : 1; + unsigned empty : 1; + unsigned indicator : 3; + unsigned order : 1; + } flags; + int32_t srs_id; + std::vector envelope; +}; + +struct GeoPackageGeometry +{ + GeoPackageGeometryHeader header; + std::vector geometry; +}; +} // namespace geopackage + +#endif // NGEN_GEOPACKAGE_GEOMETRY_H \ No newline at end of file diff --git a/include/geopackage/SQLite.hpp b/include/geopackage/SQLite.hpp new file mode 100644 index 0000000000..6322992069 --- /dev/null +++ b/include/geopackage/SQLite.hpp @@ -0,0 +1,360 @@ +#ifndef NGEN_GEOPACKAGE_SQLITE_H +#define NGEN_GEOPACKAGE_SQLITE_H + + +#include +#include +#include +#include +#include +#include + +namespace geopackage { + +/** + * @brief Deleter used to provide smart pointer support for sqlite3 structs. + */ +struct sqlite_deleter +{ + void operator()(sqlite3* db) { sqlite3_close_v2(db); } + void operator()(sqlite3_stmt* stmt) { sqlite3_finalize(stmt); } +}; + +/** + * @brief Smart pointer (unique) type for sqlite3 database + */ +using sqlite_t = std::unique_ptr; + +/** + * @brief Smart pointer (shared) type for sqlite3 prepared statements + */ +using stmt_t = std::shared_ptr; + +/** + * @brief Get a runtime error based on a function and code. + * + * @param f String denoting the function where the error originated + * @param code sqlite3 result code + * @return std::runtime_error + */ +static inline std::runtime_error sqlite_error(const std::string& f, int code) +{ + std::string errmsg = f + " returned code " + std::to_string(code); + return std::runtime_error(errmsg); +} + +/** + * @brief SQLite3 row iterator + * + * Provides a simple iterator-like implementation + * over rows of a SQLite3 query. + */ +class sqlite_iter +{ + private: + stmt_t stmt; + int iteration_step = -1; + bool iteration_finished = false; + + // column metadata + int column_count; + std::vector column_names; + std::vector column_types; + + // returns the raw pointer to the sqlite statement + sqlite3_stmt* ptr() const noexcept; + + public: + sqlite_iter(stmt_t stmt); + ~sqlite_iter(); + bool done() const noexcept; + sqlite_iter& next(); + sqlite_iter& restart(); + void close(); + int current_row() const noexcept; + int num_columns() const noexcept; + int column_index(const std::string& name) const noexcept; + const std::vector& columns() const noexcept { return this->column_names; } + const std::vector& types() const noexcept { return this->column_types; } + + template + T get(int col) const; + + template + T get(const std::string& name) const; +}; + +inline sqlite_iter::sqlite_iter(stmt_t stmt) + : stmt(stmt) +{ + this->column_count = sqlite3_column_count(this->ptr()); + this->column_names = std::vector(); + this->column_names.reserve(this->column_count); + this->column_types = std::vector(); + this->column_types.reserve(this->column_count); + + for (int i = 0; i < this->column_count; i++) { + this->column_names.push_back(sqlite3_column_name(this->ptr(), i)); + this->column_types.push_back(sqlite3_column_type(this->ptr(), i)); + } +}; + +inline sqlite_iter::~sqlite_iter() +{ + this->stmt.reset(); +} + +inline sqlite3_stmt* sqlite_iter::ptr() const noexcept +{ + return this->stmt.get(); +} + +inline bool sqlite_iter::done() const noexcept +{ + return this->iteration_finished; +} + +inline sqlite_iter& sqlite_iter::next() +{ + if (!this->done()) { + const int returncode = sqlite3_step(this->ptr()); + if (returncode == SQLITE_DONE) { + this->iteration_finished = true; + } + this->iteration_step++; + } + + return *this; +} + +inline sqlite_iter& sqlite_iter::restart() +{ + sqlite3_reset(this->ptr()); + this->iteration_step = -1; + return *this; +} + +inline void sqlite_iter::close() +{ + this->~sqlite_iter(); +} + +inline int sqlite_iter::current_row() const noexcept +{ + return this->iteration_step; +} + +inline int sqlite_iter::num_columns() const noexcept +{ + return this->column_count; +} + +inline int sqlite_iter::column_index(const std::string& name) const noexcept +{ + const ptrdiff_t pos = + std::distance(this->column_names.begin(), std::find(this->column_names.begin(), this->column_names.end(), name)); + + return pos >= this->column_names.size() ? -1 : pos; +} + +template<> +inline std::vector sqlite_iter::get>(int col) const +{ + return *reinterpret_cast*>(sqlite3_column_blob(this->ptr(), col)); +} + +template<> +inline double sqlite_iter::get(int col) const +{ + return sqlite3_column_double(this->ptr(), col); +} + +template<> +inline int sqlite_iter::get(int col) const +{ + return sqlite3_column_int(this->ptr(), col); +} + +template<> +inline std::string sqlite_iter::get(int col) const +{ + // TODO: this won't work with non-ASCII text + return std::string(reinterpret_cast(sqlite3_column_text(this->ptr(), col))); +} + +template +inline T sqlite_iter::get(const std::string& name) const +{ + const int index = this->column_index(name); + return this->get(index); +} + +/** + * @brief Wrapper around SQLite3 Databases + */ +class sqlite +{ + private: + sqlite_t conn = nullptr; + stmt_t stmt = nullptr; + + public: + sqlite() = default; + + /** + * @brief Construct a new sqlite object from a path to database + * + * @param path File path to sqlite3 database + */ + sqlite(const std::string& path); + + sqlite(sqlite& db); + sqlite& operator=(sqlite& db); + + /** + * @brief Take ownership of a sqlite3 database + * + * @param db sqlite3 database object + */ + sqlite(sqlite&& db); + + /** + * @brief Move assignment operator + * + * @param db sqlite3 database object + * @return sqlite& reference to sqlite3 database + */ + sqlite& operator=(sqlite&& db); + + /** + * @brief Destroy the sqlite object + */ + ~sqlite(); + + /** + * @brief Return the originating sqlite3 database pointer + * + * @return sqlite3* + */ + sqlite3* connection() const noexcept; + + /** + * @brief Check if SQLite database contains a given table + * + * @param table name of table + * @return true if table does exist + * @return false if table does not exist + */ + bool has_table(const std::string& table) noexcept; + + /** + * Query the SQLite Database and get the result + * @param statement String query + * @return read-only SQLite row iterator (see: [sqlite_iter]) + */ + sqlite_iter query(const std::string& statement); + + /** + * @brief TODO!!! + * + * @param statement + * @param params + * @return sqlite_iter* + */ + template + sqlite_iter query(const std::string& statement, T const&... params); +}; + +inline sqlite::sqlite(const std::string& path) +{ + sqlite3* conn; + int code = sqlite3_open_v2(path.c_str(), &conn, SQLITE_OPEN_READONLY, NULL); + if (code != SQLITE_OK) { + throw sqlite_error("sqlite3_open_v2", code); + } + this->conn = sqlite_t(conn); +} + +inline sqlite::sqlite(sqlite& db) +{ + this->conn = std::move(db.conn); + this->stmt = db.stmt; +} + +inline sqlite& sqlite::operator=(sqlite& db) +{ + this->conn = std::move(db.conn); + this->stmt = db.stmt; + return *this; +} + +inline sqlite::sqlite(sqlite&& db) +{ + this->conn = std::move(db.conn); + this->stmt = db.stmt; +} + +inline sqlite& sqlite::operator=(sqlite&& db) +{ + this->conn = std::move(db.conn); + this->stmt = db.stmt; + return *this; +} + +inline sqlite::~sqlite() +{ + this->stmt.reset(); + this->conn.reset(); +} + +inline sqlite3* sqlite::connection() const noexcept +{ + return this->conn.get(); +} + +inline bool sqlite::has_table(const std::string& table) noexcept +{ + auto q = this->query("SELECT 1 from sqlite_master WHERE type='table' AND name = ?", table); + q.next(); + return static_cast(q.get(0)); +}; + +inline sqlite_iter sqlite::query(const std::string& statement) +{ + sqlite3_stmt* stmt; + const auto cstmt = statement.c_str(); + const int code = sqlite3_prepare_v2(this->connection(), cstmt, statement.length() + 1, &stmt, NULL); + + if (code != SQLITE_OK) { + // something happened, can probably switch on result codes + // https://www.sqlite.org/rescode.html + throw sqlite_error("sqlite3_prepare_v2", code); + } + + this->stmt = stmt_t(stmt, sqlite_deleter{}); + return sqlite_iter(this->stmt); +} + +template +inline sqlite_iter sqlite::query(const std::string& statement, T const&... params) +{ + sqlite3_stmt* stmt; + const auto cstmt = statement.c_str(); + const int code = sqlite3_prepare_v2(this->connection(), cstmt, statement.length() + 1, &stmt, NULL); + + if (code != SQLITE_OK) { + throw sqlite_error("sqlite3_prepare_v2", code); + } + + std::vector binds{ { params... } }; + for (size_t i = 0; i < binds.size(); i++) { + sqlite3_bind_text(stmt, i + 1, binds[i].c_str(), -1, SQLITE_STATIC); + } + + this->stmt = stmt_t(stmt, sqlite_deleter{}); + return sqlite_iter(this->stmt); +} + +} // namespace geopackage + +#endif // NGEN_GEOPACKAGE_SQLITE_H \ No newline at end of file From da0ff438832ff913336592c06e7e1d00b984fc0e Mon Sep 17 00:00:00 2001 From: program-- Date: Mon, 1 May 2023 09:05:33 -0700 Subject: [PATCH 02/62] more work toward gpkg support; started wkb parser --- include/geopackage/GeoPackage.hpp | 79 ++++++++++++++++++++++++++++++- include/geopackage/Geometry.hpp | 31 ------------ include/geopackage/WKB.hpp | 62 ++++++++++++++++++++++++ 3 files changed, 139 insertions(+), 33 deletions(-) delete mode 100644 include/geopackage/Geometry.hpp create mode 100644 include/geopackage/WKB.hpp diff --git a/include/geopackage/GeoPackage.hpp b/include/geopackage/GeoPackage.hpp index b1b901244c..fb3a8f0186 100644 --- a/include/geopackage/GeoPackage.hpp +++ b/include/geopackage/GeoPackage.hpp @@ -10,6 +10,7 @@ #include "SQLite.hpp" #include "FeatureCollection.hpp" +#include "WKB.hpp" namespace geopackage { @@ -31,13 +32,87 @@ inline geojson::multipoint_t build_multipoint(); inline geojson::multilinestring_t build_multilinestring(); inline geojson::multipolygon_t build_multipolygon(); -inline geojson::geometry build_geometry(); +inline geojson::geometry build_geometry(const sqlite_iter& row, const geojson::FeatureType geom_type, const std::string& geom_col) +{ + const std::vector geometry_blob = row.get>(geom_col); + + geojson::geometry geometry; + + switch(geom_type) { + case geojson::FeatureType::Point: + break; + case geojson::FeatureType::LineString: + break; + case geojson::FeatureType::Polygon: + break; + case geojson::FeatureType::MultiPoint: + break; + case geojson::FeatureType::MultiLineString: + break; + case geojson::FeatureType::MultiPolygon: + break; + case geojson::FeatureType::GeometryCollection: + break; + default: + break; + } +} + +inline geojson::PropertyMap build_properties(const sqlite_iter& row, const std::string& geom_col) +{ + geojson::PropertyMap properties; + + std::map property_types; + const auto data_cols = row.columns(); + const auto data_types = row.types(); + std::transform( + data_cols.begin(), + data_cols.end(), + data_types.begin(), + std::inserter(property_types, property_types.end()), [](const std::string& name, int type) { + return std::make_pair(name, type); + } + ); + + for (auto& col : property_types) { + const auto name = col.first; + const auto type = col.second; + if (name == geom_col) { + continue; + } + + geojson::JSONProperty* property = nullptr; + switch(type) { + case SQLITE_INTEGER: + *property = geojson::JSONProperty(name, row.get(name)); + break; + case SQLITE_FLOAT: + *property = geojson::JSONProperty(name, row.get(name)); + break; + case SQLITE_TEXT: + *property = geojson::JSONProperty(name, row.get(name)); + break; + default: + *property = geojson::JSONProperty(name, "null"); + break; + } + + properties.emplace(col, std::move(*property)); + } + + return properties; +} inline geojson::Feature build_feature( const sqlite_iter& row, const std::string& geom_type, const std::string& geom_col -); +) +{ + const auto type = feature_type_map(geom_type); + const geojson::PropertyMap properties = build_properties(row, geom_col); + const geojson::geometry geometry = build_geometry(row, type, geom_col); +}; inline std::shared_ptr read(const std::string& gpkg_path, const std::string& layer = "", const std::vector& ids = {}) { diff --git a/include/geopackage/Geometry.hpp b/include/geopackage/Geometry.hpp deleted file mode 100644 index 33e0465275..0000000000 --- a/include/geopackage/Geometry.hpp +++ /dev/null @@ -1,31 +0,0 @@ -#ifndef NGEN_GEOPACKAGE_GEOMETRY_H -#define NGEN_GEOPACKAGE_GEOMETRY_H - -#include -#include -#include - -namespace geopackage { - -struct GeoPackageGeometryHeader -{ - uint8_t version; - struct { - unsigned reserved : 2; - unsigned type : 1; - unsigned empty : 1; - unsigned indicator : 3; - unsigned order : 1; - } flags; - int32_t srs_id; - std::vector envelope; -}; - -struct GeoPackageGeometry -{ - GeoPackageGeometryHeader header; - std::vector geometry; -}; -} // namespace geopackage - -#endif // NGEN_GEOPACKAGE_GEOMETRY_H \ No newline at end of file diff --git a/include/geopackage/WKB.hpp b/include/geopackage/WKB.hpp new file mode 100644 index 0000000000..86e8c99560 --- /dev/null +++ b/include/geopackage/WKB.hpp @@ -0,0 +1,62 @@ +#ifndef NGEN_GEOPACKAGE_GEOMETRY_H +#define NGEN_GEOPACKAGE_GEOMETRY_H + +#include +#include +#include "JSONGeometry.hpp" + +namespace geopackage { + +// struct geometry_blob +// { +// struct { +// uint8_t version; +// struct { +// unsigned reserved : 2; +// unsigned type : 1; +// unsigned empty : 1; +// unsigned indicator : 3; +// unsigned order : 1; +// } flags; +// int32_t srs_id; +// std::vector envelope; +// } header; +// wkb geometry; +// }; + +namespace wkb { + +static inline uint32_t bswap_32(uint32_t x) +{ + return (((x & 0xFF) << 24) | + ((x & 0xFF00) << 8) | + ((x & 0xFF0000) >> 8) | + ((x & 0xFF000000) >> 24)); +} + +static inline uint64_t bswap_64(uint64_t x) +{ + return (((x & 0xFFULL) << 56) | + ((x & 0xFF00ULL) << 40) | + ((x & 0xFF0000ULL) << 24) | + ((x & 0xFF000000ULL) << 8) | + ((x & 0xFF00000000ULL) >> 8) | + ((x & 0xFF0000000000ULL) >> 24) | + ((x & 0xFF000000000000ULL) >> 40) | + ((x & 0xFF00000000000000ULL) >> 56)); +} + +enum wkbGeometryType { + wkbPoint = 1, + wkbLineString = 2, + wkbPolygon = 3, + wkbMultiPoint = 4, + wkbMultiLineString = 5, + wkbMultiPolygon = 6, + wkbGeometryCollection = 7 +}; + +} // namespace wkb +} // namespace geopackage + +#endif // NGEN_GEOPACKAGE_GEOMETRY_H \ No newline at end of file From d5cc164bf2f1dabe0734b67ff1043e59ee0c588b Mon Sep 17 00:00:00 2001 From: program-- Date: Mon, 1 May 2023 09:05:33 -0700 Subject: [PATCH 03/62] scaffold CMake for gpkg --- CMakeLists.txt | 2 ++ src/geopackage/CMakeLists.txt | 7 +++++++ test/CMakeLists.txt | 6 ++++++ 3 files changed, 15 insertions(+) create mode 100644 src/geopackage/CMakeLists.txt diff --git a/CMakeLists.txt b/CMakeLists.txt index 377607f16b..bd22129e71 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -259,6 +259,7 @@ endif() add_subdirectory("src/core") add_dependencies(core libudunits2) add_subdirectory("src/geojson") +add_subdirectory("src/geopackage") add_subdirectory("src/realizations/catchment") add_subdirectory("src/models/tshirt") add_subdirectory("src/models/kernels/reservoir") @@ -271,6 +272,7 @@ target_link_libraries(ngen PUBLIC #NGen::core_catchment_giuh NGen::core_nexus NGen::geojson + NGen::geopackage NGen::models_tshirt NGen::realizations_catchment NGen::kernels_reservoir diff --git a/src/geopackage/CMakeLists.txt b/src/geopackage/CMakeLists.txt new file mode 100644 index 0000000000..006e4d9597 --- /dev/null +++ b/src/geopackage/CMakeLists.txt @@ -0,0 +1,7 @@ +cmake_minimum_required(VERSION 3.10) +add_library(geopackage INTERFACE) +add_library(NGen::geopackage ALIAS geopackage) +target_include_directories(geopackage INTERFACE ${PROJECT_SOURCE_DIR}/include/geopackage) + +find_package(Boost) +target_link_libraries(geopackage INTERFACE NGen::geojson Boost::boost) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 722da840d2..6ca788d148 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -84,6 +84,12 @@ add_test(test_geojson NGen::geojson ) +########################## GeoPackage Unit Tests +add_test(test_geopackage + 1 + geopackage/WKB_Test.cpp + NGen::geopackage) + ########################## Realization Config Unit Tests add_test(test_realization_config 1 From 484ee5a8309214a461ebf805979e47529c260849 Mon Sep 17 00:00:00 2001 From: program-- Date: Mon, 1 May 2023 09:05:33 -0700 Subject: [PATCH 04/62] implement (standard) wkb parser --- include/geopackage/WKB.hpp | 62 ----------- include/geopackage/wkb/reader.hpp | 176 ++++++++++++++++++++++++++++++ test/geopackage/WKB_Test.cpp | 79 ++++++++++++++ 3 files changed, 255 insertions(+), 62 deletions(-) delete mode 100644 include/geopackage/WKB.hpp create mode 100644 include/geopackage/wkb/reader.hpp create mode 100644 test/geopackage/WKB_Test.cpp diff --git a/include/geopackage/WKB.hpp b/include/geopackage/WKB.hpp deleted file mode 100644 index 86e8c99560..0000000000 --- a/include/geopackage/WKB.hpp +++ /dev/null @@ -1,62 +0,0 @@ -#ifndef NGEN_GEOPACKAGE_GEOMETRY_H -#define NGEN_GEOPACKAGE_GEOMETRY_H - -#include -#include -#include "JSONGeometry.hpp" - -namespace geopackage { - -// struct geometry_blob -// { -// struct { -// uint8_t version; -// struct { -// unsigned reserved : 2; -// unsigned type : 1; -// unsigned empty : 1; -// unsigned indicator : 3; -// unsigned order : 1; -// } flags; -// int32_t srs_id; -// std::vector envelope; -// } header; -// wkb geometry; -// }; - -namespace wkb { - -static inline uint32_t bswap_32(uint32_t x) -{ - return (((x & 0xFF) << 24) | - ((x & 0xFF00) << 8) | - ((x & 0xFF0000) >> 8) | - ((x & 0xFF000000) >> 24)); -} - -static inline uint64_t bswap_64(uint64_t x) -{ - return (((x & 0xFFULL) << 56) | - ((x & 0xFF00ULL) << 40) | - ((x & 0xFF0000ULL) << 24) | - ((x & 0xFF000000ULL) << 8) | - ((x & 0xFF00000000ULL) >> 8) | - ((x & 0xFF0000000000ULL) >> 24) | - ((x & 0xFF000000000000ULL) >> 40) | - ((x & 0xFF00000000000000ULL) >> 56)); -} - -enum wkbGeometryType { - wkbPoint = 1, - wkbLineString = 2, - wkbPolygon = 3, - wkbMultiPoint = 4, - wkbMultiLineString = 5, - wkbMultiPolygon = 6, - wkbGeometryCollection = 7 -}; - -} // namespace wkb -} // namespace geopackage - -#endif // NGEN_GEOPACKAGE_GEOMETRY_H \ No newline at end of file diff --git a/include/geopackage/wkb/reader.hpp b/include/geopackage/wkb/reader.hpp new file mode 100644 index 0000000000..c1d153f197 --- /dev/null +++ b/include/geopackage/wkb/reader.hpp @@ -0,0 +1,176 @@ +#ifndef NGEN_GEOPACKAGE_WKB_POD_H +#define NGEN_GEOPACKAGE_WKB_POD_H + +#include +#include +#include +#include +#include + +#include + +namespace geopackage { +namespace wkb { + +using byte_t = uint8_t; +using byte_vector = std::vector; + +struct point { double x, y; }; +struct linestring { std::vector points; }; +struct polygon { std::vector rings; }; +struct multipoint { std::vector points; }; +struct multilinestring { std::vector lines; }; +struct multipolygon { std::vector polygons; }; + +namespace { + +inline std::string wkt_type(const point&) { return "POINT"; } +inline std::string wkt_type(const linestring&) { return "LINESTRING"; } +inline std::string wkt_type(const polygon&) { return "POLYGON"; } +inline std::string wkt_type(const multipoint&) { return "MULTIPOINT"; } +inline std::string wkt_type(const multilinestring&) { return "MULTILINESTRING"; } +inline std::string wkt_type(const multipolygon&) { return "MULTIPOLYGON"; } + +inline std::string wkt_coords(const point& g) +{ + std::ostringstream out; + out.precision(3); + out << std::fixed << g.x << " " << g.y; + return std::move(out).str(); +} + +inline std::string wkt_coords(const linestring& g) +{ + return "(" + std::accumulate( + std::next(g.points.begin()), + g.points.end(), + wkt_coords(g.points[0]), + [](const std::string& a, const point b) { return a + "," + wkt_coords(b); } + ) + ")"; +} + +inline std::string wkt_coords(const multipoint& g) +{ + return "(" + std::accumulate( + std::next(g.points.begin()), + g.points.end(), + wkt_coords(g.points[0]), + [](const std::string& a, const point b) { return a + "," + wkt_coords(b); } + ) + ")"; +} + +inline std::string wkt_coords(const polygon& g) +{ + std::string output; + for (const auto& gg : g.rings) { + output += "(" + wkt_coords(gg) + ")"; + } + return output; +} + +inline std::string wkt_coords(const multilinestring& g) +{ + std::string output; + for (const auto& gg : g.lines) { + output += "(" + wkt_coords(gg) + ")"; + } + return output; +} + +inline std::string wkt_coords(const multipolygon& g) +{ + std::string output; + for (const auto& gg : g.polygons) { + output += "(" + wkt_coords(gg) + ")"; + } + return output; +} + +} + +template +inline std::string as_wkt(const T& g) +{ + return wkb::wkt_type(g) + " " + wkb::wkt_coords(g); +} + +template +inline void copy_from(std::vector src, T& index, S& dst, uint8_t order) +{ + std::memcpy(&dst, &src[index], sizeof(S)); + + if (order == 0x01) { + boost::endian::little_to_native_inplace(dst); + } else { + boost::endian::big_to_native_inplace(dst); + } + + index += sizeof(S); +} + +template +static inline T read_wkb_internal(const byte_vector& buffer, int& index, uint8_t order); + +template<> +inline point read_wkb_internal(const byte_vector& buffer, int& index, uint8_t order) +{ + double x, y; + copy_from(buffer, index, x, order); + copy_from(buffer, index, y, order); + return point{x, y}; +}; + +#define INTERNAL_WKB_DEF(output_t, child_t) \ + template<> \ + inline output_t read_wkb_internal( \ + const byte_vector& buffer, \ + int& index, \ + uint8_t order \ + ) \ + { \ + uint32_t count; \ + copy_from(buffer, index, count, order); \ + std::vector children(count); \ + for (auto& child : children) { \ + child = read_wkb_internal(buffer, index, order); \ + } \ + return output_t{children}; \ + } + +INTERNAL_WKB_DEF(linestring, point) + +INTERNAL_WKB_DEF(polygon, linestring) + +INTERNAL_WKB_DEF(multipoint, point) + +INTERNAL_WKB_DEF(multilinestring, linestring) + +INTERNAL_WKB_DEF(multipolygon, polygon) + +#undef INTERNAL_WKB_DEF + +template +static inline T read_wkb(const byte_vector& buffer) +{ + if (buffer.size() < 5) { + + throw std::runtime_error( + "buffer reached end before encountering WKB\n\tdebug: [" + + std::string(buffer.begin(), buffer.end()) + "]" + ); + } + + int index = 0; + const byte_t order = buffer[index]; + index++; + + uint32_t type; + copy_from(buffer, index, type, 0x00); + + return read_wkb_internal(buffer, index, order); +}; + +} // namespace wkb +} // namespace geopackage + +#endif // NGEN_GEOPACKAGE_WKB_POD_H \ No newline at end of file diff --git a/test/geopackage/WKB_Test.cpp b/test/geopackage/WKB_Test.cpp new file mode 100644 index 0000000000..a2de052963 --- /dev/null +++ b/test/geopackage/WKB_Test.cpp @@ -0,0 +1,79 @@ +#include +#include + +using namespace geopackage; + +class WKB_Test : public ::testing::Test +{ + protected: + WKB_Test() {} + + ~WKB_Test() override {} + + void SetUp() override { + this->wkb = { + 0x01, 0x03, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, + 0x00, 0x05, 0x00, 0x00, 0x00, 0x54, 0xE3, 0xA5, + 0x9B, 0xC4, 0x60, 0x25, 0x40, 0x64, 0x3B, 0xDF, + 0x4F, 0x8D, 0x17, 0x39, 0xC0, 0x5C, 0x8F, 0xC2, + 0xF5, 0x28, 0x4C, 0x41, 0x40, 0xEC, 0x51, 0xB8, + 0x1E, 0x85, 0x2B, 0x34, 0xC0, 0xD5, 0x78, 0xE9, + 0x26, 0x31, 0x68, 0x43, 0x40, 0x6F, 0x12, 0x83, + 0xC0, 0xCA, 0xD1, 0x41, 0xC0, 0x1B, 0x2F, 0xDD, + 0x24, 0x06, 0x01, 0x2B, 0x40, 0xA4, 0x70, 0x3D, + 0x0A, 0xD7, 0x93, 0x43, 0xC0, 0x54, 0xE3, 0xA5, + 0x9B, 0xC4, 0x60, 0x25, 0x40, 0x64, 0x3B, 0xDF, + 0x4F, 0x8D, 0x17, 0x39, 0xC0, + }; + + this->little_endian = { + 0x01, + 0x00, 0x00, 0x00, 0x01, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x40 + }; + + this->big_endian = { + 0x00, + 0x00, 0x00, 0x00, 0x01, + 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x40, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 + }; + } + + void TearDown() override {} + + std::vector wkb; + std::vector little_endian; + std::vector big_endian; +}; + +TEST_F(WKB_Test, wkb_read_test) +{ + wkb::polygon geom = wkb::read_wkb(this->wkb); + std::vector> expected_coordinates = { + {10.689, -25.092}, + {34.595, -20.170}, + {38.814, -35.639}, + {13.502, -39.155}, + {10.689, -25.092} + }; + + for (int i = 0; i < expected_coordinates.size(); i++) { + EXPECT_NEAR(geom.rings[0].points[i].x, expected_coordinates[i].first, 0.0001); + EXPECT_NEAR(geom.rings[0].points[i].y, expected_coordinates[i].second, 0.0001); + } + + EXPECT_EQ( + wkb::as_wkt(geom), + "POLYGON ((10.689 -25.092,34.595 -20.170,38.814 -35.639,13.502 -39.155,10.689 -25.092))" + ); +} + +TEST_F(WKB_Test, wkb_endianness_test) +{ + wkb::point geom_big = wkb::read_wkb(this->big_endian); + wkb::point geom_little = wkb::read_wkb(this->little_endian); + EXPECT_NEAR(geom_big.x, geom_little.x, 0.000001); + EXPECT_NEAR(geom_big.y, geom_little.y, 0.000001); +} \ No newline at end of file From af827439d9a57d8ec33947d0a3a7bead3c907258 Mon Sep 17 00:00:00 2001 From: program-- Date: Mon, 1 May 2023 09:05:33 -0700 Subject: [PATCH 05/62] cleanup function declarations --- include/geopackage/GeoPackage.hpp | 8 -------- 1 file changed, 8 deletions(-) diff --git a/include/geopackage/GeoPackage.hpp b/include/geopackage/GeoPackage.hpp index fb3a8f0186..ece58c5c23 100644 --- a/include/geopackage/GeoPackage.hpp +++ b/include/geopackage/GeoPackage.hpp @@ -10,7 +10,6 @@ #include "SQLite.hpp" #include "FeatureCollection.hpp" -#include "WKB.hpp" namespace geopackage { @@ -25,13 +24,6 @@ inline const geojson::FeatureType feature_type_map(const std::string& g) return geojson::FeatureType::GeometryCollection; } -inline geojson::coordinate_t build_point(); -inline geojson::linestring_t build_linestring(); -inline geojson::polygon_t build_polygon(); -inline geojson::multipoint_t build_multipoint(); -inline geojson::multilinestring_t build_multilinestring(); -inline geojson::multipolygon_t build_multipolygon(); - inline geojson::geometry build_geometry(const sqlite_iter& row, const geojson::FeatureType geom_type, const std::string& geom_col) { const std::vector geometry_blob = row.get>(geom_col); From f251db2a4255c0f9240beb69e2415a436005de27 Mon Sep 17 00:00:00 2001 From: program-- Date: Mon, 1 May 2023 09:05:34 -0700 Subject: [PATCH 06/62] add docs to wkb reader funcs --- include/geopackage/wkb/reader.hpp | 43 ++++++++++++++++++++++++++++--- 1 file changed, 40 insertions(+), 3 deletions(-) diff --git a/include/geopackage/wkb/reader.hpp b/include/geopackage/wkb/reader.hpp index c1d153f197..9243aa0bfa 100644 --- a/include/geopackage/wkb/reader.hpp +++ b/include/geopackage/wkb/reader.hpp @@ -86,16 +86,37 @@ inline std::string wkt_coords(const multipolygon& g) return output; } -} - +} // anonymous namespace + +/** + * @brief Get the WKT form from WKB structs + * + * @tparam T WKB geometry struct type + * @param g geometry object + * @return std::string @param{g} in WKT form + */ template inline std::string as_wkt(const T& g) { return wkb::wkt_type(g) + " " + wkb::wkt_coords(g); } +/** + * @brief + * Copies bytes from @param{src} to @param{dst}, + * converts the endianness to native, + * and increments @param{index} by the number of bytes + * used to store @tparam{S} + * + * @tparam T an integral type + * @tparam S a primitive type + * @param src a vector of bytes + * @param index an integral type tracking the starting position of @param{dst}'s memory + * @param dst output primitive + * @param order endianness value (0x01 == Little; 0x00 == Big) + */ template -inline void copy_from(std::vector src, T& index, S& dst, uint8_t order) +inline void copy_from(byte_vector src, T& index, S& dst, uint8_t order) { std::memcpy(&dst, &src[index], sizeof(S)); @@ -108,6 +129,15 @@ inline void copy_from(std::vector src, T& index, S& dst, uint8_t order) index += sizeof(S); } +/** + * @brief Recursively read WKB data into known structs + * + * @tparam T geometry struct type + * @param buffer vector of bytes + * @param index tracked buffer index + * @param order endianness + * @return T parsed WKB in geometry struct + */ template static inline T read_wkb_internal(const byte_vector& buffer, int& index, uint8_t order); @@ -149,6 +179,13 @@ INTERNAL_WKB_DEF(multipolygon, polygon) #undef INTERNAL_WKB_DEF +/** + * @brief Read WKB into a naive geometry struct + * + * @tparam T geometry struct type (i.e. @code{wkb::point}, @code{wkb::polygon}, etc.) + * @param buffer vector of bytes + * @return T geometry struct containing the parsed WKB values. + */ template static inline T read_wkb(const byte_vector& buffer) { From 3945ae6660a5d455040e0b149e9c07eff1962301 Mon Sep 17 00:00:00 2001 From: program-- Date: Mon, 1 May 2023 09:05:34 -0700 Subject: [PATCH 07/62] add generic geometry struct; refactor wkt funcs as visitor --- include/geopackage/wkb/reader.hpp | 277 ++++++++++++++++++++---------- test/geopackage/WKB_Test.cpp | 23 +-- 2 files changed, 197 insertions(+), 103 deletions(-) diff --git a/include/geopackage/wkb/reader.hpp b/include/geopackage/wkb/reader.hpp index 9243aa0bfa..269d197cad 100644 --- a/include/geopackage/wkb/reader.hpp +++ b/include/geopackage/wkb/reader.hpp @@ -1,13 +1,14 @@ #ifndef NGEN_GEOPACKAGE_WKB_POD_H #define NGEN_GEOPACKAGE_WKB_POD_H -#include #include #include #include #include +#include #include +#include namespace geopackage { namespace wkb { @@ -22,84 +23,96 @@ struct multipoint { std::vector points; }; struct multilinestring { std::vector lines; }; struct multipolygon { std::vector polygons; }; -namespace { +struct geometry { + using gtype = boost::variant< + point, linestring, polygon, + multipoint, multilinestring, multipolygon + >; -inline std::string wkt_type(const point&) { return "POINT"; } -inline std::string wkt_type(const linestring&) { return "LINESTRING"; } -inline std::string wkt_type(const polygon&) { return "POLYGON"; } -inline std::string wkt_type(const multipoint&) { return "MULTIPOINT"; } -inline std::string wkt_type(const multilinestring&) { return "MULTILINESTRING"; } -inline std::string wkt_type(const multipolygon&) { return "MULTIPOLYGON"; } - -inline std::string wkt_coords(const point& g) -{ - std::ostringstream out; - out.precision(3); - out << std::fixed << g.x << " " << g.y; - return std::move(out).str(); -} - -inline std::string wkt_coords(const linestring& g) -{ - return "(" + std::accumulate( - std::next(g.points.begin()), - g.points.end(), - wkt_coords(g.points[0]), - [](const std::string& a, const point b) { return a + "," + wkt_coords(b); } - ) + ")"; -} + uint32_t type; + gtype data; +}; -inline std::string wkt_coords(const multipoint& g) +class wkt : public boost::static_visitor { - return "(" + std::accumulate( - std::next(g.points.begin()), - g.points.end(), - wkt_coords(g.points[0]), - [](const std::string& a, const point b) { return a + "," + wkt_coords(b); } - ) + ")"; -} + public: + + /** + * @brief Get the WKT form from WKB structs + * + * @tparam T WKB geometry struct type + * @param g geometry object + * @return std::string @param{g} in WKT form + */ + template + std::string operator()(T& g) const + { + return this->wkt_type(g) + " " + this->wkt_coords(g); + } -inline std::string wkt_coords(const polygon& g) -{ - std::string output; - for (const auto& gg : g.rings) { - output += "(" + wkt_coords(gg) + ")"; + private: + std::string wkt_type(const point&) const { return "POINT"; } + std::string wkt_type(const linestring&) const { return "LINESTRING"; } + std::string wkt_type(const polygon&) const { return "POLYGON"; } + std::string wkt_type(const multipoint&) const { return "MULTIPOINT"; } + std::string wkt_type(const multilinestring&) const { return "MULTILINESTRING"; } + std::string wkt_type(const multipolygon&) const { return "MULTIPOLYGON"; } + + std::string wkt_coords(const point& g) const + { + std::ostringstream out; + out.precision(3); + out << std::fixed << g.x << " " << g.y; + return std::move(out).str(); + } + + std::string wkt_coords(const linestring& g) const + { + return "(" + std::accumulate( + std::next(g.points.begin()), + g.points.end(), + wkt_coords(g.points[0]), + [this](const std::string& a, const point b) { return a + "," + this->wkt_coords(b); } + ) + ")"; } - return output; -} -inline std::string wkt_coords(const multilinestring& g) -{ - std::string output; - for (const auto& gg : g.lines) { - output += "(" + wkt_coords(gg) + ")"; + std::string wkt_coords(const multipoint& g) const + { + return "(" + std::accumulate( + std::next(g.points.begin()), + g.points.end(), + wkt_coords(g.points[0]), + [this](const std::string& a, const point b) { return a + "," + this->wkt_coords(b); } + ) + ")"; } - return output; -} -inline std::string wkt_coords(const multipolygon& g) -{ - std::string output; - for (const auto& gg : g.polygons) { - output += "(" + wkt_coords(gg) + ")"; + std::string wkt_coords(const polygon& g) const + { + std::string output; + for (const auto& gg : g.rings) { + output += "(" + wkt_coords(gg) + ")"; + } + return output; } - return output; -} -} // anonymous namespace + std::string wkt_coords(const multilinestring& g) const + { + std::string output; + for (const auto& gg : g.lines) { + output += "(" + wkt_coords(gg) + ")"; + } + return output; + } -/** - * @brief Get the WKT form from WKB structs - * - * @tparam T WKB geometry struct type - * @param g geometry object - * @return std::string @param{g} in WKT form - */ -template -inline std::string as_wkt(const T& g) -{ - return wkb::wkt_type(g) + " " + wkb::wkt_coords(g); -} + std::string wkt_coords(const multipolygon& g) const + { + std::string output; + for (const auto& gg : g.polygons) { + output += "(" + wkt_coords(gg) + ")"; + } + return output; + } +}; /** * @brief @@ -150,34 +163,66 @@ inline point read_wkb_internal(const byte_vector& buffer, int& index, uin return point{x, y}; }; -#define INTERNAL_WKB_DEF(output_t, child_t) \ - template<> \ - inline output_t read_wkb_internal( \ - const byte_vector& buffer, \ - int& index, \ - uint8_t order \ - ) \ - { \ - uint32_t count; \ - copy_from(buffer, index, count, order); \ - std::vector children(count); \ - for (auto& child : children) { \ - child = read_wkb_internal(buffer, index, order); \ - } \ - return output_t{children}; \ +template<> +inline linestring read_wkb_internal(const byte_vector& buffer, int& index, uint8_t order) +{ + uint32_t count; + copy_from(buffer, index, count, order); + std::vector children(count); + for (auto& child : children) { + child = read_wkb_internal(buffer, index, order); } + return linestring{children}; +} -INTERNAL_WKB_DEF(linestring, point) - -INTERNAL_WKB_DEF(polygon, linestring) - -INTERNAL_WKB_DEF(multipoint, point) +template<> +inline polygon read_wkb_internal(const byte_vector& buffer, int& index, uint8_t order) +{ + uint32_t count; + copy_from(buffer, index, count, order); + std::vector children(count); + for (auto& child : children) { + child = read_wkb_internal(buffer, index, order); + } + + return polygon{children}; +} -INTERNAL_WKB_DEF(multilinestring, linestring) +template<> +inline multipoint read_wkb_internal(const byte_vector& buffer, int& index, uint8_t order) +{ + uint32_t count; + copy_from(buffer, index, count, order); + std::vector children(count); + for (auto& child : children) { + child = read_wkb_internal(buffer, index, order); + } + return multipoint{children}; +} -INTERNAL_WKB_DEF(multipolygon, polygon) +template<> +inline multilinestring read_wkb_internal(const byte_vector& buffer, int& index, uint8_t order) +{ + uint32_t count; + copy_from(buffer, index, count, order); + std::vector children(count); + for (auto& child : children) { + child = read_wkb_internal(buffer, index, order); + } + return multilinestring{children}; +} -#undef INTERNAL_WKB_DEF +template<> +inline multipolygon read_wkb_internal(const byte_vector& buffer, int& index, uint8_t order) +{ + uint32_t count; + copy_from(buffer, index, count, order); + std::vector children(count); + for (auto& child : children) { + child = read_wkb_internal(buffer, index, order); + } + return multipolygon{children}; +} /** * @brief Read WKB into a naive geometry struct @@ -187,7 +232,7 @@ INTERNAL_WKB_DEF(multipolygon, polygon) * @return T geometry struct containing the parsed WKB values. */ template -static inline T read_wkb(const byte_vector& buffer) +static inline T read_known_wkb(const byte_vector& buffer) { if (buffer.size() < 5) { @@ -202,11 +247,57 @@ static inline T read_wkb(const byte_vector& buffer) index++; uint32_t type; - copy_from(buffer, index, type, 0x00); + copy_from(buffer, index, type, order); return read_wkb_internal(buffer, index, order); }; +static inline wkb::geometry read_wkb(const byte_vector&buffer) +{ + if (buffer.size() < 5) { + + throw std::runtime_error( + "buffer reached end before encountering WKB\n\tdebug: [" + + std::string(buffer.begin(), buffer.end()) + "]" + ); + } + + int index = 0; + const byte_t order = buffer[index]; + index++; + + uint32_t type; + copy_from(buffer, index, type, order); + + wkb::geometry g; + g.type = type; + switch(type) { + case 1: + g.data = read_wkb_internal(buffer, index, order); + break; + case 2: + g.data = read_wkb_internal(buffer, index, order); + break; + case 3: + g.data = read_wkb_internal(buffer, index, order); + break; + case 4: + g.data = read_wkb_internal(buffer, index, order); + break; + case 5: + g.data = read_wkb_internal(buffer, index, order); + break; + case 6: + g.data = read_wkb_internal(buffer, index, order); + break; + default: + g.data = point{std::nan("0"), std::nan("0")}; + break; + } + + return std::move(g); +} + } // namespace wkb } // namespace geopackage diff --git a/test/geopackage/WKB_Test.cpp b/test/geopackage/WKB_Test.cpp index a2de052963..867fa34b47 100644 --- a/test/geopackage/WKB_Test.cpp +++ b/test/geopackage/WKB_Test.cpp @@ -50,8 +50,11 @@ class WKB_Test : public ::testing::Test TEST_F(WKB_Test, wkb_read_test) { - wkb::polygon geom = wkb::read_wkb(this->wkb); - std::vector> expected_coordinates = { + const wkb::geometry geom = wkb::read_wkb(this->wkb); + EXPECT_EQ(geom.type, 3); + + const wkb::polygon& poly = boost::get(geom.data); + const std::vector> expected_coordinates = { {10.689, -25.092}, {34.595, -20.170}, {38.814, -35.639}, @@ -60,20 +63,20 @@ TEST_F(WKB_Test, wkb_read_test) }; for (int i = 0; i < expected_coordinates.size(); i++) { - EXPECT_NEAR(geom.rings[0].points[i].x, expected_coordinates[i].first, 0.0001); - EXPECT_NEAR(geom.rings[0].points[i].y, expected_coordinates[i].second, 0.0001); + EXPECT_NEAR(poly.rings[0].points[i].x, expected_coordinates[i].first, 0.0001); + EXPECT_NEAR(poly.rings[0].points[i].y, expected_coordinates[i].second, 0.0001); } - EXPECT_EQ( - wkb::as_wkt(geom), - "POLYGON ((10.689 -25.092,34.595 -20.170,38.814 -35.639,13.502 -39.155,10.689 -25.092))" - ); + const wkb::wkt wkt_visitor; + const auto wkt = "POLYGON ((10.689 -25.092,34.595 -20.170,38.814 -35.639,13.502 -39.155,10.689 -25.092))"; + EXPECT_EQ(boost::apply_visitor(wkt_visitor, geom.data), wkt); + EXPECT_EQ(wkt_visitor(poly), wkt); } TEST_F(WKB_Test, wkb_endianness_test) { - wkb::point geom_big = wkb::read_wkb(this->big_endian); - wkb::point geom_little = wkb::read_wkb(this->little_endian); + wkb::point geom_big = wkb::read_known_wkb(this->big_endian); + wkb::point geom_little = wkb::read_known_wkb(this->little_endian); EXPECT_NEAR(geom_big.x, geom_little.x, 0.000001); EXPECT_NEAR(geom_big.y, geom_little.y, 0.000001); } \ No newline at end of file From dbb1a470e4799ab4e79af42cdfd588ade7f2b3cd Mon Sep 17 00:00:00 2001 From: program-- Date: Mon, 1 May 2023 09:05:34 -0700 Subject: [PATCH 08/62] clean up some WKB code --- include/geopackage/wkb/reader.hpp | 252 +++++++++--------------------- test/geopackage/WKB_Test.cpp | 17 +- 2 files changed, 80 insertions(+), 189 deletions(-) diff --git a/include/geopackage/wkb/reader.hpp b/include/geopackage/wkb/reader.hpp index 269d197cad..7dc51d13c7 100644 --- a/include/geopackage/wkb/reader.hpp +++ b/include/geopackage/wkb/reader.hpp @@ -1,5 +1,5 @@ -#ifndef NGEN_GEOPACKAGE_WKB_POD_H -#define NGEN_GEOPACKAGE_WKB_POD_H +#ifndef NGEN_GEOPACKAGE_WKB_READER_H +#define NGEN_GEOPACKAGE_WKB_READER_H #include #include @@ -15,104 +15,20 @@ namespace wkb { using byte_t = uint8_t; using byte_vector = std::vector; - -struct point { double x, y; }; -struct linestring { std::vector points; }; -struct polygon { std::vector rings; }; -struct multipoint { std::vector points; }; -struct multilinestring { std::vector lines; }; -struct multipolygon { std::vector polygons; }; - -struct geometry { - using gtype = boost::variant< - point, linestring, polygon, - multipoint, multilinestring, multipolygon - >; - - uint32_t type; - gtype data; -}; - -class wkt : public boost::static_visitor -{ - public: - - /** - * @brief Get the WKT form from WKB structs - * - * @tparam T WKB geometry struct type - * @param g geometry object - * @return std::string @param{g} in WKT form - */ - template - std::string operator()(T& g) const - { - return this->wkt_type(g) + " " + this->wkt_coords(g); - } - - private: - std::string wkt_type(const point&) const { return "POINT"; } - std::string wkt_type(const linestring&) const { return "LINESTRING"; } - std::string wkt_type(const polygon&) const { return "POLYGON"; } - std::string wkt_type(const multipoint&) const { return "MULTIPOINT"; } - std::string wkt_type(const multilinestring&) const { return "MULTILINESTRING"; } - std::string wkt_type(const multipolygon&) const { return "MULTIPOLYGON"; } - - std::string wkt_coords(const point& g) const - { - std::ostringstream out; - out.precision(3); - out << std::fixed << g.x << " " << g.y; - return std::move(out).str(); - } - - std::string wkt_coords(const linestring& g) const - { - return "(" + std::accumulate( - std::next(g.points.begin()), - g.points.end(), - wkt_coords(g.points[0]), - [this](const std::string& a, const point b) { return a + "," + this->wkt_coords(b); } - ) + ")"; - } - - std::string wkt_coords(const multipoint& g) const - { - return "(" + std::accumulate( - std::next(g.points.begin()), - g.points.end(), - wkt_coords(g.points[0]), - [this](const std::string& a, const point b) { return a + "," + this->wkt_coords(b); } - ) + ")"; - } - - std::string wkt_coords(const polygon& g) const - { - std::string output; - for (const auto& gg : g.rings) { - output += "(" + wkt_coords(gg) + ")"; - } - return output; - } - - std::string wkt_coords(const multilinestring& g) const - { - std::string output; - for (const auto& gg : g.lines) { - output += "(" + wkt_coords(gg) + ")"; - } - return output; - } - - std::string wkt_coords(const multipolygon& g) const - { - std::string output; - for (const auto& gg : g.polygons) { - output += "(" + wkt_coords(gg) + ")"; - } - return output; - } -}; +struct wkb_point { double x, y; }; +struct wkb_linestring { std::vector points; }; +struct wkb_polygon { std::vector rings; }; +struct wkb_multipoint { std::vector points; }; +struct wkb_multilinestring { std::vector lines; }; +struct wkb_multipolygon { std::vector polygons; }; +using wkb_geometry = boost::variant< + wkb_point, + wkb_linestring, + wkb_polygon, + wkb_multipoint, + wkb_multilinestring, + wkb_multipolygon +>; /** * @brief @@ -152,80 +68,67 @@ inline void copy_from(byte_vector src, T& index, S& dst, uint8_t order) * @return T parsed WKB in geometry struct */ template -static inline T read_wkb_internal(const byte_vector& buffer, int& index, uint8_t order); +inline T read_wkb_internal(const byte_vector& buffer, int& index, uint8_t order); + +namespace { +template +inline output_t read_wkb_compound_internal(const byte_vector& buffer, int& index, uint8_t order) +{ + uint32_t count; + copy_from(buffer, index, count, order); + std::vector children(count); + for (auto& child : children) { + child = read_wkb_internal(buffer, index, order); + } + return output_t{children}; +} +} // anonymous namespace + +#define READ_WKB_INTERNAL_SIG(output_t) output_t read_wkb_internal(const byte_vector& buffer, int& index, uint8_t order) template<> -inline point read_wkb_internal(const byte_vector& buffer, int& index, uint8_t order) +inline READ_WKB_INTERNAL_SIG(wkb_point) { double x, y; copy_from(buffer, index, x, order); copy_from(buffer, index, y, order); - return point{x, y}; + return wkb_point{x, y}; }; -template<> -inline linestring read_wkb_internal(const byte_vector& buffer, int& index, uint8_t order) +template<> +inline READ_WKB_INTERNAL_SIG(wkb_linestring) { - uint32_t count; - copy_from(buffer, index, count, order); - std::vector children(count); - for (auto& child : children) { - child = read_wkb_internal(buffer, index, order); - } - return linestring{children}; + return read_wkb_compound_internal(buffer, index, order); } -template<> -inline polygon read_wkb_internal(const byte_vector& buffer, int& index, uint8_t order) +template<> +inline READ_WKB_INTERNAL_SIG(wkb_polygon) { - uint32_t count; - copy_from(buffer, index, count, order); - std::vector children(count); - for (auto& child : children) { - child = read_wkb_internal(buffer, index, order); - } - - return polygon{children}; + return read_wkb_compound_internal(buffer, index, order); } -template<> -inline multipoint read_wkb_internal(const byte_vector& buffer, int& index, uint8_t order) +template<> +inline READ_WKB_INTERNAL_SIG(wkb_multipoint) { - uint32_t count; - copy_from(buffer, index, count, order); - std::vector children(count); - for (auto& child : children) { - child = read_wkb_internal(buffer, index, order); - } - return multipoint{children}; + return read_wkb_compound_internal(buffer, index, order); } -template<> -inline multilinestring read_wkb_internal(const byte_vector& buffer, int& index, uint8_t order) +template<> +inline READ_WKB_INTERNAL_SIG(wkb_multilinestring) { - uint32_t count; - copy_from(buffer, index, count, order); - std::vector children(count); - for (auto& child : children) { - child = read_wkb_internal(buffer, index, order); - } - return multilinestring{children}; + return read_wkb_compound_internal(buffer, index, order); } -template<> -inline multipolygon read_wkb_internal(const byte_vector& buffer, int& index, uint8_t order) +template<> +inline READ_WKB_INTERNAL_SIG(wkb_multipolygon) { - uint32_t count; - copy_from(buffer, index, count, order); - std::vector children(count); - for (auto& child : children) { - child = read_wkb_internal(buffer, index, order); - } - return multipolygon{children}; + return read_wkb_compound_internal(buffer, index, order); } +#undef READ_WKB_INTERNAL_SIG + /** - * @brief Read WKB into a naive geometry struct + * @brief Read (known) WKB into a specific geometry struct * * @tparam T geometry struct type (i.e. @code{wkb::point}, @code{wkb::polygon}, etc.) * @param buffer vector of bytes @@ -252,14 +155,16 @@ static inline T read_known_wkb(const byte_vector& buffer) return read_wkb_internal(buffer, index, order); }; -static inline wkb::geometry read_wkb(const byte_vector&buffer) +/** + * @brief Read WKB into a variant geometry struct + * + * @param buffer vector of bytes + * @return wkb::geometry geometry struct containing the parsed WKB values. + */ +static inline wkb::wkb_geometry read_wkb(const byte_vector&buffer) { if (buffer.size() < 5) { - - throw std::runtime_error( - "buffer reached end before encountering WKB\n\tdebug: [" + - std::string(buffer.begin(), buffer.end()) + "]" - ); + throw std::runtime_error("buffer reached end before encountering WKB"); } int index = 0; @@ -269,36 +174,21 @@ static inline wkb::geometry read_wkb(const byte_vector&buffer) uint32_t type; copy_from(buffer, index, type, order); - wkb::geometry g; - g.type = type; + wkb_geometry g; switch(type) { - case 1: - g.data = read_wkb_internal(buffer, index, order); - break; - case 2: - g.data = read_wkb_internal(buffer, index, order); - break; - case 3: - g.data = read_wkb_internal(buffer, index, order); - break; - case 4: - g.data = read_wkb_internal(buffer, index, order); - break; - case 5: - g.data = read_wkb_internal(buffer, index, order); - break; - case 6: - g.data = read_wkb_internal(buffer, index, order); - break; - default: - g.data = point{std::nan("0"), std::nan("0")}; - break; + case 1: g = read_wkb_internal(buffer, index, order); break; + case 2: g = read_wkb_internal(buffer, index, order); break; + case 3: g = read_wkb_internal(buffer, index, order); break; + case 4: g = read_wkb_internal(buffer, index, order); break; + case 5: g = read_wkb_internal(buffer, index, order); break; + case 6: g = read_wkb_internal(buffer, index, order); break; + default: g = wkb_point{std::nan("0"), std::nan("0")}; break; } - return std::move(g); + return g; } } // namespace wkb } // namespace geopackage -#endif // NGEN_GEOPACKAGE_WKB_POD_H \ No newline at end of file +#endif // NGEN_GEOPACKAGE_WKB_READER_H \ No newline at end of file diff --git a/test/geopackage/WKB_Test.cpp b/test/geopackage/WKB_Test.cpp index 867fa34b47..9d3c3d8131 100644 --- a/test/geopackage/WKB_Test.cpp +++ b/test/geopackage/WKB_Test.cpp @@ -1,5 +1,6 @@ #include #include +#include using namespace geopackage; @@ -50,10 +51,10 @@ class WKB_Test : public ::testing::Test TEST_F(WKB_Test, wkb_read_test) { - const wkb::geometry geom = wkb::read_wkb(this->wkb); - EXPECT_EQ(geom.type, 3); + const wkb::wkb_geometry geom = wkb::read_wkb(this->wkb); + EXPECT_EQ(geom.which() + 1, 3); // +1 since variant.which() is 0-based - const wkb::polygon& poly = boost::get(geom.data); + const wkb::wkb_polygon& poly = boost::get(geom); const std::vector> expected_coordinates = { {10.689, -25.092}, {34.595, -20.170}, @@ -67,16 +68,16 @@ TEST_F(WKB_Test, wkb_read_test) EXPECT_NEAR(poly.rings[0].points[i].y, expected_coordinates[i].second, 0.0001); } - const wkb::wkt wkt_visitor; + const wkb::wkt_visitor wkt_v; const auto wkt = "POLYGON ((10.689 -25.092,34.595 -20.170,38.814 -35.639,13.502 -39.155,10.689 -25.092))"; - EXPECT_EQ(boost::apply_visitor(wkt_visitor, geom.data), wkt); - EXPECT_EQ(wkt_visitor(poly), wkt); + EXPECT_EQ(boost::apply_visitor(wkt_v, geom), wkt); + EXPECT_EQ(wkt_v(poly), wkt); } TEST_F(WKB_Test, wkb_endianness_test) { - wkb::point geom_big = wkb::read_known_wkb(this->big_endian); - wkb::point geom_little = wkb::read_known_wkb(this->little_endian); + const auto geom_big = wkb::read_known_wkb(this->big_endian); + const auto geom_little = wkb::read_known_wkb(this->little_endian); EXPECT_NEAR(geom_big.x, geom_little.x, 0.000001); EXPECT_NEAR(geom_big.y, geom_little.y, 0.000001); } \ No newline at end of file From 2278e0b7d27610615f15e650a468012e222ffa87 Mon Sep 17 00:00:00 2001 From: program-- Date: Mon, 1 May 2023 09:05:34 -0700 Subject: [PATCH 09/62] separate wkt visitor from wkb directly --- include/geopackage/wkb/wkt.hpp | 93 ++++++++++++++++++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100644 include/geopackage/wkb/wkt.hpp diff --git a/include/geopackage/wkb/wkt.hpp b/include/geopackage/wkb/wkt.hpp new file mode 100644 index 0000000000..2f96da7bfc --- /dev/null +++ b/include/geopackage/wkb/wkt.hpp @@ -0,0 +1,93 @@ +#ifndef NGEN_GEOPACKAGE_WKB_VISITOR_WKT_H +#define NGEN_GEOPACKAGE_WKB_VISITOR_WKT_H + +#include "reader.hpp" + +namespace geopackage { +namespace wkb { + +class wkt_visitor : public boost::static_visitor +{ + public: + + /** + * @brief Get the WKT form from WKB structs + * + * @tparam T WKB geometry struct type + * @param g geometry object + * @return std::string @param{g} in WKT form + */ + template + std::string operator()(T& g) const + { + return std::string(this->wkt_type(g)) + " " + this->wkt_coords(g); + } + + private: + constexpr const char* wkt_type(const wkb_point&) const { return "POINT"; } + constexpr const char* wkt_type(const wkb_linestring&) const { return "LINESTRING"; } + constexpr const char* wkt_type(const wkb_polygon&) const { return "POLYGON"; } + constexpr const char* wkt_type(const wkb_multipoint&) const { return "MULTIPOINT"; } + constexpr const char* wkt_type(const wkb_multilinestring&) const { return "MULTILINESTRING"; } + constexpr const char* wkt_type(const wkb_multipolygon&) const { return "MULTIPOLYGON"; } + + std::string wkt_coords(const wkb_point& g) const + { + std::ostringstream out; + out.precision(3); + out << std::fixed << g.x << " " << g.y; + return std::move(out).str(); + } + + std::string wkt_coords(const wkb_linestring& g) const + { + return "(" + std::accumulate( + std::next(g.points.begin()), + g.points.end(), + wkt_coords(g.points[0]), + [this](const std::string& a, const wkb_point b) { return a + "," + this->wkt_coords(b); } + ) + ")"; + } + + std::string wkt_coords(const wkb_multipoint& g) const + { + return "(" + std::accumulate( + std::next(g.points.begin()), + g.points.end(), + wkt_coords(g.points[0]), + [this](const std::string& a, const wkb_point b) { return a + "," + this->wkt_coords(b); } + ) + ")"; + } + + std::string wkt_coords(const wkb_polygon& g) const + { + std::string output; + for (const auto& gg : g.rings) { + output += "(" + wkt_coords(gg) + ")"; + } + return output; + } + + std::string wkt_coords(const wkb_multilinestring& g) const + { + std::string output; + for (const auto& gg : g.lines) { + output += "(" + wkt_coords(gg) + ")"; + } + return output; + } + + std::string wkt_coords(const wkb_multipolygon& g) const + { + std::string output; + for (const auto& gg : g.polygons) { + output += "(" + wkt_coords(gg) + ")"; + } + return output; + } +}; + +} // namespace wkb +} // namespace geopackage + +#endif // NGEN_GEOPACKAGE_WKB_VISITOR_WKT_H \ No newline at end of file From 2463e406d811ac4acdda4d6e908b4d34e81f0239 Mon Sep 17 00:00:00 2001 From: program-- Date: Mon, 1 May 2023 09:05:34 -0700 Subject: [PATCH 10/62] lots of refactoring -- streamlined wkb -> geojson types [no ci] --- include/geopackage/GeoPackage.hpp | 167 ++++++++++++++++++++++++++---- include/geopackage/wkb/reader.hpp | 135 +++++++++++++++--------- include/geopackage/wkb/wkt.hpp | 93 ----------------- test/geopackage/WKB_Test.cpp | 33 +++--- 4 files changed, 245 insertions(+), 183 deletions(-) delete mode 100644 include/geopackage/wkb/wkt.hpp diff --git a/include/geopackage/GeoPackage.hpp b/include/geopackage/GeoPackage.hpp index ece58c5c23..f226957d72 100644 --- a/include/geopackage/GeoPackage.hpp +++ b/include/geopackage/GeoPackage.hpp @@ -1,18 +1,28 @@ #ifndef NGEN_GEOPACKAGE_H #define NGEN_GEOPACKAGE_H +#include #include #include #include #include #include -#include -#include "SQLite.hpp" +#include +#include + #include "FeatureCollection.hpp" +#include "JSONGeometry.hpp" +#include "SQLite.hpp" +#include "features/CollectionFeature.hpp" +#include "wkb/reader.hpp" + +namespace bsrs = boost::geometry::srs; + namespace geopackage { +namespace { inline const geojson::FeatureType feature_type_map(const std::string& g) { if (g == "POINT") return geojson::FeatureType::Point; @@ -23,30 +33,66 @@ inline const geojson::FeatureType feature_type_map(const std::string& g) if (g == "MULTIPOLYGON") return geojson::FeatureType::MultiPolygon; return geojson::FeatureType::GeometryCollection; } +} // anonymous namespace inline geojson::geometry build_geometry(const sqlite_iter& row, const geojson::FeatureType geom_type, const std::string& geom_col) { const std::vector geometry_blob = row.get>(geom_col); + int index = 0; + if (geometry_blob[0] != 'G' && geometry_blob[1] != 'P') { + throw std::runtime_error("expected geopackage WKB, but found invalid format instead"); + } + index += 2; + + // skip version + index++; - geojson::geometry geometry; + // flags + const bool is_extended = geometry_blob[index] & 0x00100000; + const bool is_empty = geometry_blob[index] & 0x00010000; + const uint8_t indicator = (geometry_blob[index] & 0x00001110) >> 1; + const uint8_t endian = geometry_blob[index] & 0x00000001; + index++; - switch(geom_type) { - case geojson::FeatureType::Point: - break; - case geojson::FeatureType::LineString: - break; - case geojson::FeatureType::Polygon: - break; - case geojson::FeatureType::MultiPoint: - break; - case geojson::FeatureType::MultiLineString: - break; - case geojson::FeatureType::MultiPolygon: - break; - case geojson::FeatureType::GeometryCollection: - break; - default: - break; + // Read srs_id + uint32_t srs_id; + wkb::copy_from(geometry_blob, index, srs_id, endian); + + std::vector envelope; // may be unused + if (indicator > 0 & indicator < 5) { + // not an empty envelope + + envelope.resize(4); // only 4, not supporting Z or M dims + wkb::copy_from(geometry_blob, index, envelope[0], endian); + wkb::copy_from(geometry_blob, index, envelope[1], endian); + wkb::copy_from(geometry_blob, index, envelope[2], endian); + wkb::copy_from(geometry_blob, index, envelope[3], endian); + + // ensure `index` is at beginning of data + if (indicator == 2 || indicator == 3) { + index += 2 * sizeof(double); + } else if (indicator == 4) { + index += 4 * sizeof(double); + } + } + + const std::vector geometry_data(geometry_blob.begin() + index, geometry_blob.end()); + geojson::geometry geometry = wkb::read_wkb(geometry_data); + + if (srs_id != 4326) { + return geometry; + } else { + geojson::geometry projected; + + // project coordinates from whatever they are to 4326 + boost::geometry::srs::transformation<> tr{ + bsrs::epsg(srs_id), + bsrs::epsg(4326) + }; + + tr.forward(geometry, projected); + + return projected; } } @@ -101,9 +147,84 @@ inline geojson::Feature build_feature( const std::string& geom_col ) { - const auto type = feature_type_map(geom_type); - const geojson::PropertyMap properties = build_properties(row, geom_col); - const geojson::geometry geometry = build_geometry(row, type, geom_col); + const auto type = feature_type_map(geom_type); + const auto id = row.get("id"); + std::vector bounding_box = {0, 0, 0, 0}; // TODO + geojson::PropertyMap properties = std::move(build_properties(row, geom_col)); + geojson::geometry geometry = std::move(build_geometry(row, type, geom_col)); + + switch(type) { + case geojson::FeatureType::Point: + return std::make_shared(geojson::PointFeature( + boost::get(geometry), + id, + properties, + bounding_box, + std::vector(), + std::vector(), + {} + )); + case geojson::FeatureType::LineString: + return std::make_shared(geojson::LineStringFeature( + boost::get(geometry), + id, + properties, + bounding_box, + std::vector(), + std::vector(), + {} + )); + case geojson::FeatureType::Polygon: + return std::make_shared(geojson::PolygonFeature( + boost::get(geometry), + id, + properties, + bounding_box, + std::vector(), + std::vector(), + {} + )); + case geojson::FeatureType::MultiPoint: + return std::make_shared(geojson::MultiPointFeature( + boost::get(geometry), + id, + properties, + bounding_box, + std::vector(), + std::vector(), + {} + )); + case geojson::FeatureType::MultiLineString: + return std::make_shared(geojson::MultiLineStringFeature( + boost::get(geometry), + id, + properties, + bounding_box, + std::vector(), + std::vector(), + {} + )); + case geojson::FeatureType::MultiPolygon: + return std::make_shared(geojson::MultiPolygonFeature( + boost::get(geometry), + id, + properties, + bounding_box, + std::vector(), + std::vector(), + {} + )); + default: + return std::make_shared(geojson::CollectionFeature( + std::vector{geometry}, + id, + properties, + bounding_box, + std::vector(), + std::vector(), + {} + )); + } }; inline std::shared_ptr read(const std::string& gpkg_path, const std::string& layer = "", const std::vector& ids = {}) diff --git a/include/geopackage/wkb/reader.hpp b/include/geopackage/wkb/reader.hpp index 7dc51d13c7..cf773a425e 100644 --- a/include/geopackage/wkb/reader.hpp +++ b/include/geopackage/wkb/reader.hpp @@ -8,27 +8,14 @@ #include #include -#include + +#include "JSONGeometry.hpp" namespace geopackage { namespace wkb { using byte_t = uint8_t; using byte_vector = std::vector; -struct wkb_point { double x, y; }; -struct wkb_linestring { std::vector points; }; -struct wkb_polygon { std::vector rings; }; -struct wkb_multipoint { std::vector points; }; -struct wkb_multilinestring { std::vector lines; }; -struct wkb_multipolygon { std::vector polygons; }; -using wkb_geometry = boost::variant< - wkb_point, - wkb_linestring, - wkb_polygon, - wkb_multipoint, - wkb_multilinestring, - wkb_multipolygon ->; /** * @brief @@ -70,59 +57,106 @@ inline void copy_from(byte_vector src, T& index, S& dst, uint8_t order) template inline T read_wkb_internal(const byte_vector& buffer, int& index, uint8_t order); -namespace { -template -inline output_t read_wkb_compound_internal(const byte_vector& buffer, int& index, uint8_t order) -{ - uint32_t count; - copy_from(buffer, index, count, order); - std::vector children(count); - for (auto& child : children) { - child = read_wkb_internal(buffer, index, order); - } - return output_t{children}; -} -} // anonymous namespace - #define READ_WKB_INTERNAL_SIG(output_t) output_t read_wkb_internal(const byte_vector& buffer, int& index, uint8_t order) template<> -inline READ_WKB_INTERNAL_SIG(wkb_point) +inline READ_WKB_INTERNAL_SIG(geojson::coordinate_t) { double x, y; copy_from(buffer, index, x, order); copy_from(buffer, index, y, order); - return wkb_point{x, y}; + return geojson::coordinate_t{x, y}; }; template<> -inline READ_WKB_INTERNAL_SIG(wkb_linestring) +inline READ_WKB_INTERNAL_SIG(geojson::linestring_t) { - return read_wkb_compound_internal(buffer, index, order); + uint32_t count; + copy_from(buffer, index, count, order); + + geojson::linestring_t linestring; + linestring.resize(count); + + for (auto& child : linestring) { + child = read_wkb_internal(buffer, index, order); + } + + return linestring; } template<> -inline READ_WKB_INTERNAL_SIG(wkb_polygon) +inline READ_WKB_INTERNAL_SIG(geojson::polygon_t) { - return read_wkb_compound_internal(buffer, index, order); + uint32_t count; + copy_from(buffer, index, count, order); + + geojson::polygon_t polygon; + + if (count > 1) { + polygon.inners().resize(count - 1); + } + + auto outer = read_wkb_internal(buffer, index, order); + polygon.outer().reserve(outer.size()); + for (auto& p : outer) { + polygon.outer().push_back(p); + } + + for (uint32_t i = 1; i < count; i++) { + auto inner = read_wkb_internal(buffer, index, order); + polygon.inners().at(i).reserve(inner.size()); + for (auto& p : inner) { + polygon.inners().at(i).push_back(p); + } + } + + return polygon; } template<> -inline READ_WKB_INTERNAL_SIG(wkb_multipoint) +inline READ_WKB_INTERNAL_SIG(geojson::multipoint_t) { - return read_wkb_compound_internal(buffer, index, order); + uint32_t count; + copy_from(buffer, index, count, order); + + geojson::multipoint_t mp; + mp.resize(count); + for (auto& point : mp) { + point = read_wkb_internal(buffer, index, order); + } + + return mp; } template<> -inline READ_WKB_INTERNAL_SIG(wkb_multilinestring) +inline READ_WKB_INTERNAL_SIG(geojson::multilinestring_t) { - return read_wkb_compound_internal(buffer, index, order); + uint32_t count; + copy_from(buffer, index, count, order); + + geojson::multilinestring_t ml; + ml.resize(count); + for (auto& line : ml) { + auto l = read_wkb_internal(buffer, index, order); + + } + + return ml; } template<> -inline READ_WKB_INTERNAL_SIG(wkb_multipolygon) +inline READ_WKB_INTERNAL_SIG(geojson::multipolygon_t) { - return read_wkb_compound_internal(buffer, index, order); + uint32_t count; + copy_from(buffer, index, count, order); + + geojson::multipolygon_t mpl; + mpl.resize(count); + for (auto& polygon : mpl) { + polygon = read_wkb_internal(buffer, index, order); + } + + return mpl; } #undef READ_WKB_INTERNAL_SIG @@ -161,7 +195,7 @@ static inline T read_known_wkb(const byte_vector& buffer) * @param buffer vector of bytes * @return wkb::geometry geometry struct containing the parsed WKB values. */ -static inline wkb::wkb_geometry read_wkb(const byte_vector&buffer) +static inline geojson::geometry read_wkb(const byte_vector&buffer) { if (buffer.size() < 5) { throw std::runtime_error("buffer reached end before encountering WKB"); @@ -174,15 +208,14 @@ static inline wkb::wkb_geometry read_wkb(const byte_vector&buffer) uint32_t type; copy_from(buffer, index, type, order); - wkb_geometry g; + geojson::geometry g = geojson::coordinate_t{std::nan("0"), std::nan("0")}; switch(type) { - case 1: g = read_wkb_internal(buffer, index, order); break; - case 2: g = read_wkb_internal(buffer, index, order); break; - case 3: g = read_wkb_internal(buffer, index, order); break; - case 4: g = read_wkb_internal(buffer, index, order); break; - case 5: g = read_wkb_internal(buffer, index, order); break; - case 6: g = read_wkb_internal(buffer, index, order); break; - default: g = wkb_point{std::nan("0"), std::nan("0")}; break; + case 1: g = read_wkb_internal(buffer, index, order); break; + case 2: g = read_wkb_internal(buffer, index, order); break; + case 3: g = read_wkb_internal(buffer, index, order); break; + case 4: g = read_wkb_internal(buffer, index, order); break; + case 5: g = read_wkb_internal(buffer, index, order); break; + case 6: g = read_wkb_internal(buffer, index, order); break; } return g; @@ -191,4 +224,4 @@ static inline wkb::wkb_geometry read_wkb(const byte_vector&buffer) } // namespace wkb } // namespace geopackage -#endif // NGEN_GEOPACKAGE_WKB_READER_H \ No newline at end of file +#endif // NGEN_GEOPACKAGE_WKB_READER_H diff --git a/include/geopackage/wkb/wkt.hpp b/include/geopackage/wkb/wkt.hpp deleted file mode 100644 index 2f96da7bfc..0000000000 --- a/include/geopackage/wkb/wkt.hpp +++ /dev/null @@ -1,93 +0,0 @@ -#ifndef NGEN_GEOPACKAGE_WKB_VISITOR_WKT_H -#define NGEN_GEOPACKAGE_WKB_VISITOR_WKT_H - -#include "reader.hpp" - -namespace geopackage { -namespace wkb { - -class wkt_visitor : public boost::static_visitor -{ - public: - - /** - * @brief Get the WKT form from WKB structs - * - * @tparam T WKB geometry struct type - * @param g geometry object - * @return std::string @param{g} in WKT form - */ - template - std::string operator()(T& g) const - { - return std::string(this->wkt_type(g)) + " " + this->wkt_coords(g); - } - - private: - constexpr const char* wkt_type(const wkb_point&) const { return "POINT"; } - constexpr const char* wkt_type(const wkb_linestring&) const { return "LINESTRING"; } - constexpr const char* wkt_type(const wkb_polygon&) const { return "POLYGON"; } - constexpr const char* wkt_type(const wkb_multipoint&) const { return "MULTIPOINT"; } - constexpr const char* wkt_type(const wkb_multilinestring&) const { return "MULTILINESTRING"; } - constexpr const char* wkt_type(const wkb_multipolygon&) const { return "MULTIPOLYGON"; } - - std::string wkt_coords(const wkb_point& g) const - { - std::ostringstream out; - out.precision(3); - out << std::fixed << g.x << " " << g.y; - return std::move(out).str(); - } - - std::string wkt_coords(const wkb_linestring& g) const - { - return "(" + std::accumulate( - std::next(g.points.begin()), - g.points.end(), - wkt_coords(g.points[0]), - [this](const std::string& a, const wkb_point b) { return a + "," + this->wkt_coords(b); } - ) + ")"; - } - - std::string wkt_coords(const wkb_multipoint& g) const - { - return "(" + std::accumulate( - std::next(g.points.begin()), - g.points.end(), - wkt_coords(g.points[0]), - [this](const std::string& a, const wkb_point b) { return a + "," + this->wkt_coords(b); } - ) + ")"; - } - - std::string wkt_coords(const wkb_polygon& g) const - { - std::string output; - for (const auto& gg : g.rings) { - output += "(" + wkt_coords(gg) + ")"; - } - return output; - } - - std::string wkt_coords(const wkb_multilinestring& g) const - { - std::string output; - for (const auto& gg : g.lines) { - output += "(" + wkt_coords(gg) + ")"; - } - return output; - } - - std::string wkt_coords(const wkb_multipolygon& g) const - { - std::string output; - for (const auto& gg : g.polygons) { - output += "(" + wkt_coords(gg) + ")"; - } - return output; - } -}; - -} // namespace wkb -} // namespace geopackage - -#endif // NGEN_GEOPACKAGE_WKB_VISITOR_WKT_H \ No newline at end of file diff --git a/test/geopackage/WKB_Test.cpp b/test/geopackage/WKB_Test.cpp index 9d3c3d8131..0f495e4a1a 100644 --- a/test/geopackage/WKB_Test.cpp +++ b/test/geopackage/WKB_Test.cpp @@ -1,6 +1,8 @@ +#include "JSONGeometry.hpp" +#include #include + #include -#include using namespace geopackage; @@ -29,7 +31,7 @@ class WKB_Test : public ::testing::Test this->little_endian = { 0x01, - 0x00, 0x00, 0x00, 0x01, + 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x40 }; @@ -51,10 +53,10 @@ class WKB_Test : public ::testing::Test TEST_F(WKB_Test, wkb_read_test) { - const wkb::wkb_geometry geom = wkb::read_wkb(this->wkb); + const geojson::geometry geom = wkb::read_wkb(this->wkb); EXPECT_EQ(geom.which() + 1, 3); // +1 since variant.which() is 0-based - const wkb::wkb_polygon& poly = boost::get(geom); + const geojson::polygon_t& poly = boost::get(geom); const std::vector> expected_coordinates = { {10.689, -25.092}, {34.595, -20.170}, @@ -64,20 +66,19 @@ TEST_F(WKB_Test, wkb_read_test) }; for (int i = 0; i < expected_coordinates.size(); i++) { - EXPECT_NEAR(poly.rings[0].points[i].x, expected_coordinates[i].first, 0.0001); - EXPECT_NEAR(poly.rings[0].points[i].y, expected_coordinates[i].second, 0.0001); + EXPECT_NEAR(poly.outer().at(i).get<0>(), expected_coordinates[i].first, 0.0001); + EXPECT_NEAR(poly.outer().at(i).get<1>(), expected_coordinates[i].second, 0.0001); } - - const wkb::wkt_visitor wkt_v; - const auto wkt = "POLYGON ((10.689 -25.092,34.595 -20.170,38.814 -35.639,13.502 -39.155,10.689 -25.092))"; - EXPECT_EQ(boost::apply_visitor(wkt_v, geom), wkt); - EXPECT_EQ(wkt_v(poly), wkt); } TEST_F(WKB_Test, wkb_endianness_test) { - const auto geom_big = wkb::read_known_wkb(this->big_endian); - const auto geom_little = wkb::read_known_wkb(this->little_endian); - EXPECT_NEAR(geom_big.x, geom_little.x, 0.000001); - EXPECT_NEAR(geom_big.y, geom_little.y, 0.000001); -} \ No newline at end of file + const auto geom_big = wkb::read_known_wkb(this->big_endian); + const auto geom_little = wkb::read_known_wkb(this->little_endian); + EXPECT_NEAR(geom_big.get<0>(), geom_little.get<0>(), 0.000001); + EXPECT_NEAR(geom_big.get<1>(), geom_little.get<1>(), 0.000001); + EXPECT_NEAR(geom_big.get<0>(), 2.0, 0.000001); + EXPECT_NEAR(geom_big.get<1>(), 4.0, 0.000001); + EXPECT_NEAR(geom_big.get<0>(), 2.0, 0.000001); + EXPECT_NEAR(geom_big.get<1>(), 4.0, 0.000001); +} From d30f2b023d341b45f936ff519975f6e7ece26c9e Mon Sep 17 00:00:00 2001 From: program-- Date: Mon, 1 May 2023 09:05:34 -0700 Subject: [PATCH 11/62] fixed wkb reading multi geometries and added more wkb tests --- include/geopackage/wkb/reader.hpp | 23 +++- test/geopackage/WKB_Test.cpp | 177 ++++++++++++++++++++++++------ 2 files changed, 164 insertions(+), 36 deletions(-) diff --git a/include/geopackage/wkb/reader.hpp b/include/geopackage/wkb/reader.hpp index cf773a425e..9964781bc3 100644 --- a/include/geopackage/wkb/reader.hpp +++ b/include/geopackage/wkb/reader.hpp @@ -113,6 +113,21 @@ inline READ_WKB_INTERNAL_SIG(geojson::polygon_t) return polygon; } +template +inline T read_multi_wkb_internal(const byte_vector& buffer, int& index, uint32_t expected_type) { + const byte_t new_order = buffer[index]; + index++; + + uint32_t type; + copy_from(buffer, index, type, new_order); + + if (type != expected_type) { + throw std::runtime_error("expected type " + std::to_string(expected_type) + ", but found type: " + std::to_string(type)); + } + + return read_wkb_internal(buffer, index, new_order); +}; + template<> inline READ_WKB_INTERNAL_SIG(geojson::multipoint_t) { @@ -121,8 +136,9 @@ inline READ_WKB_INTERNAL_SIG(geojson::multipoint_t) geojson::multipoint_t mp; mp.resize(count); + for (auto& point : mp) { - point = read_wkb_internal(buffer, index, order); + point = read_multi_wkb_internal(buffer, index, 1); } return mp; @@ -137,8 +153,7 @@ inline READ_WKB_INTERNAL_SIG(geojson::multilinestring_t) geojson::multilinestring_t ml; ml.resize(count); for (auto& line : ml) { - auto l = read_wkb_internal(buffer, index, order); - + line = read_multi_wkb_internal(buffer, index, 2); } return ml; @@ -153,7 +168,7 @@ inline READ_WKB_INTERNAL_SIG(geojson::multipolygon_t) geojson::multipolygon_t mpl; mpl.resize(count); for (auto& polygon : mpl) { - polygon = read_wkb_internal(buffer, index, order); + polygon = read_multi_wkb_internal(buffer, index, 3); } return mpl; diff --git a/test/geopackage/WKB_Test.cpp b/test/geopackage/WKB_Test.cpp index 0f495e4a1a..7bb5c90287 100644 --- a/test/geopackage/WKB_Test.cpp +++ b/test/geopackage/WKB_Test.cpp @@ -1,5 +1,4 @@ #include "JSONGeometry.hpp" -#include #include #include @@ -14,46 +13,137 @@ class WKB_Test : public ::testing::Test ~WKB_Test() override {} void SetUp() override { - this->wkb = { - 0x01, 0x03, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, - 0x00, 0x05, 0x00, 0x00, 0x00, 0x54, 0xE3, 0xA5, - 0x9B, 0xC4, 0x60, 0x25, 0x40, 0x64, 0x3B, 0xDF, - 0x4F, 0x8D, 0x17, 0x39, 0xC0, 0x5C, 0x8F, 0xC2, - 0xF5, 0x28, 0x4C, 0x41, 0x40, 0xEC, 0x51, 0xB8, - 0x1E, 0x85, 0x2B, 0x34, 0xC0, 0xD5, 0x78, 0xE9, - 0x26, 0x31, 0x68, 0x43, 0x40, 0x6F, 0x12, 0x83, - 0xC0, 0xCA, 0xD1, 0x41, 0xC0, 0x1B, 0x2F, 0xDD, - 0x24, 0x06, 0x01, 0x2B, 0x40, 0xA4, 0x70, 0x3D, - 0x0A, 0xD7, 0x93, 0x43, 0xC0, 0x54, 0xE3, 0xA5, - 0x9B, 0xC4, 0x60, 0x25, 0x40, 0x64, 0x3B, 0xDF, - 0x4F, 0x8D, 0x17, 0x39, 0xC0, - }; - - this->little_endian = { + this->wkb["point"] = { 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x40 }; - this->big_endian = { + this->wkb["point_big"] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }; + + // LINESTRING(30 10, 10 30, 40 40) + this->wkb["linestring"] = { + 0x01, + 0x02, 0x00, 0x00, 0x00, + 0x03, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3e, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x24, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x24, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3e, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x44, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x44, 0x40 + }; + + this->wkb["polygon"] = { + 0x01, + 0x03, 0x00, 0x00, 0x00, + 0x01, 0x00, 0x00, 0x00, + 0x05, 0x00, 0x00, 0x00, + 0x54, 0xE3, 0xA5, 0x9B, 0xC4, 0x60, 0x25, 0x40, + 0x64, 0x3B, 0xDF, 0x4F, 0x8D, 0x17, 0x39, 0xC0, + 0x5C, 0x8F, 0xC2, 0xF5, 0x28, 0x4C, 0x41, 0x40, + 0xEC, 0x51, 0xB8, 0x1E, 0x85, 0x2B, 0x34, 0xC0, + 0xD5, 0x78, 0xE9, 0x26, 0x31, 0x68, 0x43, 0x40, + 0x6F, 0x12, 0x83, 0xC0, 0xCA, 0xD1, 0x41, 0xC0, + 0x1B, 0x2F, 0xDD, 0x24, 0x06, 0x01, 0x2B, 0x40, + 0xA4, 0x70, 0x3D, 0x0A, 0xD7, 0x93, 0x43, 0xC0, + 0x54, 0xE3, 0xA5, 0x9B, 0xC4, 0x60, 0x25, 0x40, + 0x64, 0x3B, 0xDF, 0x4F, 0x8D, 0x17, 0x39, 0xC0 + }; + + // MULTIPOINT(10 40,40 30,20 20,30 10) + this->wkb["multipoint"] = { + 0x01, + 0x04, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, + 0x01, + 0x01, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x24, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x44, 0x40, + 0x01, + 0x01, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x44, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3e, 0x40, + 0x01, + 0x01, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x34, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x34, 0x40, + 0x01, + 0x01, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3e, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x24, 0x40 + }; + + // MULTILINESTRING((10 10,20 20,10 40),(40 40,30 30,40 20,30 10)) + this->wkb["multilinestring"] = { + 0x01, + 0x05, 0x00, 0x00, 0x00, + 0x02, 0x00, 0x00, 0x00, + 0x01, + 0x02, 0x00, 0x00, 0x00, + 0x03, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x24, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x24, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x34, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x34, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x24, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x44, 0x40, + 0x01, + 0x02, 0x00, 0x00, 0x00, + 0x04, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x44, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x44, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3e, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3e, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x44, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x34, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3e, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x24, 0x40 + }; } void TearDown() override {} - std::vector wkb; - std::vector little_endian; - std::vector big_endian; + std::map> wkb; }; -TEST_F(WKB_Test, wkb_read_test) +TEST_F(WKB_Test, wkb_point_test) // also tests endianness +{ + const auto geom_big = wkb::read_known_wkb(this->wkb["point_big"]); + const auto geom_little = wkb::read_known_wkb(this->wkb["point"]); + EXPECT_NEAR(geom_big.get<0>(), geom_little.get<0>(), 0.000001); + EXPECT_NEAR(geom_big.get<1>(), geom_little.get<1>(), 0.000001); + EXPECT_NEAR(geom_big.get<0>(), 2.0, 0.000001); + EXPECT_NEAR(geom_big.get<1>(), 4.0, 0.000001); + EXPECT_NEAR(geom_big.get<0>(), 2.0, 0.000001); + EXPECT_NEAR(geom_big.get<1>(), 4.0, 0.000001); +} + +TEST_F(WKB_Test, wkb_linestring_test) +{ + const geojson::geometry geom = wkb::read_wkb(this->wkb["linestring"]); + EXPECT_EQ(geom.which() + 1, 2); + + const geojson::linestring_t& line = boost::get(geom); + const std::vector> expected_coordinates = { + {30, 10}, {10, 30}, {40, 40} + }; + + for (int i = 0; i < expected_coordinates.size(); i++) { + EXPECT_NEAR(line[i].get<0>(), expected_coordinates[i].first, 0.0001); + EXPECT_NEAR(line[i].get<1>(), expected_coordinates[i].second, 0.0001); + } +} + +TEST_F(WKB_Test, wkb_polygon_test) { - const geojson::geometry geom = wkb::read_wkb(this->wkb); + const geojson::geometry geom = wkb::read_wkb(this->wkb["polygon"]); EXPECT_EQ(geom.which() + 1, 3); // +1 since variant.which() is 0-based const geojson::polygon_t& poly = boost::get(geom); @@ -71,14 +161,37 @@ TEST_F(WKB_Test, wkb_read_test) } } -TEST_F(WKB_Test, wkb_endianness_test) +TEST_F(WKB_Test, wkb_multipoint_test) { - const auto geom_big = wkb::read_known_wkb(this->big_endian); - const auto geom_little = wkb::read_known_wkb(this->little_endian); - EXPECT_NEAR(geom_big.get<0>(), geom_little.get<0>(), 0.000001); - EXPECT_NEAR(geom_big.get<1>(), geom_little.get<1>(), 0.000001); - EXPECT_NEAR(geom_big.get<0>(), 2.0, 0.000001); - EXPECT_NEAR(geom_big.get<1>(), 4.0, 0.000001); - EXPECT_NEAR(geom_big.get<0>(), 2.0, 0.000001); - EXPECT_NEAR(geom_big.get<1>(), 4.0, 0.000001); + const geojson::geometry geom = wkb::read_wkb(this->wkb["multipoint"]); + EXPECT_EQ(geom.which() + 1, 4); + + const geojson::multipoint_t& mp = boost::get(geom); + const std::vector> expected_coordinates = { + {10, 40}, {40, 30}, {20, 20}, {30, 10} + }; + + for (int i = 0; i < expected_coordinates.size(); i++) { + EXPECT_NEAR(mp[i].get<0>(), expected_coordinates[i].first, 0.0001); + EXPECT_NEAR(mp[i].get<1>(), expected_coordinates[i].second, 0.0001); + } +} + +TEST_F(WKB_Test, wkb_multilinestring_test) +{ + const geojson::geometry geom = wkb::read_wkb(this->wkb["multilinestring"]); + EXPECT_EQ(geom.which() + 1, 5); + + const geojson::multilinestring_t& mp = boost::get(geom); + const std::vector>> expected_coordinates = { + { {10, 10}, {20, 20}, {10, 40} }, + { {40, 40}, {30, 30}, {40, 20}, {30, 10} } + }; + + for (int i = 0; i < expected_coordinates.size(); i++) { + for (int j = 0; j < expected_coordinates[i].size(); j++) { + EXPECT_NEAR(mp[i][j].get<0>(), expected_coordinates[i][j].first, 0.0001); + EXPECT_NEAR(mp[i][j].get<1>(), expected_coordinates[i][j].second, 0.0001); + } + } } From 425128ca627efcbd89c8cd2ed0b7f10f52735ec7 Mon Sep 17 00:00:00 2001 From: program-- Date: Mon, 1 May 2023 11:57:26 -0700 Subject: [PATCH 12/62] added sqlite dep and tests --- CMakeLists.txt | 2 + include/geopackage/SQLite.hpp | 68 ++++++++++++------- src/geopackage/CMakeLists.txt | 4 +- test/CMakeLists.txt | 4 +- test/geopackage/SQLite_Test.cpp | 111 ++++++++++++++++++++++++++++++++ test/geopackage/WKB_Test.cpp | 2 +- 6 files changed, 162 insertions(+), 29 deletions(-) create mode 100644 test/geopackage/SQLite_Test.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index bd22129e71..b5d5156377 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -113,6 +113,8 @@ endif() find_package(Boost 1.72.0 REQUIRED) +find_package(SQLite3 REQUIRED) # required for now + # UDUNITS # Since UDUNITS is currently not really optional (yet) let's make it default to on... if (NOT (DEFINED UDUNITS_ACTIVE)) diff --git a/include/geopackage/SQLite.hpp b/include/geopackage/SQLite.hpp index 6322992069..9cdfab55cb 100644 --- a/include/geopackage/SQLite.hpp +++ b/include/geopackage/SQLite.hpp @@ -11,6 +11,14 @@ namespace geopackage { +const auto sqlite_get_notstarted_error = std::runtime_error( + "sqlite iteration is has not started, get() is not callable (call sqlite_iter::next() before)" +); + +const auto sqlite_get_done_error = std::runtime_error( + "sqlite iteration is done, get() is not callable" +); + /** * @brief Deleter used to provide smart pointer support for sqlite3 structs. */ @@ -64,9 +72,11 @@ class sqlite_iter // returns the raw pointer to the sqlite statement sqlite3_stmt* ptr() const noexcept; + // checks if int is out of range, and throws error if so + void handle_get_index(int); + public: sqlite_iter(stmt_t stmt); - ~sqlite_iter(); bool done() const noexcept; sqlite_iter& next(); sqlite_iter& restart(); @@ -78,10 +88,10 @@ class sqlite_iter const std::vector& types() const noexcept { return this->column_types; } template - T get(int col) const; + T get(int col); template - T get(const std::string& name) const; + T get(const std::string&); }; inline sqlite_iter::sqlite_iter(stmt_t stmt) @@ -99,14 +109,26 @@ inline sqlite_iter::sqlite_iter(stmt_t stmt) } }; -inline sqlite_iter::~sqlite_iter() +inline sqlite3_stmt* sqlite_iter::ptr() const noexcept { - this->stmt.reset(); + return this->stmt.get(); } -inline sqlite3_stmt* sqlite_iter::ptr() const noexcept +inline void sqlite_iter::handle_get_index(int col) { - return this->stmt.get(); + if (this->done()) { + throw sqlite_get_done_error; + } + + if (this->current_row() == -1) { + throw sqlite_get_notstarted_error; + } + + if (col < 0 || col >= this->column_count) { + throw std::out_of_range( + "column " + std::to_string(col) + " out of range of " + std::to_string(this->column_count) + " columns" + ); + } } inline bool sqlite_iter::done() const noexcept @@ -131,6 +153,7 @@ inline sqlite_iter& sqlite_iter::restart() { sqlite3_reset(this->ptr()); this->iteration_step = -1; + this->iteration_finished = false; return *this; } @@ -158,32 +181,36 @@ inline int sqlite_iter::column_index(const std::string& name) const noexcept } template<> -inline std::vector sqlite_iter::get>(int col) const +inline std::vector sqlite_iter::get>(int col) { + this->handle_get_index(col); return *reinterpret_cast*>(sqlite3_column_blob(this->ptr(), col)); } template<> -inline double sqlite_iter::get(int col) const +inline double sqlite_iter::get(int col) { + this->handle_get_index(col); return sqlite3_column_double(this->ptr(), col); } template<> -inline int sqlite_iter::get(int col) const +inline int sqlite_iter::get(int col) { + this->handle_get_index(col); return sqlite3_column_int(this->ptr(), col); } template<> -inline std::string sqlite_iter::get(int col) const +inline std::string sqlite_iter::get(int col) { + this->handle_get_index(col); // TODO: this won't work with non-ASCII text return std::string(reinterpret_cast(sqlite3_column_text(this->ptr(), col))); } template -inline T sqlite_iter::get(const std::string& name) const +inline T sqlite_iter::get(const std::string& name) { const int index = this->column_index(name); return this->get(index); @@ -226,11 +253,6 @@ class sqlite */ sqlite& operator=(sqlite&& db); - /** - * @brief Destroy the sqlite object - */ - ~sqlite(); - /** * @brief Return the originating sqlite3 database pointer * @@ -301,12 +323,6 @@ inline sqlite& sqlite::operator=(sqlite&& db) return *this; } -inline sqlite::~sqlite() -{ - this->stmt.reset(); - this->conn.reset(); -} - inline sqlite3* sqlite::connection() const noexcept { return this->conn.get(); @@ -316,7 +332,11 @@ inline bool sqlite::has_table(const std::string& table) noexcept { auto q = this->query("SELECT 1 from sqlite_master WHERE type='table' AND name = ?", table); q.next(); - return static_cast(q.get(0)); + if (q.done()) { + return false; + } else { + return static_cast(q.get(0)); + } }; inline sqlite_iter sqlite::query(const std::string& statement) diff --git a/src/geopackage/CMakeLists.txt b/src/geopackage/CMakeLists.txt index 006e4d9597..0eede20816 100644 --- a/src/geopackage/CMakeLists.txt +++ b/src/geopackage/CMakeLists.txt @@ -2,6 +2,4 @@ cmake_minimum_required(VERSION 3.10) add_library(geopackage INTERFACE) add_library(NGen::geopackage ALIAS geopackage) target_include_directories(geopackage INTERFACE ${PROJECT_SOURCE_DIR}/include/geopackage) - -find_package(Boost) -target_link_libraries(geopackage INTERFACE NGen::geojson Boost::boost) +target_link_libraries(geopackage INTERFACE NGen::geojson Boost::boost libsqlite3) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 6ca788d148..6700c9a23f 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -86,8 +86,10 @@ add_test(test_geojson ########################## GeoPackage Unit Tests add_test(test_geopackage - 1 + 2 geopackage/WKB_Test.cpp + geopackage/SQLite_Test.cpp + NGen::core NGen::geopackage) ########################## Realization Config Unit Tests diff --git a/test/geopackage/SQLite_Test.cpp b/test/geopackage/SQLite_Test.cpp new file mode 100644 index 0000000000..a8995d3fcb --- /dev/null +++ b/test/geopackage/SQLite_Test.cpp @@ -0,0 +1,111 @@ +#include "gtest/gtest.h" +#include +#include + +#include "SQLite.hpp" +#include "FileChecker.h" + +using namespace geopackage; + +class SQLite_Test : public ::testing::Test +{ + protected: + void SetUp() override + { + this->path = utils::FileChecker::find_first_readable({ + "test/data/routing/gauge_01073000.gpkg", + "../test/data/routing/gauge_01073000.gpkg", + "../../test/data/routing/gauge_01073000.gpkg" + }); + + if (this->path.empty()) { + FAIL() << "can't find gauge_01073000.gpkg"; + } + + ASSERT_NO_THROW(this->db = sqlite(this->path)); + } + + void TearDown() override {}; + + std::string path; + sqlite db; +}; + +TEST_F(SQLite_Test, sqlite_access_test) +{ + // user wants metadata + EXPECT_TRUE(this->db.has_table("gpkg_contents")); + EXPECT_FALSE(this->db.has_table("some_fake_table")); +} + +TEST_F(SQLite_Test, sqlite_query_test) +{ + if (this->db.connection() == nullptr) { + FAIL() << "database is not loaded"; + } + + // user provides a query + const std::string query = "SELECT * FROM gpkg_contents LIMIT 1"; + sqlite_iter iter = this->db.query(query); + + EXPECT_EQ(iter.num_columns(), 10); + EXPECT_EQ(iter.columns(), std::vector({ + "table_name", + "data_type", + "identifier", + "description", + "last_change", + "min_x", + "min_y", + "max_x", + "max_y", + "srs_id" + })); + + // user iterates over row + ASSERT_NO_THROW(iter.next()); + + // using column indices + EXPECT_EQ(iter.get(0), "flowpaths"); + EXPECT_EQ(iter.get(1), "features"); + EXPECT_EQ(iter.get(2), "flowpaths"); + EXPECT_EQ(iter.get(3), ""); + EXPECT_EQ(iter.get(4), "2022-10-25T14:33:51.668Z"); + EXPECT_EQ(iter.get(5), 1995218.564876059); + EXPECT_EQ(iter.get(6), 2502240.321178956); + EXPECT_EQ(iter.get(7), 2002525.992495368); + EXPECT_EQ(iter.get(8), 2508383.058762011); + EXPECT_EQ(iter.get(9), 5070); + + // using column_names + EXPECT_EQ(iter.get("table_name"), "flowpaths"); + EXPECT_EQ(iter.get("data_type"), "features"); + EXPECT_EQ(iter.get("identifier"), "flowpaths"); + EXPECT_EQ(iter.get("description"), ""); + EXPECT_EQ(iter.get("last_change"), "2022-10-25T14:33:51.668Z"); + EXPECT_EQ(iter.get("min_x"), 1995218.564876059); + EXPECT_EQ(iter.get("min_y"), 2502240.321178956); + EXPECT_EQ(iter.get("max_x"), 2002525.992495368); + EXPECT_EQ(iter.get("max_y"), 2508383.058762011); + EXPECT_EQ(iter.get("srs_id"), 5070); + + // reiteration + EXPECT_EQ(iter.current_row(), 0); + iter.restart(); + EXPECT_FALSE(iter.done()); + EXPECT_EQ(iter.current_row(), -1); + ASSERT_ANY_THROW(iter.get(0)); + iter.next(); + EXPECT_EQ(iter.current_row(), 0); + + // finishing + iter.next(); + EXPECT_TRUE(iter.done()); + EXPECT_EQ(iter.current_row(), 1); + ASSERT_THROW((iter.get(0)), std::runtime_error); + + // next should be idempotent when iteration is done + ASSERT_NO_THROW(iter.next()); + EXPECT_TRUE(iter.done()); + EXPECT_EQ(iter.current_row(), 1); +} \ No newline at end of file diff --git a/test/geopackage/WKB_Test.cpp b/test/geopackage/WKB_Test.cpp index 7bb5c90287..617645504e 100644 --- a/test/geopackage/WKB_Test.cpp +++ b/test/geopackage/WKB_Test.cpp @@ -1,4 +1,3 @@ -#include "JSONGeometry.hpp" #include #include @@ -40,6 +39,7 @@ class WKB_Test : public ::testing::Test 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x44, 0x40 }; + // POLYGON((10.689 -25.092, 34.595 -20.170, 38.814 -35.639, 13.502 -39.155, 10.689 -25.092)) this->wkb["polygon"] = { 0x01, 0x03, 0x00, 0x00, 0x00, From 8a87439a0744e70d127a269d6b32010c3647c0bf Mon Sep 17 00:00:00 2001 From: program-- Date: Mon, 1 May 2023 12:00:38 -0700 Subject: [PATCH 13/62] readd const modifier on member functions --- include/geopackage/SQLite.hpp | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/include/geopackage/SQLite.hpp b/include/geopackage/SQLite.hpp index 9cdfab55cb..a94dda629b 100644 --- a/include/geopackage/SQLite.hpp +++ b/include/geopackage/SQLite.hpp @@ -73,7 +73,7 @@ class sqlite_iter sqlite3_stmt* ptr() const noexcept; // checks if int is out of range, and throws error if so - void handle_get_index(int); + void handle_get_index(int) const; public: sqlite_iter(stmt_t stmt); @@ -88,10 +88,10 @@ class sqlite_iter const std::vector& types() const noexcept { return this->column_types; } template - T get(int col); + T get(int col) const; template - T get(const std::string&); + T get(const std::string&) const; }; inline sqlite_iter::sqlite_iter(stmt_t stmt) @@ -114,7 +114,7 @@ inline sqlite3_stmt* sqlite_iter::ptr() const noexcept return this->stmt.get(); } -inline void sqlite_iter::handle_get_index(int col) +inline void sqlite_iter::handle_get_index(int col) const { if (this->done()) { throw sqlite_get_done_error; @@ -181,28 +181,28 @@ inline int sqlite_iter::column_index(const std::string& name) const noexcept } template<> -inline std::vector sqlite_iter::get>(int col) +inline std::vector sqlite_iter::get>(int col) const { this->handle_get_index(col); return *reinterpret_cast*>(sqlite3_column_blob(this->ptr(), col)); } template<> -inline double sqlite_iter::get(int col) +inline double sqlite_iter::get(int col) const { this->handle_get_index(col); return sqlite3_column_double(this->ptr(), col); } template<> -inline int sqlite_iter::get(int col) +inline int sqlite_iter::get(int col) const { this->handle_get_index(col); return sqlite3_column_int(this->ptr(), col); } template<> -inline std::string sqlite_iter::get(int col) +inline std::string sqlite_iter::get(int col) const { this->handle_get_index(col); // TODO: this won't work with non-ASCII text @@ -210,7 +210,7 @@ inline std::string sqlite_iter::get(int col) } template -inline T sqlite_iter::get(const std::string& name) +inline T sqlite_iter::get(const std::string& name) const { const int index = this->column_index(name); return this->get(index); From 5d899b6cc6f6a4ca6d6684b7e97c5e7953c97201 Mon Sep 17 00:00:00 2001 From: program-- Date: Mon, 1 May 2023 12:30:31 -0700 Subject: [PATCH 14/62] separate GeoPackage impl from headers --- include/geopackage/GeoPackage.hpp | 312 ++--------------------- src/geopackage/CMakeLists.txt | 10 +- src/geopackage/GeoPackage/feature.cpp | 104 ++++++++ src/geopackage/GeoPackage/geometry.cpp | 73 ++++++ src/geopackage/GeoPackage/properties.cpp | 49 ++++ src/geopackage/GeoPackage/read.cpp | 75 ++++++ 6 files changed, 330 insertions(+), 293 deletions(-) create mode 100644 src/geopackage/GeoPackage/feature.cpp create mode 100644 src/geopackage/GeoPackage/geometry.cpp create mode 100644 src/geopackage/GeoPackage/properties.cpp create mode 100644 src/geopackage/GeoPackage/read.cpp diff --git a/include/geopackage/GeoPackage.hpp b/include/geopackage/GeoPackage.hpp index f226957d72..c72beb862d 100644 --- a/include/geopackage/GeoPackage.hpp +++ b/include/geopackage/GeoPackage.hpp @@ -1,301 +1,33 @@ #ifndef NGEN_GEOPACKAGE_H #define NGEN_GEOPACKAGE_H -#include -#include -#include -#include -#include -#include - -#include -#include - #include "FeatureCollection.hpp" - -#include "JSONGeometry.hpp" #include "SQLite.hpp" -#include "features/CollectionFeature.hpp" -#include "wkb/reader.hpp" - -namespace bsrs = boost::geometry::srs; namespace geopackage { -namespace { -inline const geojson::FeatureType feature_type_map(const std::string& g) -{ - if (g == "POINT") return geojson::FeatureType::Point; - if (g == "LINESTRING") return geojson::FeatureType::LineString; - if (g == "POLYGON") return geojson::FeatureType::Polygon; - if (g == "MULTIPOINT") return geojson::FeatureType::MultiPoint; - if (g == "MULTILINESTRING") return geojson::FeatureType::MultiLineString; - if (g == "MULTIPOLYGON") return geojson::FeatureType::MultiPolygon; - return geojson::FeatureType::GeometryCollection; -} -} // anonymous namespace - -inline geojson::geometry build_geometry(const sqlite_iter& row, const geojson::FeatureType geom_type, const std::string& geom_col) -{ - const std::vector geometry_blob = row.get>(geom_col); - int index = 0; - if (geometry_blob[0] != 'G' && geometry_blob[1] != 'P') { - throw std::runtime_error("expected geopackage WKB, but found invalid format instead"); - } - index += 2; - - // skip version - index++; - - // flags - const bool is_extended = geometry_blob[index] & 0x00100000; - const bool is_empty = geometry_blob[index] & 0x00010000; - const uint8_t indicator = (geometry_blob[index] & 0x00001110) >> 1; - const uint8_t endian = geometry_blob[index] & 0x00000001; - index++; - - // Read srs_id - uint32_t srs_id; - wkb::copy_from(geometry_blob, index, srs_id, endian); - - std::vector envelope; // may be unused - if (indicator > 0 & indicator < 5) { - // not an empty envelope - - envelope.resize(4); // only 4, not supporting Z or M dims - wkb::copy_from(geometry_blob, index, envelope[0], endian); - wkb::copy_from(geometry_blob, index, envelope[1], endian); - wkb::copy_from(geometry_blob, index, envelope[2], endian); - wkb::copy_from(geometry_blob, index, envelope[3], endian); - - // ensure `index` is at beginning of data - if (indicator == 2 || indicator == 3) { - index += 2 * sizeof(double); - } else if (indicator == 4) { - index += 4 * sizeof(double); - } - } - - const std::vector geometry_data(geometry_blob.begin() + index, geometry_blob.end()); - geojson::geometry geometry = wkb::read_wkb(geometry_data); - - if (srs_id != 4326) { - return geometry; - } else { - geojson::geometry projected; - - // project coordinates from whatever they are to 4326 - boost::geometry::srs::transformation<> tr{ - bsrs::epsg(srs_id), - bsrs::epsg(4326) - }; - - tr.forward(geometry, projected); - - return projected; - } -} - -inline geojson::PropertyMap build_properties(const sqlite_iter& row, const std::string& geom_col) -{ - geojson::PropertyMap properties; - - std::map property_types; - const auto data_cols = row.columns(); - const auto data_types = row.types(); - std::transform( - data_cols.begin(), - data_cols.end(), - data_types.begin(), - std::inserter(property_types, property_types.end()), [](const std::string& name, int type) { - return std::make_pair(name, type); - } - ); - - for (auto& col : property_types) { - const auto name = col.first; - const auto type = col.second; - if (name == geom_col) { - continue; - } - - geojson::JSONProperty* property = nullptr; - switch(type) { - case SQLITE_INTEGER: - *property = geojson::JSONProperty(name, row.get(name)); - break; - case SQLITE_FLOAT: - *property = geojson::JSONProperty(name, row.get(name)); - break; - case SQLITE_TEXT: - *property = geojson::JSONProperty(name, row.get(name)); - break; - default: - *property = geojson::JSONProperty(name, "null"); - break; - } - - properties.emplace(col, std::move(*property)); - } - - return properties; -} - -inline geojson::Feature build_feature( - const sqlite_iter& row, - const std::string& geom_type, - const std::string& geom_col -) -{ - const auto type = feature_type_map(geom_type); - const auto id = row.get("id"); - std::vector bounding_box = {0, 0, 0, 0}; // TODO - geojson::PropertyMap properties = std::move(build_properties(row, geom_col)); - geojson::geometry geometry = std::move(build_geometry(row, type, geom_col)); - - switch(type) { - case geojson::FeatureType::Point: - return std::make_shared(geojson::PointFeature( - boost::get(geometry), - id, - properties, - bounding_box, - std::vector(), - std::vector(), - {} - )); - case geojson::FeatureType::LineString: - return std::make_shared(geojson::LineStringFeature( - boost::get(geometry), - id, - properties, - bounding_box, - std::vector(), - std::vector(), - {} - )); - case geojson::FeatureType::Polygon: - return std::make_shared(geojson::PolygonFeature( - boost::get(geometry), - id, - properties, - bounding_box, - std::vector(), - std::vector(), - {} - )); - case geojson::FeatureType::MultiPoint: - return std::make_shared(geojson::MultiPointFeature( - boost::get(geometry), - id, - properties, - bounding_box, - std::vector(), - std::vector(), - {} - )); - case geojson::FeatureType::MultiLineString: - return std::make_shared(geojson::MultiLineStringFeature( - boost::get(geometry), - id, - properties, - bounding_box, - std::vector(), - std::vector(), - {} - )); - case geojson::FeatureType::MultiPolygon: - return std::make_shared(geojson::MultiPolygonFeature( - boost::get(geometry), - id, - properties, - bounding_box, - std::vector(), - std::vector(), - {} - )); - default: - return std::make_shared(geojson::CollectionFeature( - std::vector{geometry}, - id, - properties, - bounding_box, - std::vector(), - std::vector(), - {} - )); - } -}; - -inline std::shared_ptr read(const std::string& gpkg_path, const std::string& layer = "", const std::vector& ids = {}) -{ - sqlite db(gpkg_path); - // Check if layer exists - if (!db.has_table(layer)) { - throw std::runtime_error(""); - } - - // Layer exists, getting statement for it - std::string joined_ids = ""; - if (!ids.empty()) { - std::accumulate( - ids.begin(), - ids.end(), - joined_ids, - [](const std::string& origin, const std::string& append) { - return origin.empty() ? append : origin + "," + append; - } - ); - - joined_ids = " WHERE id (" + joined_ids + ")"; - } - - // Get layer bounding box - sqlite_iter query_get_layer_bbox = db.query("SELECT min_x, min_y, max_x, max_y FROM gpkg_contents WHERE table_name = ?", layer); - query_get_layer_bbox.next(); - const double min_x = query_get_layer_bbox.get(0); - const double min_y = query_get_layer_bbox.get(1); - const double max_x = query_get_layer_bbox.get(2); - const double max_y = query_get_layer_bbox.get(3); - - // Get number of features - sqlite_iter query_get_layer_count = db.query("SELECT COUNT(*) FROM " + layer); - query_get_layer_count.next(); - const int layer_feature_count = query_get_layer_count.get(0); - - // Get layer feature metadata (geometry column name + type) - sqlite_iter query_get_layer_geom_meta = db.query("SELECT column_name, geometry_type_name FROM gpkg_geometry_columns WHERE table_name = ?", layer); - const std::string layer_geometry_column = query_get_layer_geom_meta.get(0); - const std::string layer_geometry_type = query_get_layer_geom_meta.get(1); - - sqlite_iter query_get_layer_data_meta = db.query("SELECT column_name FROM gpkg_data_columns WHERE table_name = ?", layer); - std::vector layer_data_columns; - query_get_layer_data_meta.next(); - while (!query_get_layer_data_meta.done()) { - layer_data_columns.push_back(query_get_layer_data_meta.get(0)); - query_get_layer_data_meta.next(); - } - - // Get layer - sqlite_iter query_get_layer = db.query("SELECT * FROM " + layer + joined_ids + " ORDER BY id"); - std::vector features; - features.reserve(layer_feature_count); - query_get_layer.next(); - while(!query_get_layer.done()) { - features.push_back(build_feature( - query_get_layer, - layer_geometry_type, - layer_geometry_column - )); - } - - const auto fc = geojson::FeatureCollection( - std::move(features), - {min_x, min_y, max_x, max_y} - ); - - return std::make_shared(fc); -} +geojson::geometry build_geometry( + const sqlite_iter& row, + const geojson::FeatureType geom_type, + const std::string& geom_col +); + +geojson::PropertyMap build_properties( + const sqlite_iter& row, + const std::string& geom_col +); + +geojson::Feature build_feature( + const sqlite_iter& row, + const std::string& geom_type, + const std::string& geom_col +); + +std::shared_ptr read( + const std::string& gpkg_path, + const std::string& layer, + const std::vector& ids +); } // namespace geopackage #endif // NGEN_GEOPACKAGE_H \ No newline at end of file diff --git a/src/geopackage/CMakeLists.txt b/src/geopackage/CMakeLists.txt index 0eede20816..36cf185f10 100644 --- a/src/geopackage/CMakeLists.txt +++ b/src/geopackage/CMakeLists.txt @@ -1,5 +1,9 @@ cmake_minimum_required(VERSION 3.10) -add_library(geopackage INTERFACE) +add_library(geopackage STATIC GeoPackage/geometry.cpp + GeoPackage/properties.cpp + GeoPackage/feature.cpp + GeoPackage/read.cpp +) add_library(NGen::geopackage ALIAS geopackage) -target_include_directories(geopackage INTERFACE ${PROJECT_SOURCE_DIR}/include/geopackage) -target_link_libraries(geopackage INTERFACE NGen::geojson Boost::boost libsqlite3) +target_include_directories(geopackage PUBLIC ${PROJECT_SOURCE_DIR}/include/geopackage) +target_link_libraries(geopackage PRIVATE NGen::geojson Boost::boost libsqlite3) diff --git a/src/geopackage/GeoPackage/feature.cpp b/src/geopackage/GeoPackage/feature.cpp new file mode 100644 index 0000000000..996b757e8e --- /dev/null +++ b/src/geopackage/GeoPackage/feature.cpp @@ -0,0 +1,104 @@ +#include "GeoPackage.hpp" + +const geojson::FeatureType feature_type_map(const std::string& g) +{ + if (g == "POINT") return geojson::FeatureType::Point; + if (g == "LINESTRING") return geojson::FeatureType::LineString; + if (g == "POLYGON") return geojson::FeatureType::Polygon; + if (g == "MULTIPOINT") return geojson::FeatureType::MultiPoint; + if (g == "MULTILINESTRING") return geojson::FeatureType::MultiLineString; + if (g == "MULTIPOLYGON") return geojson::FeatureType::MultiPolygon; + return geojson::FeatureType::GeometryCollection; +} + +geojson::Feature geopackage::build_feature( + const sqlite_iter& row, + const std::string& geom_type, + const std::string& geom_col +) +{ + const auto type = feature_type_map(geom_type); + const auto id = row.get("id"); + geojson::PropertyMap properties = std::move(build_properties(row, geom_col)); + geojson::geometry geometry = std::move(build_geometry(row, type, geom_col)); + const auto geometry_bbox = bg::return_envelope>(geometry); + const std::vector bounding_box = { + geometry_bbox.min_corner().get<0>(), + geometry_bbox.min_corner().get<1>(), + geometry_bbox.max_corner().get<0>(), + geometry_bbox.max_corner().get<1>() + }; + + switch(type) { + case geojson::FeatureType::Point: + return std::make_shared(geojson::PointFeature( + boost::get(geometry), + id, + properties, + bounding_box, + std::vector(), + std::vector(), + {} + )); + case geojson::FeatureType::LineString: + return std::make_shared(geojson::LineStringFeature( + boost::get(geometry), + id, + properties, + bounding_box, + std::vector(), + std::vector(), + {} + )); + case geojson::FeatureType::Polygon: + return std::make_shared(geojson::PolygonFeature( + boost::get(geometry), + id, + properties, + bounding_box, + std::vector(), + std::vector(), + {} + )); + case geojson::FeatureType::MultiPoint: + return std::make_shared(geojson::MultiPointFeature( + boost::get(geometry), + id, + properties, + bounding_box, + std::vector(), + std::vector(), + {} + )); + case geojson::FeatureType::MultiLineString: + return std::make_shared(geojson::MultiLineStringFeature( + boost::get(geometry), + id, + properties, + bounding_box, + std::vector(), + std::vector(), + {} + )); + case geojson::FeatureType::MultiPolygon: + return std::make_shared(geojson::MultiPolygonFeature( + boost::get(geometry), + id, + properties, + bounding_box, + std::vector(), + std::vector(), + {} + )); + default: + return std::make_shared(geojson::CollectionFeature( + std::vector{geometry}, + id, + properties, + bounding_box, + std::vector(), + std::vector(), + {} + )); + } +} \ No newline at end of file diff --git a/src/geopackage/GeoPackage/geometry.cpp b/src/geopackage/GeoPackage/geometry.cpp new file mode 100644 index 0000000000..fbe9d0a139 --- /dev/null +++ b/src/geopackage/GeoPackage/geometry.cpp @@ -0,0 +1,73 @@ +#include "GeoPackage.hpp" + +#include +#include + +#include "wkb/reader.hpp" + +namespace bsrs = boost::geometry::srs; + +geojson::geometry geopackage::build_geometry( + const sqlite_iter& row, + const geojson::FeatureType geom_type, + const std::string& geom_col +) +{ + const std::vector geometry_blob = row.get>(geom_col); + int index = 0; + if (geometry_blob[0] != 'G' && geometry_blob[1] != 'P') { + throw std::runtime_error("expected geopackage WKB, but found invalid format instead"); + } + index += 2; + + // skip version + index++; + + // flags + const bool is_extended = geometry_blob[index] & 0x00100000; + const bool is_empty = geometry_blob[index] & 0x00010000; + const uint8_t indicator = (geometry_blob[index] & 0x00001110) >> 1; + const uint8_t endian = geometry_blob[index] & 0x00000001; + index++; + + // Read srs_id + uint32_t srs_id; + wkb::copy_from(geometry_blob, index, srs_id, endian); + + std::vector envelope; // may be unused + if (indicator > 0 & indicator < 5) { + // not an empty envelope + + envelope.resize(4); // only 4, not supporting Z or M dims + wkb::copy_from(geometry_blob, index, envelope[0], endian); + wkb::copy_from(geometry_blob, index, envelope[1], endian); + wkb::copy_from(geometry_blob, index, envelope[2], endian); + wkb::copy_from(geometry_blob, index, envelope[3], endian); + + // ensure `index` is at beginning of data + if (indicator == 2 || indicator == 3) { + index += 2 * sizeof(double); + } else if (indicator == 4) { + index += 4 * sizeof(double); + } + } + + const std::vector geometry_data(geometry_blob.begin() + index, geometry_blob.end()); + geojson::geometry geometry = wkb::read_wkb(geometry_data); + + if (srs_id != 4326) { + return geometry; + } else { + geojson::geometry projected; + + // project coordinates from whatever they are to 4326 + boost::geometry::srs::transformation<> tr{ + bsrs::epsg(srs_id), + bsrs::epsg(4326) + }; + + tr.forward(geometry, projected); + + return projected; + } +} \ No newline at end of file diff --git a/src/geopackage/GeoPackage/properties.cpp b/src/geopackage/GeoPackage/properties.cpp new file mode 100644 index 0000000000..4ab43ec681 --- /dev/null +++ b/src/geopackage/GeoPackage/properties.cpp @@ -0,0 +1,49 @@ +#include "GeoPackage.hpp" + +geojson::PropertyMap geopackage::build_properties( + const sqlite_iter& row, + const std::string& geom_col +) +{ + geojson::PropertyMap properties; + + std::map property_types; + const auto data_cols = row.columns(); + const auto data_types = row.types(); + std::transform( + data_cols.begin(), + data_cols.end(), + data_types.begin(), + std::inserter(property_types, property_types.end()), [](const std::string& name, int type) { + return std::make_pair(name, type); + } + ); + + for (auto& col : property_types) { + const auto name = col.first; + const auto type = col.second; + if (name == geom_col) { + continue; + } + + geojson::JSONProperty* property = nullptr; + switch(type) { + case SQLITE_INTEGER: + *property = geojson::JSONProperty(name, row.get(name)); + break; + case SQLITE_FLOAT: + *property = geojson::JSONProperty(name, row.get(name)); + break; + case SQLITE_TEXT: + *property = geojson::JSONProperty(name, row.get(name)); + break; + default: + *property = geojson::JSONProperty(name, "null"); + break; + } + + properties.emplace(col, std::move(*property)); + } + + return properties; +} \ No newline at end of file diff --git a/src/geopackage/GeoPackage/read.cpp b/src/geopackage/GeoPackage/read.cpp new file mode 100644 index 0000000000..441f2220fc --- /dev/null +++ b/src/geopackage/GeoPackage/read.cpp @@ -0,0 +1,75 @@ +#include "GeoPackage.hpp" + +std::shared_ptr geopackage::read( + const std::string& gpkg_path, + const std::string& layer = "", + const std::vector& ids = {} +) +{ + sqlite db(gpkg_path); + // Check if layer exists + if (!db.has_table(layer)) { + throw std::runtime_error(""); + } + + // Layer exists, getting statement for it + std::string joined_ids = ""; + if (!ids.empty()) { + std::accumulate( + ids.begin(), + ids.end(), + joined_ids, + [](const std::string& origin, const std::string& append) { + return origin.empty() ? append : origin + "," + append; + } + ); + + joined_ids = " WHERE id (" + joined_ids + ")"; + } + + // Get layer bounding box + sqlite_iter query_get_layer_bbox = db.query("SELECT min_x, min_y, max_x, max_y FROM gpkg_contents WHERE table_name = ?", layer); + query_get_layer_bbox.next(); + const double min_x = query_get_layer_bbox.get(0); + const double min_y = query_get_layer_bbox.get(1); + const double max_x = query_get_layer_bbox.get(2); + const double max_y = query_get_layer_bbox.get(3); + + // Get number of features + sqlite_iter query_get_layer_count = db.query("SELECT COUNT(*) FROM " + layer); + query_get_layer_count.next(); + const int layer_feature_count = query_get_layer_count.get(0); + + // Get layer feature metadata (geometry column name + type) + sqlite_iter query_get_layer_geom_meta = db.query("SELECT column_name, geometry_type_name FROM gpkg_geometry_columns WHERE table_name = ?", layer); + const std::string layer_geometry_column = query_get_layer_geom_meta.get(0); + const std::string layer_geometry_type = query_get_layer_geom_meta.get(1); + + sqlite_iter query_get_layer_data_meta = db.query("SELECT column_name FROM gpkg_data_columns WHERE table_name = ?", layer); + std::vector layer_data_columns; + query_get_layer_data_meta.next(); + while (!query_get_layer_data_meta.done()) { + layer_data_columns.push_back(query_get_layer_data_meta.get(0)); + query_get_layer_data_meta.next(); + } + + // Get layer + sqlite_iter query_get_layer = db.query("SELECT * FROM " + layer + joined_ids + " ORDER BY id"); + std::vector features; + features.reserve(layer_feature_count); + query_get_layer.next(); + while(!query_get_layer.done()) { + features.push_back(build_feature( + query_get_layer, + layer_geometry_type, + layer_geometry_column + )); + } + + const auto fc = geojson::FeatureCollection( + std::move(features), + {min_x, min_y, max_x, max_y} + ); + + return std::make_shared(fc); +} \ No newline at end of file From c92fd6571751b399b02f6038a7b7f7e7ea872914 Mon Sep 17 00:00:00 2001 From: program-- Date: Mon, 1 May 2023 12:30:57 -0700 Subject: [PATCH 15/62] add geopackage tests --- test/CMakeLists.txt | 3 ++- test/geopackage/GeoPackage_Test.cpp | 30 +++++++++++++++++++++++++++++ test/geopackage/SQLite_Test.cpp | 2 -- 3 files changed, 32 insertions(+), 3 deletions(-) create mode 100644 test/geopackage/GeoPackage_Test.cpp diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 6700c9a23f..29a48ccf57 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -86,9 +86,10 @@ add_test(test_geojson ########################## GeoPackage Unit Tests add_test(test_geopackage - 2 + 3 geopackage/WKB_Test.cpp geopackage/SQLite_Test.cpp + geopackage/GeoPackage_Test.cpp NGen::core NGen::geopackage) diff --git a/test/geopackage/GeoPackage_Test.cpp b/test/geopackage/GeoPackage_Test.cpp new file mode 100644 index 0000000000..fd4c24d171 --- /dev/null +++ b/test/geopackage/GeoPackage_Test.cpp @@ -0,0 +1,30 @@ +#include + +#include "GeoPackage.hpp" +#include "FileChecker.h" + +class GeoPackage_Test : public ::testing::Test +{ + protected: + void SetUp() override + { + this->path = utils::FileChecker::find_first_readable({ + "test/data/routing/gauge_01073000.gpkg", + "../test/data/routing/gauge_01073000.gpkg", + "../../test/data/routing/gauge_01073000.gpkg" + }); + + if (this->path.empty()) { + FAIL() << "can't find gauge_01073000.gpkg"; + } + } + + void TearDown() override {}; + + std::string path; +}; + +TEST_F(GeoPackage_Test, geopackage_read_test) +{ + const auto gpkg = geopackage::read(this->path, "flowpaths"); +} \ No newline at end of file diff --git a/test/geopackage/SQLite_Test.cpp b/test/geopackage/SQLite_Test.cpp index a8995d3fcb..7695b93695 100644 --- a/test/geopackage/SQLite_Test.cpp +++ b/test/geopackage/SQLite_Test.cpp @@ -1,6 +1,4 @@ -#include "gtest/gtest.h" #include -#include #include "SQLite.hpp" #include "FileChecker.h" From 94229a87ad491f0362962aa95de02e71cfff5fe5 Mon Sep 17 00:00:00 2001 From: program-- Date: Mon, 1 May 2023 12:32:27 -0700 Subject: [PATCH 16/62] move impl out to parent folder --- src/geopackage/{GeoPackage => }/feature.cpp | 0 src/geopackage/{GeoPackage => }/geometry.cpp | 0 src/geopackage/{GeoPackage => }/properties.cpp | 0 src/geopackage/{GeoPackage => }/read.cpp | 0 4 files changed, 0 insertions(+), 0 deletions(-) rename src/geopackage/{GeoPackage => }/feature.cpp (100%) rename src/geopackage/{GeoPackage => }/geometry.cpp (100%) rename src/geopackage/{GeoPackage => }/properties.cpp (100%) rename src/geopackage/{GeoPackage => }/read.cpp (100%) diff --git a/src/geopackage/GeoPackage/feature.cpp b/src/geopackage/feature.cpp similarity index 100% rename from src/geopackage/GeoPackage/feature.cpp rename to src/geopackage/feature.cpp diff --git a/src/geopackage/GeoPackage/geometry.cpp b/src/geopackage/geometry.cpp similarity index 100% rename from src/geopackage/GeoPackage/geometry.cpp rename to src/geopackage/geometry.cpp diff --git a/src/geopackage/GeoPackage/properties.cpp b/src/geopackage/properties.cpp similarity index 100% rename from src/geopackage/GeoPackage/properties.cpp rename to src/geopackage/properties.cpp diff --git a/src/geopackage/GeoPackage/read.cpp b/src/geopackage/read.cpp similarity index 100% rename from src/geopackage/GeoPackage/read.cpp rename to src/geopackage/read.cpp From 8f54e9570956fb6eaac4178abb327418dc8dbcd8 Mon Sep 17 00:00:00 2001 From: program-- Date: Mon, 1 May 2023 12:46:01 -0700 Subject: [PATCH 17/62] reorg wkb impl; move wkb::copy_from -> utils::copy_from --- .../geopackage/{wkb/reader.hpp => WKB.hpp} | 59 +++++-------------- include/utilities/EndianCopy.hpp | 40 +++++++++++++ src/geopackage/CMakeLists.txt | 9 +-- src/geopackage/geometry.cpp | 12 ++-- 4 files changed, 65 insertions(+), 55 deletions(-) rename include/geopackage/{wkb/reader.hpp => WKB.hpp} (78%) create mode 100644 include/utilities/EndianCopy.hpp diff --git a/include/geopackage/wkb/reader.hpp b/include/geopackage/WKB.hpp similarity index 78% rename from include/geopackage/wkb/reader.hpp rename to include/geopackage/WKB.hpp index 9964781bc3..ba303e1756 100644 --- a/include/geopackage/wkb/reader.hpp +++ b/include/geopackage/WKB.hpp @@ -1,14 +1,7 @@ #ifndef NGEN_GEOPACKAGE_WKB_READER_H #define NGEN_GEOPACKAGE_WKB_READER_H -#include -#include -#include -#include -#include - -#include - +#include "EndianCopy.hpp" #include "JSONGeometry.hpp" namespace geopackage { @@ -17,33 +10,7 @@ namespace wkb { using byte_t = uint8_t; using byte_vector = std::vector; -/** - * @brief - * Copies bytes from @param{src} to @param{dst}, - * converts the endianness to native, - * and increments @param{index} by the number of bytes - * used to store @tparam{S} - * - * @tparam T an integral type - * @tparam S a primitive type - * @param src a vector of bytes - * @param index an integral type tracking the starting position of @param{dst}'s memory - * @param dst output primitive - * @param order endianness value (0x01 == Little; 0x00 == Big) - */ -template -inline void copy_from(byte_vector src, T& index, S& dst, uint8_t order) -{ - std::memcpy(&dst, &src[index], sizeof(S)); - - if (order == 0x01) { - boost::endian::little_to_native_inplace(dst); - } else { - boost::endian::big_to_native_inplace(dst); - } - - index += sizeof(S); -} +namespace { /** * @brief Recursively read WKB data into known structs @@ -63,8 +30,8 @@ template<> inline READ_WKB_INTERNAL_SIG(geojson::coordinate_t) { double x, y; - copy_from(buffer, index, x, order); - copy_from(buffer, index, y, order); + utils::copy_from(buffer, index, x, order); + utils::copy_from(buffer, index, y, order); return geojson::coordinate_t{x, y}; }; @@ -72,7 +39,7 @@ template<> inline READ_WKB_INTERNAL_SIG(geojson::linestring_t) { uint32_t count; - copy_from(buffer, index, count, order); + utils::copy_from(buffer, index, count, order); geojson::linestring_t linestring; linestring.resize(count); @@ -88,7 +55,7 @@ template<> inline READ_WKB_INTERNAL_SIG(geojson::polygon_t) { uint32_t count; - copy_from(buffer, index, count, order); + utils::copy_from(buffer, index, count, order); geojson::polygon_t polygon; @@ -119,7 +86,7 @@ inline T read_multi_wkb_internal(const byte_vector& buffer, int& index, uint32_t index++; uint32_t type; - copy_from(buffer, index, type, new_order); + utils::copy_from(buffer, index, type, new_order); if (type != expected_type) { throw std::runtime_error("expected type " + std::to_string(expected_type) + ", but found type: " + std::to_string(type)); @@ -132,7 +99,7 @@ template<> inline READ_WKB_INTERNAL_SIG(geojson::multipoint_t) { uint32_t count; - copy_from(buffer, index, count, order); + utils::copy_from(buffer, index, count, order); geojson::multipoint_t mp; mp.resize(count); @@ -148,7 +115,7 @@ template<> inline READ_WKB_INTERNAL_SIG(geojson::multilinestring_t) { uint32_t count; - copy_from(buffer, index, count, order); + utils::copy_from(buffer, index, count, order); geojson::multilinestring_t ml; ml.resize(count); @@ -163,7 +130,7 @@ template<> inline READ_WKB_INTERNAL_SIG(geojson::multipolygon_t) { uint32_t count; - copy_from(buffer, index, count, order); + utils::copy_from(buffer, index, count, order); geojson::multipolygon_t mpl; mpl.resize(count); @@ -176,6 +143,8 @@ inline READ_WKB_INTERNAL_SIG(geojson::multipolygon_t) #undef READ_WKB_INTERNAL_SIG +} // anonymouse namespace + /** * @brief Read (known) WKB into a specific geometry struct * @@ -199,7 +168,7 @@ static inline T read_known_wkb(const byte_vector& buffer) index++; uint32_t type; - copy_from(buffer, index, type, order); + utils::copy_from(buffer, index, type, order); return read_wkb_internal(buffer, index, order); }; @@ -221,7 +190,7 @@ static inline geojson::geometry read_wkb(const byte_vector&buffer) index++; uint32_t type; - copy_from(buffer, index, type, order); + utils::copy_from(buffer, index, type, order); geojson::geometry g = geojson::coordinate_t{std::nan("0"), std::nan("0")}; switch(type) { diff --git a/include/utilities/EndianCopy.hpp b/include/utilities/EndianCopy.hpp new file mode 100644 index 0000000000..35e254e039 --- /dev/null +++ b/include/utilities/EndianCopy.hpp @@ -0,0 +1,40 @@ +#ifndef NGEN_ENDIANCOPY_H +#define NGEN_ENDIANCOPY_H + +#include + +#include + +namespace utils { + +/** + * @brief + * Copies bytes from @param{src} to @param{dst}, + * converts the endianness to native, + * and increments @param{index} by the number of bytes + * used to store @tparam{S} + * + * @tparam T an integral type + * @tparam S a primitive type + * @param src a vector of bytes + * @param index an integral type tracking the starting position of @param{dst}'s memory + * @param dst output primitive + * @param order endianness value (0x01 == Little; 0x00 == Big) + */ +template +void copy_from(std::vector src, T& index, S& dst, uint8_t order) +{ + std::memcpy(&dst, &src[index], sizeof(S)); + + if (order == 0x01) { + boost::endian::little_to_native_inplace(dst); + } else { + boost::endian::big_to_native_inplace(dst); + } + + index += sizeof(S); +} + +} + +#endif // NGEN_ENDIANCOPY_H \ No newline at end of file diff --git a/src/geopackage/CMakeLists.txt b/src/geopackage/CMakeLists.txt index 36cf185f10..20096cd54c 100644 --- a/src/geopackage/CMakeLists.txt +++ b/src/geopackage/CMakeLists.txt @@ -1,9 +1,10 @@ cmake_minimum_required(VERSION 3.10) -add_library(geopackage STATIC GeoPackage/geometry.cpp - GeoPackage/properties.cpp - GeoPackage/feature.cpp - GeoPackage/read.cpp +add_library(geopackage STATIC geometry.cpp + properties.cpp + feature.cpp + read.cpp ) add_library(NGen::geopackage ALIAS geopackage) target_include_directories(geopackage PUBLIC ${PROJECT_SOURCE_DIR}/include/geopackage) +target_include_directories(geopackage PRIVATE ${PROJECT_SOURCE_DIR}/include/utilities) target_link_libraries(geopackage PRIVATE NGen::geojson Boost::boost libsqlite3) diff --git a/src/geopackage/geometry.cpp b/src/geopackage/geometry.cpp index fbe9d0a139..325da4d562 100644 --- a/src/geopackage/geometry.cpp +++ b/src/geopackage/geometry.cpp @@ -3,7 +3,7 @@ #include #include -#include "wkb/reader.hpp" +#include "WKB.hpp" namespace bsrs = boost::geometry::srs; @@ -32,17 +32,17 @@ geojson::geometry geopackage::build_geometry( // Read srs_id uint32_t srs_id; - wkb::copy_from(geometry_blob, index, srs_id, endian); + utils::copy_from(geometry_blob, index, srs_id, endian); std::vector envelope; // may be unused if (indicator > 0 & indicator < 5) { // not an empty envelope envelope.resize(4); // only 4, not supporting Z or M dims - wkb::copy_from(geometry_blob, index, envelope[0], endian); - wkb::copy_from(geometry_blob, index, envelope[1], endian); - wkb::copy_from(geometry_blob, index, envelope[2], endian); - wkb::copy_from(geometry_blob, index, envelope[3], endian); + utils::copy_from(geometry_blob, index, envelope[0], endian); + utils::copy_from(geometry_blob, index, envelope[1], endian); + utils::copy_from(geometry_blob, index, envelope[2], endian); + utils::copy_from(geometry_blob, index, envelope[3], endian); // ensure `index` is at beginning of data if (indicator == 2 || indicator == 3) { From f5b6af926eed04c0c927e983debeac977774dd98 Mon Sep 17 00:00:00 2001 From: program-- Date: Mon, 1 May 2023 12:54:08 -0700 Subject: [PATCH 18/62] separate sqlite impl from header --- include/geopackage/SQLite.hpp | 204 ++--------------------------- src/geopackage/CMakeLists.txt | 2 + src/geopackage/sqlite/database.cpp | 71 ++++++++++ src/geopackage/sqlite/iterator.cpp | 118 +++++++++++++++++ 4 files changed, 200 insertions(+), 195 deletions(-) create mode 100644 src/geopackage/sqlite/database.cpp create mode 100644 src/geopackage/sqlite/iterator.cpp diff --git a/include/geopackage/SQLite.hpp b/include/geopackage/SQLite.hpp index a94dda629b..a18beb9d89 100644 --- a/include/geopackage/SQLite.hpp +++ b/include/geopackage/SQLite.hpp @@ -1,12 +1,10 @@ #ifndef NGEN_GEOPACKAGE_SQLITE_H #define NGEN_GEOPACKAGE_SQLITE_H - -#include -#include -#include #include #include +#include + #include namespace geopackage { @@ -45,7 +43,7 @@ using stmt_t = std::shared_ptr; * @param code sqlite3 result code * @return std::runtime_error */ -static inline std::runtime_error sqlite_error(const std::string& f, int code) +inline std::runtime_error sqlite_error(const std::string& f, int code) { std::string errmsg = f + " returned code " + std::to_string(code); return std::runtime_error(errmsg); @@ -94,121 +92,6 @@ class sqlite_iter T get(const std::string&) const; }; -inline sqlite_iter::sqlite_iter(stmt_t stmt) - : stmt(stmt) -{ - this->column_count = sqlite3_column_count(this->ptr()); - this->column_names = std::vector(); - this->column_names.reserve(this->column_count); - this->column_types = std::vector(); - this->column_types.reserve(this->column_count); - - for (int i = 0; i < this->column_count; i++) { - this->column_names.push_back(sqlite3_column_name(this->ptr(), i)); - this->column_types.push_back(sqlite3_column_type(this->ptr(), i)); - } -}; - -inline sqlite3_stmt* sqlite_iter::ptr() const noexcept -{ - return this->stmt.get(); -} - -inline void sqlite_iter::handle_get_index(int col) const -{ - if (this->done()) { - throw sqlite_get_done_error; - } - - if (this->current_row() == -1) { - throw sqlite_get_notstarted_error; - } - - if (col < 0 || col >= this->column_count) { - throw std::out_of_range( - "column " + std::to_string(col) + " out of range of " + std::to_string(this->column_count) + " columns" - ); - } -} - -inline bool sqlite_iter::done() const noexcept -{ - return this->iteration_finished; -} - -inline sqlite_iter& sqlite_iter::next() -{ - if (!this->done()) { - const int returncode = sqlite3_step(this->ptr()); - if (returncode == SQLITE_DONE) { - this->iteration_finished = true; - } - this->iteration_step++; - } - - return *this; -} - -inline sqlite_iter& sqlite_iter::restart() -{ - sqlite3_reset(this->ptr()); - this->iteration_step = -1; - this->iteration_finished = false; - return *this; -} - -inline void sqlite_iter::close() -{ - this->~sqlite_iter(); -} - -inline int sqlite_iter::current_row() const noexcept -{ - return this->iteration_step; -} - -inline int sqlite_iter::num_columns() const noexcept -{ - return this->column_count; -} - -inline int sqlite_iter::column_index(const std::string& name) const noexcept -{ - const ptrdiff_t pos = - std::distance(this->column_names.begin(), std::find(this->column_names.begin(), this->column_names.end(), name)); - - return pos >= this->column_names.size() ? -1 : pos; -} - -template<> -inline std::vector sqlite_iter::get>(int col) const -{ - this->handle_get_index(col); - return *reinterpret_cast*>(sqlite3_column_blob(this->ptr(), col)); -} - -template<> -inline double sqlite_iter::get(int col) const -{ - this->handle_get_index(col); - return sqlite3_column_double(this->ptr(), col); -} - -template<> -inline int sqlite_iter::get(int col) const -{ - this->handle_get_index(col); - return sqlite3_column_int(this->ptr(), col); -} - -template<> -inline std::string sqlite_iter::get(int col) const -{ - this->handle_get_index(col); - // TODO: this won't work with non-ASCII text - return std::string(reinterpret_cast(sqlite3_column_text(this->ptr(), col))); -} - template inline T sqlite_iter::get(const std::string& name) const { @@ -217,7 +100,7 @@ inline T sqlite_iter::get(const std::string& name) const } /** - * @brief Wrapper around SQLite3 Databases + * @brief Wrapper around a SQLite3 database */ class sqlite { @@ -272,89 +155,20 @@ class sqlite /** * Query the SQLite Database and get the result * @param statement String query - * @return read-only SQLite row iterator (see: [sqlite_iter]) + * @return sqlite_iter SQLite row iterator */ sqlite_iter query(const std::string& statement); /** - * @brief TODO!!! - * - * @param statement - * @param params - * @return sqlite_iter* + * Query the SQLite Database with a bound statement and get the result + * @param statement String query with parameters + * @param params parameters to bind to statement + * @return sqlite_iter SQLite row iterator */ template sqlite_iter query(const std::string& statement, T const&... params); }; -inline sqlite::sqlite(const std::string& path) -{ - sqlite3* conn; - int code = sqlite3_open_v2(path.c_str(), &conn, SQLITE_OPEN_READONLY, NULL); - if (code != SQLITE_OK) { - throw sqlite_error("sqlite3_open_v2", code); - } - this->conn = sqlite_t(conn); -} - -inline sqlite::sqlite(sqlite& db) -{ - this->conn = std::move(db.conn); - this->stmt = db.stmt; -} - -inline sqlite& sqlite::operator=(sqlite& db) -{ - this->conn = std::move(db.conn); - this->stmt = db.stmt; - return *this; -} - -inline sqlite::sqlite(sqlite&& db) -{ - this->conn = std::move(db.conn); - this->stmt = db.stmt; -} - -inline sqlite& sqlite::operator=(sqlite&& db) -{ - this->conn = std::move(db.conn); - this->stmt = db.stmt; - return *this; -} - -inline sqlite3* sqlite::connection() const noexcept -{ - return this->conn.get(); -} - -inline bool sqlite::has_table(const std::string& table) noexcept -{ - auto q = this->query("SELECT 1 from sqlite_master WHERE type='table' AND name = ?", table); - q.next(); - if (q.done()) { - return false; - } else { - return static_cast(q.get(0)); - } -}; - -inline sqlite_iter sqlite::query(const std::string& statement) -{ - sqlite3_stmt* stmt; - const auto cstmt = statement.c_str(); - const int code = sqlite3_prepare_v2(this->connection(), cstmt, statement.length() + 1, &stmt, NULL); - - if (code != SQLITE_OK) { - // something happened, can probably switch on result codes - // https://www.sqlite.org/rescode.html - throw sqlite_error("sqlite3_prepare_v2", code); - } - - this->stmt = stmt_t(stmt, sqlite_deleter{}); - return sqlite_iter(this->stmt); -} - template inline sqlite_iter sqlite::query(const std::string& statement, T const&... params) { diff --git a/src/geopackage/CMakeLists.txt b/src/geopackage/CMakeLists.txt index 20096cd54c..56b5f060f2 100644 --- a/src/geopackage/CMakeLists.txt +++ b/src/geopackage/CMakeLists.txt @@ -3,6 +3,8 @@ add_library(geopackage STATIC geometry.cpp properties.cpp feature.cpp read.cpp + sqlite/iterator.cpp + sqlite/database.cpp ) add_library(NGen::geopackage ALIAS geopackage) target_include_directories(geopackage PUBLIC ${PROJECT_SOURCE_DIR}/include/geopackage) diff --git a/src/geopackage/sqlite/database.cpp b/src/geopackage/sqlite/database.cpp new file mode 100644 index 0000000000..50868ad0bf --- /dev/null +++ b/src/geopackage/sqlite/database.cpp @@ -0,0 +1,71 @@ +#include "SQLite.hpp" + +using namespace geopackage; + +sqlite::sqlite(const std::string& path) +{ + sqlite3* conn; + int code = sqlite3_open_v2(path.c_str(), &conn, SQLITE_OPEN_READONLY, NULL); + if (code != SQLITE_OK) { + throw sqlite_error("sqlite3_open_v2", code); + } + this->conn = sqlite_t(conn); +} + +sqlite::sqlite(sqlite& db) +{ + this->conn = std::move(db.conn); + this->stmt = db.stmt; +} + +sqlite& sqlite::operator=(sqlite& db) +{ + this->conn = std::move(db.conn); + this->stmt = db.stmt; + return *this; +} + +sqlite::sqlite(sqlite&& db) +{ + this->conn = std::move(db.conn); + this->stmt = db.stmt; +} + +sqlite& sqlite::operator=(sqlite&& db) +{ + this->conn = std::move(db.conn); + this->stmt = db.stmt; + return *this; +} + +sqlite3* sqlite::connection() const noexcept +{ + return this->conn.get(); +} + +bool sqlite::has_table(const std::string& table) noexcept +{ + auto q = this->query("SELECT 1 from sqlite_master WHERE type='table' AND name = ?", table); + q.next(); + if (q.done()) { + return false; + } else { + return static_cast(q.get(0)); + } +}; + +sqlite_iter sqlite::query(const std::string& statement) +{ + sqlite3_stmt* stmt; + const auto cstmt = statement.c_str(); + const int code = sqlite3_prepare_v2(this->connection(), cstmt, statement.length() + 1, &stmt, NULL); + + if (code != SQLITE_OK) { + // something happened, can probably switch on result codes + // https://www.sqlite.org/rescode.html + throw sqlite_error("sqlite3_prepare_v2", code); + } + + this->stmt = stmt_t(stmt, sqlite_deleter{}); + return sqlite_iter(this->stmt); +} \ No newline at end of file diff --git a/src/geopackage/sqlite/iterator.cpp b/src/geopackage/sqlite/iterator.cpp new file mode 100644 index 0000000000..dcc7f5ba96 --- /dev/null +++ b/src/geopackage/sqlite/iterator.cpp @@ -0,0 +1,118 @@ +#include "SQLite.hpp" + +using namespace geopackage; + +sqlite_iter::sqlite_iter(stmt_t stmt) + : stmt(stmt) +{ + this->column_count = sqlite3_column_count(this->ptr()); + this->column_names = std::vector(); + this->column_names.reserve(this->column_count); + this->column_types = std::vector(); + this->column_types.reserve(this->column_count); + + for (int i = 0; i < this->column_count; i++) { + this->column_names.push_back(sqlite3_column_name(this->ptr(), i)); + this->column_types.push_back(sqlite3_column_type(this->ptr(), i)); + } +}; + +sqlite3_stmt* sqlite_iter::ptr() const noexcept +{ + return this->stmt.get(); +} + +void sqlite_iter::handle_get_index(int col) const +{ + if (this->done()) { + throw sqlite_get_done_error; + } + + if (this->current_row() == -1) { + throw sqlite_get_notstarted_error; + } + + if (col < 0 || col >= this->column_count) { + throw std::out_of_range( + "column " + std::to_string(col) + " out of range of " + std::to_string(this->column_count) + " columns" + ); + } +} + +bool sqlite_iter::done() const noexcept +{ + return this->iteration_finished; +} + +sqlite_iter& sqlite_iter::next() +{ + if (!this->done()) { + const int returncode = sqlite3_step(this->ptr()); + if (returncode == SQLITE_DONE) { + this->iteration_finished = true; + } + this->iteration_step++; + } + + return *this; +} + +sqlite_iter& sqlite_iter::restart() +{ + sqlite3_reset(this->ptr()); + this->iteration_step = -1; + this->iteration_finished = false; + return *this; +} + +void sqlite_iter::close() +{ + this->~sqlite_iter(); +} + +int sqlite_iter::current_row() const noexcept +{ + return this->iteration_step; +} + +int sqlite_iter::num_columns() const noexcept +{ + return this->column_count; +} + +int sqlite_iter::column_index(const std::string& name) const noexcept +{ + const ptrdiff_t pos = + std::distance(this->column_names.begin(), std::find(this->column_names.begin(), this->column_names.end(), name)); + + return pos >= this->column_names.size() ? -1 : pos; +} + +template<> +std::vector sqlite_iter::get>(int col) const +{ + this->handle_get_index(col); + return *reinterpret_cast*>(sqlite3_column_blob(this->ptr(), col)); +} + +template<> +double sqlite_iter::get(int col) const +{ + this->handle_get_index(col); + return sqlite3_column_double(this->ptr(), col); +} + +template<> +int sqlite_iter::get(int col) const +{ + this->handle_get_index(col); + return sqlite3_column_int(this->ptr(), col); +} + +template<> +std::string sqlite_iter::get(int col) const +{ + this->handle_get_index(col); + // TODO: this won't work with non-ASCII text + return std::string(reinterpret_cast(sqlite3_column_text(this->ptr(), col))); +} \ No newline at end of file From 9fe5baec08c68f6500fefbac35c9ce643771210b Mon Sep 17 00:00:00 2001 From: program-- Date: Tue, 2 May 2023 13:06:22 -0700 Subject: [PATCH 19/62] finally got projections working, gpkg read success --- include/geopackage/SQLite.hpp | 3 +- include/geopackage/WKB.hpp | 279 ++++++++++++++++++---------- src/geopackage/CMakeLists.txt | 2 +- src/geopackage/geometry.cpp | 37 ++-- src/geopackage/properties.cpp | 42 +++-- src/geopackage/read.cpp | 30 ++- src/geopackage/sqlite/database.cpp | 3 +- src/geopackage/sqlite/iterator.cpp | 11 +- test/geopackage/GeoPackage_Test.cpp | 3 +- test/geopackage/WKB_Test.cpp | 39 ++-- 10 files changed, 259 insertions(+), 190 deletions(-) diff --git a/include/geopackage/SQLite.hpp b/include/geopackage/SQLite.hpp index a18beb9d89..1d1a5a3be2 100644 --- a/include/geopackage/SQLite.hpp +++ b/include/geopackage/SQLite.hpp @@ -177,7 +177,8 @@ inline sqlite_iter sqlite::query(const std::string& statement, T const&... param const int code = sqlite3_prepare_v2(this->connection(), cstmt, statement.length() + 1, &stmt, NULL); if (code != SQLITE_OK) { - throw sqlite_error("sqlite3_prepare_v2", code); + + throw sqlite_error("[with statement: " + std::string(sqlite3_errmsg(this->conn.get())) + "] " + "sqlite3_prepare_v2", code); } std::vector binds{ { params... } }; diff --git a/include/geopackage/WKB.hpp b/include/geopackage/WKB.hpp index ba303e1756..d688e98870 100644 --- a/include/geopackage/WKB.hpp +++ b/include/geopackage/WKB.hpp @@ -1,76 +1,92 @@ -#ifndef NGEN_GEOPACKAGE_WKB_READER_H -#define NGEN_GEOPACKAGE_WKB_READER_H +#ifndef NGEN_GEOPACKAGE_WKB_H +#define NGEN_GEOPACKAGE_WKB_H #include "EndianCopy.hpp" #include "JSONGeometry.hpp" +#include +#include +#include +#include -namespace geopackage { -namespace wkb { - -using byte_t = uint8_t; -using byte_vector = std::vector; +namespace bg = boost::geometry; -namespace { +namespace geopackage { -/** - * @brief Recursively read WKB data into known structs - * - * @tparam T geometry struct type - * @param buffer vector of bytes - * @param index tracked buffer index - * @param order endianness - * @return T parsed WKB in geometry struct - */ -template -inline T read_wkb_internal(const byte_vector& buffer, int& index, uint8_t order); +struct wkb { + using point_t = bg::model::point; + using linestring_t = bg::model::linestring; + using polygon_t = bg::model::polygon; + using multipoint_t = bg::model::multi_point; + using multilinestring_t = bg::model::multi_linestring; + using multipolygon_t = bg::model::multi_polygon; + using geometry = boost::variant< + point_t, + linestring_t, + polygon_t, + multipoint_t, + multilinestring_t, + multipolygon_t + >; + + using byte_t = uint8_t; + using byte_vector = std::vector; + + struct wgs84; + + wkb() = delete; + static geometry read(const byte_vector&); + + private: + static point_t read_point(const byte_vector&, int&, uint8_t); + static linestring_t read_linestring(const byte_vector&, int&, uint8_t); + static polygon_t read_polygon(const byte_vector&, int&, uint8_t); + static multipoint_t read_multipoint(const byte_vector&, int&, uint8_t); + static multilinestring_t read_multilinestring(const byte_vector&, int&, uint8_t); + static multipolygon_t read_multipolygon(const byte_vector&, int&, uint8_t); +}; -#define READ_WKB_INTERNAL_SIG(output_t) output_t read_wkb_internal(const byte_vector& buffer, int& index, uint8_t order) -template<> -inline READ_WKB_INTERNAL_SIG(geojson::coordinate_t) +inline typename wkb::point_t wkb::read_point(const byte_vector& buffer, int& index, uint8_t order) { double x, y; utils::copy_from(buffer, index, x, order); utils::copy_from(buffer, index, y, order); - return geojson::coordinate_t{x, y}; -}; + return point_t{x, y}; +} -template<> -inline READ_WKB_INTERNAL_SIG(geojson::linestring_t) +inline typename wkb::linestring_t wkb::read_linestring(const byte_vector& buffer, int& index, uint8_t order) { uint32_t count; utils::copy_from(buffer, index, count, order); - geojson::linestring_t linestring; + linestring_t linestring; linestring.resize(count); - for (auto& child : linestring) { - child = read_wkb_internal(buffer, index, order); + child = read_point(buffer, index, order); } return linestring; } -template<> -inline READ_WKB_INTERNAL_SIG(geojson::polygon_t) +inline typename wkb::polygon_t wkb::read_polygon(const byte_vector& buffer, int& index, uint8_t order) { uint32_t count; utils::copy_from(buffer, index, count, order); - geojson::polygon_t polygon; + polygon_t polygon; if (count > 1) { polygon.inners().resize(count - 1); } - auto outer = read_wkb_internal(buffer, index, order); + auto outer = read_linestring(buffer, index, order); polygon.outer().reserve(outer.size()); for (auto& p : outer) { polygon.outer().push_back(p); } for (uint32_t i = 1; i < count; i++) { - auto inner = read_wkb_internal(buffer, index, order); + auto inner = read_linestring(buffer, index, order); polygon.inners().at(i).reserve(inner.size()); for (auto& p : inner) { polygon.inners().at(i).push_back(p); @@ -80,87 +96,79 @@ inline READ_WKB_INTERNAL_SIG(geojson::polygon_t) return polygon; } -template -inline T read_multi_wkb_internal(const byte_vector& buffer, int& index, uint32_t expected_type) { - const byte_t new_order = buffer[index]; - index++; - - uint32_t type; - utils::copy_from(buffer, index, type, new_order); - - if (type != expected_type) { - throw std::runtime_error("expected type " + std::to_string(expected_type) + ", but found type: " + std::to_string(type)); - } - - return read_wkb_internal(buffer, index, new_order); -}; - -template<> -inline READ_WKB_INTERNAL_SIG(geojson::multipoint_t) +inline typename wkb::multipoint_t wkb::read_multipoint(const byte_vector& buffer, int& index, uint8_t order) { uint32_t count; utils::copy_from(buffer, index, count, order); - geojson::multipoint_t mp; + multipoint_t mp; mp.resize(count); for (auto& point : mp) { - point = read_multi_wkb_internal(buffer, index, 1); + const byte_t new_order = buffer[index]; + index++; + + uint32_t type; + utils::copy_from(buffer, index, type, new_order); + + point = read_point(buffer, index, new_order); } return mp; } -template<> -inline READ_WKB_INTERNAL_SIG(geojson::multilinestring_t) +inline typename wkb::multilinestring_t wkb::read_multilinestring(const byte_vector& buffer, int& index, uint8_t order) { uint32_t count; utils::copy_from(buffer, index, count, order); - geojson::multilinestring_t ml; + multilinestring_t ml; ml.resize(count); for (auto& line : ml) { - line = read_multi_wkb_internal(buffer, index, 2); + const byte_t new_order = buffer[index]; + index++; + + uint32_t type; + utils::copy_from(buffer, index, type, new_order); + + line = read_linestring(buffer, index, new_order); } return ml; } -template<> -inline READ_WKB_INTERNAL_SIG(geojson::multipolygon_t) +inline typename wkb::multipolygon_t wkb::read_multipolygon(const byte_vector& buffer, int& index, uint8_t order) { uint32_t count; utils::copy_from(buffer, index, count, order); - geojson::multipolygon_t mpl; + multipolygon_t mpl; mpl.resize(count); for (auto& polygon : mpl) { - polygon = read_multi_wkb_internal(buffer, index, 3); + const byte_t new_order = buffer[index]; + index++; + + uint32_t type; + utils::copy_from(buffer, index, type, new_order); + + polygon = read_polygon(buffer, index, new_order); } return mpl; } -#undef READ_WKB_INTERNAL_SIG - -} // anonymouse namespace /** - * @brief Read (known) WKB into a specific geometry struct + * @brief Read WKB into a variant geometry struct * - * @tparam T geometry struct type (i.e. @code{wkb::point}, @code{wkb::polygon}, etc.) - * @param buffer vector of bytes - * @return T geometry struct containing the parsed WKB values. + * @tparam CoordinateSystem boost coordinate system (i.e. boost::geometry::cs::cartesian) + * @param buffer buffer vector of bytes + * @return g_geometry_t Variant geometry struct */ -template -static inline T read_known_wkb(const byte_vector& buffer) +inline typename wkb::geometry wkb::read(const byte_vector& buffer) { if (buffer.size() < 5) { - - throw std::runtime_error( - "buffer reached end before encountering WKB\n\tdebug: [" + - std::string(buffer.begin(), buffer.end()) + "]" - ); + throw std::runtime_error("buffer reached end before encountering WKB"); } int index = 0; @@ -170,42 +178,107 @@ static inline T read_known_wkb(const byte_vector& buffer) uint32_t type; utils::copy_from(buffer, index, type, order); - return read_wkb_internal(buffer, index, order); -}; + geometry g = point_t{std::nan("0"), std::nan("0")}; + switch(type) { + case 1: g = read_point(buffer, index, order); break; + case 2: g = read_linestring(buffer, index, order); break; + case 3: g = read_polygon(buffer, index, order); break; + case 4: g = read_multipoint(buffer, index, order); break; + case 5: g = read_multilinestring(buffer, index, order); break; + case 6: g = read_multipolygon(buffer, index, order); break; + } + return g; +} -/** - * @brief Read WKB into a variant geometry struct - * - * @param buffer vector of bytes - * @return wkb::geometry geometry struct containing the parsed WKB values. - */ -static inline geojson::geometry read_wkb(const byte_vector&buffer) +using namespace bg::srs; +const auto epsg5070 = bg::srs::dpar::parameters<>( + dpar::proj_aea +)( + bg::srs::dpar::ellps_grs80 +)( + bg::srs::dpar::towgs84, {0,0,0,0,0,0,0} +)( + bg::srs::dpar::lat_0, 23 +)( + bg::srs::dpar::lon_0, -96 +)( + bg::srs::dpar::lat_1, 29.5 +)( + bg::srs::dpar::lat_2, 45.5 +)( + bg::srs::dpar::x_0, 0 +)( + bg::srs::dpar::y_0, 0 +); + +struct wkb::wgs84 : public boost::static_visitor { - if (buffer.size() < 5) { - throw std::runtime_error("buffer reached end before encountering WKB"); + wgs84(uint32_t srs) : srs(srs) + { + if (srs == 5070) { + this->tr = std::make_unique>( + bg::srs::transformation{ + epsg5070, + bg::srs::dpar::parameters<>(bg::srs::dpar::proj_longlat)(bg::srs::dpar::datum_wgs84)(bg::srs::dpar::no_defs) + } + ); + } else { + this->tr = std::make_unique>( + bg::srs::transformation{ + bg::srs::epsg(srs), + bg::srs::epsg(4326) + } + ); + } + }; + + geojson::geometry operator()(point_t& g) + { + geojson::coordinate_t h; + this->tr->forward(g, h); + return h; } - int index = 0; - const byte_t order = buffer[index]; - index++; + geojson::geometry operator()(linestring_t& g) + { + geojson::linestring_t h; + this->tr->forward(g, h); + return h; + } - uint32_t type; - utils::copy_from(buffer, index, type, order); + geojson::geometry operator()(polygon_t& g) + { + geojson::polygon_t h; + this->tr->forward(g, h); + return h; + } - geojson::geometry g = geojson::coordinate_t{std::nan("0"), std::nan("0")}; - switch(type) { - case 1: g = read_wkb_internal(buffer, index, order); break; - case 2: g = read_wkb_internal(buffer, index, order); break; - case 3: g = read_wkb_internal(buffer, index, order); break; - case 4: g = read_wkb_internal(buffer, index, order); break; - case 5: g = read_wkb_internal(buffer, index, order); break; - case 6: g = read_wkb_internal(buffer, index, order); break; + geojson::geometry operator()(multipoint_t& g) + { + geojson::multipoint_t h; + this->tr->forward(g, h); + return h; } - return g; -} + geojson::geometry operator()(multilinestring_t& g) + { + geojson::multilinestring_t h; + this->tr->forward(g, h); + return h; + } + + geojson::geometry operator()(multipolygon_t& g) + { + geojson::multipolygon_t h; + this->tr->forward(g, h); + return h; + } + + private: + uint32_t srs; + std::unique_ptr> tr; +}; -} // namespace wkb } // namespace geopackage -#endif // NGEN_GEOPACKAGE_WKB_READER_H +#endif // NGEN_GEOPACKAGE_WKB_H diff --git a/src/geopackage/CMakeLists.txt b/src/geopackage/CMakeLists.txt index 56b5f060f2..3f146358f8 100644 --- a/src/geopackage/CMakeLists.txt +++ b/src/geopackage/CMakeLists.txt @@ -9,4 +9,4 @@ add_library(geopackage STATIC geometry.cpp add_library(NGen::geopackage ALIAS geopackage) target_include_directories(geopackage PUBLIC ${PROJECT_SOURCE_DIR}/include/geopackage) target_include_directories(geopackage PRIVATE ${PROJECT_SOURCE_DIR}/include/utilities) -target_link_libraries(geopackage PRIVATE NGen::geojson Boost::boost libsqlite3) +target_link_libraries(geopackage PUBLIC NGen::geojson Boost::boost libsqlite3) diff --git a/src/geopackage/geometry.cpp b/src/geopackage/geometry.cpp index 325da4d562..e867c4b939 100644 --- a/src/geopackage/geometry.cpp +++ b/src/geopackage/geometry.cpp @@ -1,12 +1,13 @@ #include "GeoPackage.hpp" +#include #include #include +#include +#include "EndianCopy.hpp" #include "WKB.hpp" -namespace bsrs = boost::geometry::srs; - geojson::geometry geopackage::build_geometry( const sqlite_iter& row, const geojson::FeatureType geom_type, @@ -14,19 +15,16 @@ geojson::geometry geopackage::build_geometry( ) { const std::vector geometry_blob = row.get>(geom_col); - int index = 0; - if (geometry_blob[0] != 'G' && geometry_blob[1] != 'P') { + if (geometry_blob[0] != 'G' || geometry_blob[1] != 'P') { throw std::runtime_error("expected geopackage WKB, but found invalid format instead"); } - index += 2; - - // skip version - index++; + + int index = 3; // skip version // flags const bool is_extended = geometry_blob[index] & 0x00100000; const bool is_empty = geometry_blob[index] & 0x00010000; - const uint8_t indicator = (geometry_blob[index] & 0x00001110) >> 1; + const uint8_t indicator = (geometry_blob[index] >> 1) & 0x00000111; const uint8_t endian = geometry_blob[index] & 0x00000001; index++; @@ -52,22 +50,13 @@ geojson::geometry geopackage::build_geometry( } } - const std::vector geometry_data(geometry_blob.begin() + index, geometry_blob.end()); - geojson::geometry geometry = wkb::read_wkb(geometry_data); - - if (srs_id != 4326) { + if (!is_empty) { + const std::vector geometry_data(geometry_blob.begin() + index, geometry_blob.end()); + auto wkb_geometry = wkb::read(geometry_data); + wkb::wgs84 pvisitor{srs_id}; + geojson::geometry geometry = boost::apply_visitor(pvisitor, wkb_geometry); return geometry; } else { - geojson::geometry projected; - - // project coordinates from whatever they are to 4326 - boost::geometry::srs::transformation<> tr{ - bsrs::epsg(srs_id), - bsrs::epsg(4326) - }; - - tr.forward(geometry, projected); - - return projected; + return geojson::geometry{}; } } \ No newline at end of file diff --git a/src/geopackage/properties.cpp b/src/geopackage/properties.cpp index 4ab43ec681..da2ac8bf46 100644 --- a/src/geopackage/properties.cpp +++ b/src/geopackage/properties.cpp @@ -1,4 +1,21 @@ #include "GeoPackage.hpp" +#include "JSONProperty.hpp" + +geojson::JSONProperty get_property(const geopackage::sqlite_iter& row, std::string name, int type) +{ + if (type == SQLITE_INTEGER) { + auto val = row.get(name); + return geojson::JSONProperty(name, val); + } else if (type == SQLITE_FLOAT) { + auto val = row.get(name); + return geojson::JSONProperty(name, val); + } else if (type == SQLITE_TEXT) { + auto val = row.get(name); + return geojson::JSONProperty(name, val); + } else { + return geojson::JSONProperty(name, "null"); + } +} geojson::PropertyMap geopackage::build_properties( const sqlite_iter& row, @@ -14,36 +31,21 @@ geojson::PropertyMap geopackage::build_properties( data_cols.begin(), data_cols.end(), data_types.begin(), - std::inserter(property_types, property_types.end()), [](const std::string& name, int type) { + std::inserter(property_types, property_types.end()), [](std::string name, int type) { return std::make_pair(name, type); } ); for (auto& col : property_types) { - const auto name = col.first; + auto name = col.first; const auto type = col.second; if (name == geom_col) { continue; } - geojson::JSONProperty* property = nullptr; - switch(type) { - case SQLITE_INTEGER: - *property = geojson::JSONProperty(name, row.get(name)); - break; - case SQLITE_FLOAT: - *property = geojson::JSONProperty(name, row.get(name)); - break; - case SQLITE_TEXT: - *property = geojson::JSONProperty(name, row.get(name)); - break; - default: - *property = geojson::JSONProperty(name, "null"); - break; - } - - properties.emplace(col, std::move(*property)); + geojson::JSONProperty property = get_property(row, std::move(name), type); + properties.emplace(name, std::move(property)); } return properties; -} \ No newline at end of file +} diff --git a/src/geopackage/read.cpp b/src/geopackage/read.cpp index 441f2220fc..c621b3985c 100644 --- a/src/geopackage/read.cpp +++ b/src/geopackage/read.cpp @@ -7,9 +7,10 @@ std::shared_ptr geopackage::read( ) { sqlite db(gpkg_path); + // Check if layer exists if (!db.has_table(layer)) { - throw std::runtime_error(""); + throw std::runtime_error("table " + layer + " does not exist"); } // Layer exists, getting statement for it @@ -42,34 +43,27 @@ std::shared_ptr geopackage::read( // Get layer feature metadata (geometry column name + type) sqlite_iter query_get_layer_geom_meta = db.query("SELECT column_name, geometry_type_name FROM gpkg_geometry_columns WHERE table_name = ?", layer); + query_get_layer_geom_meta.next(); const std::string layer_geometry_column = query_get_layer_geom_meta.get(0); const std::string layer_geometry_type = query_get_layer_geom_meta.get(1); - sqlite_iter query_get_layer_data_meta = db.query("SELECT column_name FROM gpkg_data_columns WHERE table_name = ?", layer); - std::vector layer_data_columns; - query_get_layer_data_meta.next(); - while (!query_get_layer_data_meta.done()) { - layer_data_columns.push_back(query_get_layer_data_meta.get(0)); - query_get_layer_data_meta.next(); - } - // Get layer - sqlite_iter query_get_layer = db.query("SELECT * FROM " + layer + joined_ids + " ORDER BY id"); + sqlite_iter query_get_layer = db.query("SELECT * FROM " + layer); + query_get_layer.next(); + std::vector features; features.reserve(layer_feature_count); - query_get_layer.next(); while(!query_get_layer.done()) { - features.push_back(build_feature( + geojson::Feature feature = build_feature( query_get_layer, layer_geometry_type, layer_geometry_column - )); + ); + + features.emplace_back(feature); + query_get_layer.next(); } - const auto fc = geojson::FeatureCollection( - std::move(features), - {min_x, min_y, max_x, max_y} - ); - + const auto fc = geojson::FeatureCollection(std::move(features), {min_x, min_y, max_x, max_y}); return std::make_shared(fc); } \ No newline at end of file diff --git a/src/geopackage/sqlite/database.cpp b/src/geopackage/sqlite/database.cpp index 50868ad0bf..6131c36010 100644 --- a/src/geopackage/sqlite/database.cpp +++ b/src/geopackage/sqlite/database.cpp @@ -46,6 +46,7 @@ sqlite3* sqlite::connection() const noexcept bool sqlite::has_table(const std::string& table) noexcept { auto q = this->query("SELECT 1 from sqlite_master WHERE type='table' AND name = ?", table); + q.next(); if (q.done()) { return false; @@ -63,7 +64,7 @@ sqlite_iter sqlite::query(const std::string& statement) if (code != SQLITE_OK) { // something happened, can probably switch on result codes // https://www.sqlite.org/rescode.html - throw sqlite_error("sqlite3_prepare_v2", code); + throw sqlite_error("[with statement: " + statement + "] " + "sqlite3_prepare_v2", code); } this->stmt = stmt_t(stmt, sqlite_deleter{}); diff --git a/src/geopackage/sqlite/iterator.cpp b/src/geopackage/sqlite/iterator.cpp index dcc7f5ba96..30bd1419ee 100644 --- a/src/geopackage/sqlite/iterator.cpp +++ b/src/geopackage/sqlite/iterator.cpp @@ -1,3 +1,5 @@ +#include + #include "SQLite.hpp" using namespace geopackage; @@ -92,7 +94,10 @@ template<> std::vector sqlite_iter::get>(int col) const { this->handle_get_index(col); - return *reinterpret_cast*>(sqlite3_column_blob(this->ptr(), col)); + int size = sqlite3_column_bytes(this->ptr(), col); + const uint8_t* ptr = static_cast(sqlite3_column_blob(this->ptr(), col)); + std::vector blob(ptr, ptr+size); + return blob; } template<> @@ -114,5 +119,7 @@ std::string sqlite_iter::get(int col) const { this->handle_get_index(col); // TODO: this won't work with non-ASCII text - return std::string(reinterpret_cast(sqlite3_column_text(this->ptr(), col))); + int size = sqlite3_column_bytes(this->ptr(), col); + const unsigned char* ptr = sqlite3_column_text(this->ptr(), col); + return std::string(ptr, ptr+size); } \ No newline at end of file diff --git a/test/geopackage/GeoPackage_Test.cpp b/test/geopackage/GeoPackage_Test.cpp index fd4c24d171..f96ff6813e 100644 --- a/test/geopackage/GeoPackage_Test.cpp +++ b/test/geopackage/GeoPackage_Test.cpp @@ -1,3 +1,4 @@ +#include #include #include "GeoPackage.hpp" @@ -26,5 +27,5 @@ class GeoPackage_Test : public ::testing::Test TEST_F(GeoPackage_Test, geopackage_read_test) { - const auto gpkg = geopackage::read(this->path, "flowpaths"); + const auto gpkg = geopackage::read(this->path, "flowpaths", {}); } \ No newline at end of file diff --git a/test/geopackage/WKB_Test.cpp b/test/geopackage/WKB_Test.cpp index 617645504e..b03b5cf370 100644 --- a/test/geopackage/WKB_Test.cpp +++ b/test/geopackage/WKB_Test.cpp @@ -1,6 +1,7 @@ +#include #include -#include +#include using namespace geopackage; @@ -12,14 +13,14 @@ class WKB_Test : public ::testing::Test ~WKB_Test() override {} void SetUp() override { - this->wkb["point"] = { + this->wkb_bytes["point"] = { 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x40 }; - this->wkb["point_big"] = { + this->wkb_bytes["point_big"] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, @@ -27,7 +28,7 @@ class WKB_Test : public ::testing::Test }; // LINESTRING(30 10, 10 30, 40 40) - this->wkb["linestring"] = { + this->wkb_bytes["linestring"] = { 0x01, 0x02, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, @@ -40,7 +41,7 @@ class WKB_Test : public ::testing::Test }; // POLYGON((10.689 -25.092, 34.595 -20.170, 38.814 -35.639, 13.502 -39.155, 10.689 -25.092)) - this->wkb["polygon"] = { + this->wkb_bytes["polygon"] = { 0x01, 0x03, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, @@ -58,7 +59,7 @@ class WKB_Test : public ::testing::Test }; // MULTIPOINT(10 40,40 30,20 20,30 10) - this->wkb["multipoint"] = { + this->wkb_bytes["multipoint"] = { 0x01, 0x04, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, @@ -81,7 +82,7 @@ class WKB_Test : public ::testing::Test }; // MULTILINESTRING((10 10,20 20,10 40),(40 40,30 30,40 20,30 10)) - this->wkb["multilinestring"] = { + this->wkb_bytes["multilinestring"] = { 0x01, 0x05, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, @@ -110,13 +111,13 @@ class WKB_Test : public ::testing::Test void TearDown() override {} - std::map> wkb; + std::map> wkb_bytes; }; TEST_F(WKB_Test, wkb_point_test) // also tests endianness -{ - const auto geom_big = wkb::read_known_wkb(this->wkb["point_big"]); - const auto geom_little = wkb::read_known_wkb(this->wkb["point"]); +{ + const auto geom_big = boost::get(wkb::read(this->wkb_bytes["point_big"])); + const auto geom_little = boost::get(wkb::read(this->wkb_bytes["point"])); EXPECT_NEAR(geom_big.get<0>(), geom_little.get<0>(), 0.000001); EXPECT_NEAR(geom_big.get<1>(), geom_little.get<1>(), 0.000001); EXPECT_NEAR(geom_big.get<0>(), 2.0, 0.000001); @@ -127,10 +128,10 @@ TEST_F(WKB_Test, wkb_point_test) // also tests endianness TEST_F(WKB_Test, wkb_linestring_test) { - const geojson::geometry geom = wkb::read_wkb(this->wkb["linestring"]); + const wkb::geometry geom = wkb::read(this->wkb_bytes["linestring"]); EXPECT_EQ(geom.which() + 1, 2); - const geojson::linestring_t& line = boost::get(geom); + const wkb::linestring_t& line = boost::get(geom); const std::vector> expected_coordinates = { {30, 10}, {10, 30}, {40, 40} }; @@ -143,10 +144,10 @@ TEST_F(WKB_Test, wkb_linestring_test) TEST_F(WKB_Test, wkb_polygon_test) { - const geojson::geometry geom = wkb::read_wkb(this->wkb["polygon"]); + const wkb::geometry geom = wkb::read(this->wkb_bytes["polygon"]); EXPECT_EQ(geom.which() + 1, 3); // +1 since variant.which() is 0-based - const geojson::polygon_t& poly = boost::get(geom); + const wkb::polygon_t& poly = boost::get(geom); const std::vector> expected_coordinates = { {10.689, -25.092}, {34.595, -20.170}, @@ -163,10 +164,10 @@ TEST_F(WKB_Test, wkb_polygon_test) TEST_F(WKB_Test, wkb_multipoint_test) { - const geojson::geometry geom = wkb::read_wkb(this->wkb["multipoint"]); + const wkb::geometry geom = wkb::read(this->wkb_bytes["multipoint"]); EXPECT_EQ(geom.which() + 1, 4); - const geojson::multipoint_t& mp = boost::get(geom); + const wkb::multipoint_t& mp = boost::get(geom); const std::vector> expected_coordinates = { {10, 40}, {40, 30}, {20, 20}, {30, 10} }; @@ -179,10 +180,10 @@ TEST_F(WKB_Test, wkb_multipoint_test) TEST_F(WKB_Test, wkb_multilinestring_test) { - const geojson::geometry geom = wkb::read_wkb(this->wkb["multilinestring"]); + const wkb::geometry geom = wkb::read(this->wkb_bytes["multilinestring"]); EXPECT_EQ(geom.which() + 1, 5); - const geojson::multilinestring_t& mp = boost::get(geom); + const wkb::multilinestring_t& mp = boost::get(geom); const std::vector>> expected_coordinates = { { {10, 10}, {20, 20}, {10, 40} }, { {40, 40}, {30, 30}, {40, 20}, {30, 10} } From 8d3dc38f8662616c3b7bd45d4a618ac170e0310c Mon Sep 17 00:00:00 2001 From: program-- Date: Wed, 3 May 2023 10:31:57 -0700 Subject: [PATCH 20/62] fix projection/bounding box issues; added test gpkg file; started on gpkg tests --- include/geopackage/GeoPackage.hpp | 5 +- include/geopackage/WKB.hpp | 91 ++++++++++++++++++++++++++-- src/geopackage/feature.cpp | 34 ++++------- src/geopackage/geometry.cpp | 17 +++--- src/geopackage/read.cpp | 27 +++++---- test/data/geopackage/example.gpkg | Bin 0 -> 106496 bytes test/geopackage/GeoPackage_Test.cpp | 27 +++++++-- 7 files changed, 145 insertions(+), 56 deletions(-) create mode 100644 test/data/geopackage/example.gpkg diff --git a/include/geopackage/GeoPackage.hpp b/include/geopackage/GeoPackage.hpp index c72beb862d..07932c2327 100644 --- a/include/geopackage/GeoPackage.hpp +++ b/include/geopackage/GeoPackage.hpp @@ -8,8 +8,8 @@ namespace geopackage { geojson::geometry build_geometry( const sqlite_iter& row, - const geojson::FeatureType geom_type, - const std::string& geom_col + const std::string& geom_col, + std::vector& bounding_box ); geojson::PropertyMap build_properties( @@ -19,7 +19,6 @@ geojson::PropertyMap build_properties( geojson::Feature build_feature( const sqlite_iter& row, - const std::string& geom_type, const std::string& geom_col ); diff --git a/include/geopackage/WKB.hpp b/include/geopackage/WKB.hpp index d688e98870..27c4dbd8df 100644 --- a/include/geopackage/WKB.hpp +++ b/include/geopackage/WKB.hpp @@ -178,7 +178,7 @@ inline typename wkb::geometry wkb::read(const byte_vector& buffer) uint32_t type; utils::copy_from(buffer, index, type, order); - geometry g = point_t{std::nan("0"), std::nan("0")}; + geometry g; switch(type) { case 1: g = read_point(buffer, index, order); break; case 2: g = read_linestring(buffer, index, order); break; @@ -186,7 +186,9 @@ inline typename wkb::geometry wkb::read(const byte_vector& buffer) case 4: g = read_multipoint(buffer, index, order); break; case 5: g = read_multilinestring(buffer, index, order); break; case 6: g = read_multipolygon(buffer, index, order); break; + default: g = point_t{std::nan("0"), std::nan("0")}; break; } + return g; } @@ -234,6 +236,10 @@ struct wkb::wgs84 : public boost::static_visitor geojson::geometry operator()(point_t& g) { + if (this->srs == 4326) { + return geojson::coordinate_t(g.get<0>(), g.get<1>()); + } + geojson::coordinate_t h; this->tr->forward(g, h); return h; @@ -242,35 +248,108 @@ struct wkb::wgs84 : public boost::static_visitor geojson::geometry operator()(linestring_t& g) { geojson::linestring_t h; - this->tr->forward(g, h); + + if (this->srs == 4326) { + h.reserve(g.size()); + for (auto&& gg : g) { + h.emplace_back( + std::move(gg.get<0>()), + std::move(gg.get<1>()) + ); + } + } else { + this->tr->forward(g, h); + } return h; } geojson::geometry operator()(polygon_t& g) { geojson::polygon_t h; - this->tr->forward(g, h); + + if(this->srs == 4326) { + h.outer().reserve(g.outer().size()); + for (auto&& gg : g.outer()) { + h.outer().emplace_back( + std::move(gg.get<0>()), + std::move(gg.get<1>()) + ); + } + + h.inners().resize(g.inners().size()); + auto&& inner_g = g.inners().begin(); + auto&& inner_h = h.inners().begin(); + for (; inner_g != g.inners().end(); inner_g++, inner_h++) { + inner_h->reserve(inner_g->size()); + for (auto&& gg : *inner_g) { + inner_h->emplace_back( + std::move(gg.get<0>()), + std::move(gg.get<1>()) + ); + } + } + } else { + this->tr->forward(g, h); + } return h; } geojson::geometry operator()(multipoint_t& g) { geojson::multipoint_t h; - this->tr->forward(g, h); + + if (this->srs == 4326) { + h.reserve(g.size()); + for (auto&& gg : g) { + h.emplace_back( + std::move(gg.get<0>()), + std::move(gg.get<1>()) + ); + } + } else { + this->tr->forward(g, h); + } + return h; } geojson::geometry operator()(multilinestring_t& g) { geojson::multilinestring_t h; - this->tr->forward(g, h); + + if (this->srs == 4326) { + h.resize(g.size()); + auto&& line_g = g.begin(); + auto&& line_h = h.begin(); + for(; line_g != g.end(); line_g++, line_h++) { + *line_h = std::move( + boost::get(this->operator()(*line_g)) + ); + } + } else { + this->tr->forward(g, h); + } + return h; } geojson::geometry operator()(multipolygon_t& g) { geojson::multipolygon_t h; - this->tr->forward(g, h); + + if (this->srs == 4326) { + h.resize(g.size()); + auto&& polygon_g = g.begin(); + auto&& polygon_h = h.begin(); + for (; polygon_g != g.end(); polygon_g++, polygon_h++) { + *polygon_h = std::move( + boost::get(this->operator()(*polygon_g)) + ); + } + } else { + this->tr->forward(g, h); + } + return h; } diff --git a/src/geopackage/feature.cpp b/src/geopackage/feature.cpp index 996b757e8e..6edfcfe362 100644 --- a/src/geopackage/feature.cpp +++ b/src/geopackage/feature.cpp @@ -1,35 +1,25 @@ #include "GeoPackage.hpp" -const geojson::FeatureType feature_type_map(const std::string& g) -{ - if (g == "POINT") return geojson::FeatureType::Point; - if (g == "LINESTRING") return geojson::FeatureType::LineString; - if (g == "POLYGON") return geojson::FeatureType::Polygon; - if (g == "MULTIPOINT") return geojson::FeatureType::MultiPoint; - if (g == "MULTILINESTRING") return geojson::FeatureType::MultiLineString; - if (g == "MULTIPOLYGON") return geojson::FeatureType::MultiPolygon; - return geojson::FeatureType::GeometryCollection; -} - geojson::Feature geopackage::build_feature( const sqlite_iter& row, - const std::string& geom_type, const std::string& geom_col ) { - const auto type = feature_type_map(geom_type); + std::vector bounding_box(4); const auto id = row.get("id"); geojson::PropertyMap properties = std::move(build_properties(row, geom_col)); - geojson::geometry geometry = std::move(build_geometry(row, type, geom_col)); - const auto geometry_bbox = bg::return_envelope>(geometry); - const std::vector bounding_box = { - geometry_bbox.min_corner().get<0>(), - geometry_bbox.min_corner().get<1>(), - geometry_bbox.max_corner().get<0>(), - geometry_bbox.max_corner().get<1>() - }; + geojson::geometry geometry = std::move(build_geometry(row, geom_col, bounding_box)); - switch(type) { + const auto wkb_type = geojson::FeatureType(geometry.which() + 1); + if (wkb_type == geojson::FeatureType::Point) { + const auto& pt = boost::get(geometry); + bounding_box[0] = pt.get<0>(); + bounding_box[1] = pt.get<1>(); + bounding_box[2] = pt.get<0>(); + bounding_box[3] = pt.get<1>(); + } + + switch(wkb_type) { case geojson::FeatureType::Point: return std::make_shared(geojson::PointFeature( boost::get(geometry), diff --git a/src/geopackage/geometry.cpp b/src/geopackage/geometry.cpp index e867c4b939..52778ef323 100644 --- a/src/geopackage/geometry.cpp +++ b/src/geopackage/geometry.cpp @@ -10,8 +10,8 @@ geojson::geometry geopackage::build_geometry( const sqlite_iter& row, - const geojson::FeatureType geom_type, - const std::string& geom_col + const std::string& geom_col, + std::vector& bounding_box ) { const std::vector geometry_blob = row.get>(geom_col); @@ -32,15 +32,14 @@ geojson::geometry geopackage::build_geometry( uint32_t srs_id; utils::copy_from(geometry_blob, index, srs_id, endian); - std::vector envelope; // may be unused if (indicator > 0 & indicator < 5) { // not an empty envelope - - envelope.resize(4); // only 4, not supporting Z or M dims - utils::copy_from(geometry_blob, index, envelope[0], endian); - utils::copy_from(geometry_blob, index, envelope[1], endian); - utils::copy_from(geometry_blob, index, envelope[2], endian); - utils::copy_from(geometry_blob, index, envelope[3], endian); + bounding_box.clear(); + bounding_box.resize(4); // only 4, not supporting Z or M dims + utils::copy_from(geometry_blob, index, bounding_box[0], endian); // min_x + utils::copy_from(geometry_blob, index, bounding_box[2], endian); // max_x + utils::copy_from(geometry_blob, index, bounding_box[1], endian); // min_y + utils::copy_from(geometry_blob, index, bounding_box[3], endian); // max_y // ensure `index` is at beginning of data if (indicator == 2 || indicator == 3) { diff --git a/src/geopackage/read.cpp b/src/geopackage/read.cpp index c621b3985c..25903f61e8 100644 --- a/src/geopackage/read.cpp +++ b/src/geopackage/read.cpp @@ -27,14 +27,6 @@ std::shared_ptr geopackage::read( joined_ids = " WHERE id (" + joined_ids + ")"; } - - // Get layer bounding box - sqlite_iter query_get_layer_bbox = db.query("SELECT min_x, min_y, max_x, max_y FROM gpkg_contents WHERE table_name = ?", layer); - query_get_layer_bbox.next(); - const double min_x = query_get_layer_bbox.get(0); - const double min_y = query_get_layer_bbox.get(1); - const double max_x = query_get_layer_bbox.get(2); - const double max_y = query_get_layer_bbox.get(3); // Get number of features sqlite_iter query_get_layer_count = db.query("SELECT COUNT(*) FROM " + layer); @@ -42,10 +34,9 @@ std::shared_ptr geopackage::read( const int layer_feature_count = query_get_layer_count.get(0); // Get layer feature metadata (geometry column name + type) - sqlite_iter query_get_layer_geom_meta = db.query("SELECT column_name, geometry_type_name FROM gpkg_geometry_columns WHERE table_name = ?", layer); + sqlite_iter query_get_layer_geom_meta = db.query("SELECT column_name FROM gpkg_geometry_columns WHERE table_name = ?", layer); query_get_layer_geom_meta.next(); const std::string layer_geometry_column = query_get_layer_geom_meta.get(0); - const std::string layer_geometry_type = query_get_layer_geom_meta.get(1); // Get layer sqlite_iter query_get_layer = db.query("SELECT * FROM " + layer); @@ -56,14 +47,26 @@ std::shared_ptr geopackage::read( while(!query_get_layer.done()) { geojson::Feature feature = build_feature( query_get_layer, - layer_geometry_type, layer_geometry_column ); - + features.emplace_back(feature); query_get_layer.next(); } + // get layer bounding box + double min_x = std::numeric_limits::infinity(); + double min_y = std::numeric_limits::infinity(); + double max_x = -std::numeric_limits::infinity(); + double max_y = -std::numeric_limits::infinity(); + for (const auto& feature : features) { + const auto& bbox = feature->get_bounding_box(); + min_x = bbox[0] < min_x ? bbox[0] : min_x; + min_y = bbox[1] < min_y ? bbox[1] : min_y; + max_x = bbox[2] > max_x ? bbox[2] : max_x; + max_y = bbox[3] > max_y ? bbox[3] : max_y; + } + const auto fc = geojson::FeatureCollection(std::move(features), {min_x, min_y, max_x, max_y}); return std::make_shared(fc); } \ No newline at end of file diff --git a/test/data/geopackage/example.gpkg b/test/data/geopackage/example.gpkg new file mode 100644 index 0000000000000000000000000000000000000000..09a5e736e316508ac28d53181d3fa3129dc844d6 GIT binary patch literal 106496 zcmeI5TWlNIdBsTA)CI0Q=AadC5LBFN*}&77dW} zoEct*H-)mD_|pG?mzi_EbNhYYxy_L0q;7Ajrb6;%y_7M@0(X|=bk z{UG$@RQ-Z}^BfoWj9o{z-i|ojou{X{>HL2UQWux!OXmKX=a=@0QwZlE00JNY0w4ea zAOHd&00JNY0w4eaza;`^U!@UyJY)28VC=8Car!a7^IJ*_(SiU7fB*=900@8p2!H?x zfB*>eN8tB4_jS%Ye4&)qa!Ov+lpHB2<$|86?5J6iN~Yq8xLB?GHBOvD*>YLWsal3U z>(!OKqAOZfA;!L8Dy4hBt{?8JoR`1wpqa{AMmH5h&1m&Re%*+?aYC9x$0Ty}@?Yjh zE^%JhL-pYNok)sYS!CJEDW;kgw^EU_pGU+*WG!{iZ+rTM^{|wVS(ozM%9TaGklMN> zCKJ*1rpV=md7iRtMz1PL( z^1}RO=ChTI#zdN0L{}7TSIzGDh1o9RS5{`{n0PuKmG1d-N`Z<~$?4g-<;A6i`IY&l zxs{d0g>HFRnVX-t^6+ky7IHIVs%AAu1IH6d=@!kJ?OfIhHFq(blDg%1(av$PE;&=I zMMawQ4Weo!r)1Sqrbxe8E)7!Fh`FPX8hfL%qv&Nd$L9aM=bt(H!T|yx00JNY0w4ea zAOHd&00JNY0wC~g6IkX)z307S*ZE<7#6@q}n0X~*R&{zpU-Pr`3sbX8Q?m=w+|ugO z^6Kp3^yS&5yLt8O=j=!Ryyt&7`oaMMAOHd&00JNY0w4eaAOHd&00JQJf)Q{Jct_a1 z0nGnjFvX%cAOHd&00JNY0w4eaAOHd&00JQ3AW)zG+rR(+lEbQe4mdak0T2KI5C8!X z009sH0T2KI5CDM}g1|68;N{ule;1zrzYq#VAwU2GKmY_l00ck)1V8`;KmY_lpeupa z=l?(DJU{I!21gJ80T2KI5C8!X009sH0T2KI5CDPag}^!9?e(5_4G*(N{&d%c65VSd zCQ8Ze`ucyr;5@&0UJ8SJf&d7B00@8p2!H?xfB*=900@8p2=qbV%qg$E_8+?wfZzZ3 zK@1`w00JNY0w4eaAOHd&00JNY0wD055^(8U#K7i0w4eaAOHd&00JNY0w4eaAOHf-EdjRwALjqht@@C05C8!X z009sH0T2KI5C8!X009u_ivXVg_k{^UAOHd&00JNY0w4eaAOHd&00JQJToV`@|2gMz z{fry`-!uO?{-g14o#D^?*~rf54@V!k|7hgG;NK7Kp1R`t8UI&YtdC-j|J?QcKExmb z0#AX!;Ws_*iQ7}d4=f{X1g_kelL0A}Nye);bGX&XZ;;BoXh7BS5G;5yNY@NHTGU29HGJ zBqeT&YZ6I@qbV^MzL`i$0twi8BxQ}5nfpbB1Um8)2#^oea*>wAAlb5Jn0iLl%vEO= zvKdV)o8-Piii%+nb0?#b*+7Wgxh2wE2jk+MX-9^yk=f8yBF5JbuZ`O<2m64jYKEel z?XXS)(~hA5Ga65cN$D~1*>OG{d^g61e5om(Q?;D3XM9jpO-0UB&9ZeZx090R+Fizv z*3P(nQ&ar6S1jGP`?p_@)nSKkZi=McmjtayWGz!th$Ox%k$8f>)0>-uL-*0RBr>aS z_|k-uQ>lLt~?$P?WCmZTV3RzF2S%Df^vOV{^d#)wo zDT!90xI`)sWYa8>tzTju7Z>bhN|x3UF0=57_c5sA59QMw8AvW<_36pe4QO4O)zrA{Y@rfFSciLp`|PU~}K zTcF(&V_;>ZrOH)l=&aL9((&l+H0_z1m9RygR#7IewJP2r+*i$Px2CN8__h=s8S=fi ze5@@slv1+@dA_SnoQ@8kJSw{nX-{F~;FUqQZ)S!+xYsnyazUrsvfC>w15SsbPJ?uM zJM?l)!;@DsW>r^cDpjr7IjoBt;q<26^Mt}fL%#IPvACS+*6N+p$$j)Sm)jQz@SpBB z%hpn^`o%zJnd)9nLp!EGbhMa}SygJ45!4)Q^wqqo=(dM#y+TegvbtI^RXTXHI^25E z0|B9!F-$p2$6*E9ztJ(H6pb}|I8ql3(!pHbR7*-QFu6T7S(=*6Nt3r$Cu6IV`MUuj zpp|z6p^)Hg$pzwU!G)5e;ig`W2D`0a=&TGns}8K^c$C()(YUUh8uG;g$7;B)g<3_Z zUk(Gyw&#}g__3E)(=>=-{$7(G?g|9yZ7iamg>=>I@?5{ ztQRtx`VsAp>Xy-HWu0EM;*|9q?W($V-#I0(YN{1Qn>n`IU1)6h-?$O(eQx|8#_qcx z4h2vB$G|79MeZ~1eNSOb9{rP1@9FQIo*4P=iRWt;A!5=Uc>Qk(E4Hi%Q|G3P9gKILo4M5$_6Q@(;u+hN)?waxmN$yqbMmOyVd z=6=yvY_=19v3~U;L9^MS zwun8oCbcye-EO4Kwcbc;4RDRj)$XO;`ijlg*B{$QG}M;VpRpFTC2!=>QF7d=lMhu+ zJhnjq5C8!X=!?Li^pZUanfnc`yLB`CF*{CY9KHKZ`?mYj8sXL2dOH*SH}$rBv(sEZ zYh!(>FRYK(XWCewtx3p}!N~sq@ABN`JfG1I93TJ!AOHd&00JNY0w4eaAOHd&00Ji@ z@GaLMcc7h%Y;gmh*EiVvWcw{EmbdQO-;F50xWT*V1^aP=b<_px*Z))t$bE6cc4~{D z5#bj%QVRX4L+%^=DQ?3@6X#jtJYSEf5%EuMY^b_nviU#n`I4hA93TJ!AOHd&00JNY z0w4eaAOHd&00J)*fnk2Y%d_9zyV(4HVEi-A^B0~)`h^1oKmY_l00ck)1V8`;KmY_l z00f>Z0(S?7yxyq~F0x53`X+w!-b|bdjd1vR2$zT|-y7!0tsWPP&3i##;P=t{tQ3 zik7t(4XZn!D4lvB`mgNVl$D;$4+T1sYDxDo6JYEA4|u-Ed4A&g0sX=O0w4eaAOHd& z00JNY0w4eaAOHd&@H->$`XIe|VLhPlcJE@~_Xg?x2fpPt09(4Y%k2l(g+Y4%V5CK~ z<-+>?|M-7%^o0WiKmY_l00ck)1V8`;KmY_l00dq%0*6Dqo13_F{wTqo-O~+PYLC$i zxlED$3c#cf|7Z3}#cPrE@Mf%9G}ZNTiSC^ht3B<1^E4sX>K7zu`Ca$N!&3AeQKp+V zg#Ao{$*f1o17*L>?+=Op{{8Zvf0f)La{`$c$bvu?$$RAEYcn)NkFYUuJsOs!?Jd#& zW~Ndps#$BBfSC`q9QzT&r9UvrS|dC?Y2@|VOwVH2|Ns9n=lSuArec%_1V8`;KmY_l z00ck)1V8`;KmY_@Z~`HI*yrWA>myua-@WMtCX?b;aar*M?(p@(H zcd_;Veu)(TUvR5H@jw6sKmY_l00ck)1V8`;KmY{(-w_ya4R9j^JpHhq|Bp^_^o0Wi zKmY_l00ck)1V8`;K)^}hkGJTqTHpOTbz+-!v4g+)uN=qTY+J6{)?CcpWx4-5JEiWQ zSYP-60T2LzXNmxu|8w>237@H)Au}KV0w4eaAOHd&00JNY0w4eaAOHeSiU6Dczr_6u z=lR?5zZOQ;LeI$ay7WR&_ zf|}Vit%LLXmn`<9pagL7Sz*2}wUu03tb(RX~}mW)4WeRVB8%bivZ?Y-{y zUAi=U5bBnETf(&z--#xrba<0UUCH}NIu(sah!r-d<^)nwwLSWo*<+uowr`#9hmKQ;rSB1z1RE=c%SN_bQHUhIO9wf%fsSCMYYjdv4k$?d7h z($r*5n!L3-8C#vq-wg->t-Koug@jU0&Z|YbU0bdmdm}35XB())S6EBLQ&KWat(j!? zk}j9>s+@m7HWEoO8i_NjCc$Q;A=(LV(6M%WjrO!HjHq5hosm_#M%Kv1@zI42N5AIw z1p@q0x>KJmiFm!IzGQR1-AtPC)K|=-5!Ke6YmK2P83path = utils::FileChecker::find_first_readable({ - "test/data/routing/gauge_01073000.gpkg", - "../test/data/routing/gauge_01073000.gpkg", - "../../test/data/routing/gauge_01073000.gpkg" + "test/data/geopackage/example.gpkg", + "../test/data/geopackage/example.gpkg", + "../../test/data/geopackage/example.gpkg" }); if (this->path.empty()) { @@ -27,5 +27,24 @@ class GeoPackage_Test : public ::testing::Test TEST_F(GeoPackage_Test, geopackage_read_test) { - const auto gpkg = geopackage::read(this->path, "flowpaths", {}); + const auto gpkg = geopackage::read(this->path, "test", {}); + EXPECT_NE(gpkg->find("First"), -1); + EXPECT_NE(gpkg->find("Second"), -1); + const auto bbox = gpkg->get_bounding_box(); + EXPECT_EQ(bbox.size(), 4); + EXPECT_EQ(bbox[0], 102.0); + EXPECT_EQ(bbox[1], 0.0); + EXPECT_EQ(bbox[2], 105.0); + EXPECT_EQ(bbox[3], 1.0); + EXPECT_EQ(2, gpkg->get_size()); + + const auto& first = gpkg->get_feature(0); + const auto& third = gpkg->get_feature(2); + EXPECT_EQ(first->get_id(), "First"); + + const auto& point = boost::get(first->geometry()); + EXPECT_EQ(point.get<0>(), 102.0); + EXPECT_EQ(point.get<1>(), 0.5); + + ASSERT_TRUE(third == nullptr); } \ No newline at end of file From 432a5723ac3b169ec4233ffb0e322f41695b977d Mon Sep 17 00:00:00 2001 From: program-- Date: Wed, 3 May 2023 10:33:50 -0700 Subject: [PATCH 21/62] remove unnecessary headers --- include/geopackage/WKB.hpp | 2 -- src/geopackage/geometry.cpp | 5 ----- 2 files changed, 7 deletions(-) diff --git a/include/geopackage/WKB.hpp b/include/geopackage/WKB.hpp index 27c4dbd8df..c20b8576dd 100644 --- a/include/geopackage/WKB.hpp +++ b/include/geopackage/WKB.hpp @@ -4,8 +4,6 @@ #include "EndianCopy.hpp" #include "JSONGeometry.hpp" #include -#include -#include #include namespace bg = boost::geometry; diff --git a/src/geopackage/geometry.cpp b/src/geopackage/geometry.cpp index 52778ef323..7be3ec241c 100644 --- a/src/geopackage/geometry.cpp +++ b/src/geopackage/geometry.cpp @@ -1,10 +1,5 @@ #include "GeoPackage.hpp" -#include -#include -#include -#include - #include "EndianCopy.hpp" #include "WKB.hpp" From 1ebfc13e7277abd0cf8ea1f68904ea68a4e4c5b5 Mon Sep 17 00:00:00 2001 From: program-- Date: Wed, 3 May 2023 11:26:01 -0700 Subject: [PATCH 22/62] improve documentation for gpkg and wkb --- include/geopackage/GeoPackage.hpp | 30 +++++++++++++++++ include/geopackage/WKB.hpp | 53 +++++++++++++++---------------- 2 files changed, 55 insertions(+), 28 deletions(-) diff --git a/include/geopackage/GeoPackage.hpp b/include/geopackage/GeoPackage.hpp index 07932c2327..0b0cd9971d 100644 --- a/include/geopackage/GeoPackage.hpp +++ b/include/geopackage/GeoPackage.hpp @@ -6,22 +6,52 @@ namespace geopackage { +/** + * Build a geometry object from GeoPackage WKB. + * + * @param[in] row SQLite iterator at the row containing a geometry column + * @param[in] geom_col Name of geometry column containing GPKG WKB + * @param[out] bounding_box Bounding box of the geometry to output + * @return geojson::geometry GPKG WKB converted and projected to a boost geometry model + */ geojson::geometry build_geometry( const sqlite_iter& row, const std::string& geom_col, std::vector& bounding_box ); +/** + * Build properties from GeoPackage table columns. + * + * @param[in] row SQLite iterator at the row containing the data columns + * @param[in] geom_col Name of geometry column containing GPKG WKB to ignore + * @return geojson::PropertyMap PropertyMap of properties from the given row + */ geojson::PropertyMap build_properties( const sqlite_iter& row, const std::string& geom_col ); +/** + * Build a feature from a GPKG table row + * + * @param[in] row SQLite iterator at the row to build a feature from + * @param[in] geom_col Name of geometry column containing GPKG WKB + * @return geojson::Feature Feature containing geometry and properties from the given row + */ geojson::Feature build_feature( const sqlite_iter& row, const std::string& geom_col ); +/** + * Build a feature collection from a GPKG layer + * + * @param[in] gpkg_path Path to GPKG file + * @param[in] layer Layer name within GPKG file to create a collection from + * @param[in] ids optional subset of feature IDs to capture (if empty, the entire layer is converted) + * @return std::shared_ptr + */ std::shared_ptr read( const std::string& gpkg_path, const std::string& layer, diff --git a/include/geopackage/WKB.hpp b/include/geopackage/WKB.hpp index c20b8576dd..1020d50e2f 100644 --- a/include/geopackage/WKB.hpp +++ b/include/geopackage/WKB.hpp @@ -10,6 +10,9 @@ namespace bg = boost::geometry; namespace geopackage { +/** + * WKB reader struct + */ struct wkb { using point_t = bg::model::point; using linestring_t = bg::model::linestring; @@ -32,7 +35,13 @@ struct wkb { struct wgs84; wkb() = delete; - static geometry read(const byte_vector&); + + /** + * Read WKB from a given buffer + * @param[in] buffer byte vector buffer + * @return geometry wkb::geometry struct containing the geometry data from the buffer + */ + static geometry read(const byte_vector& buffer); private: static point_t read_point(const byte_vector&, int&, uint8_t); @@ -156,13 +165,6 @@ inline typename wkb::multipolygon_t wkb::read_multipolygon(const byte_vector& bu } -/** - * @brief Read WKB into a variant geometry struct - * - * @tparam CoordinateSystem boost coordinate system (i.e. boost::geometry::cs::cartesian) - * @param buffer buffer vector of bytes - * @return g_geometry_t Variant geometry struct - */ inline typename wkb::geometry wkb::read(const byte_vector& buffer) { if (buffer.size() < 5) { @@ -190,26 +192,21 @@ inline typename wkb::geometry wkb::read(const byte_vector& buffer) return g; } -using namespace bg::srs; -const auto epsg5070 = bg::srs::dpar::parameters<>( - dpar::proj_aea -)( - bg::srs::dpar::ellps_grs80 -)( - bg::srs::dpar::towgs84, {0,0,0,0,0,0,0} -)( - bg::srs::dpar::lat_0, 23 -)( - bg::srs::dpar::lon_0, -96 -)( - bg::srs::dpar::lat_1, 29.5 -)( - bg::srs::dpar::lat_2, 45.5 -)( - bg::srs::dpar::x_0, 0 -)( - bg::srs::dpar::y_0, 0 -); +/** + * EPSG 5070 projection definition for use with boost::geometry. + * + * @note this is required because boost 1.72.0 does not + * have an EPSG definition for 5070 in boost::srs::epsg. + */ +const auto epsg5070 = bg::srs::dpar::parameters<>(bg::srs::dpar::proj_aea) + (bg::srs::dpar::ellps_grs80) + (bg::srs::dpar::towgs84, {0,0,0,0,0,0,0}) + (bg::srs::dpar::lat_0, 23) + (bg::srs::dpar::lon_0, -96) + (bg::srs::dpar::lat_1, 29.5) + (bg::srs::dpar::lat_2, 45.5) + (bg::srs::dpar::x_0, 0) + (bg::srs::dpar::y_0, 0); struct wkb::wgs84 : public boost::static_visitor { From 609a0a9788ee748d5966f2ea5730b61a02b105fd Mon Sep 17 00:00:00 2001 From: program-- Date: Wed, 3 May 2023 14:28:11 -0700 Subject: [PATCH 23/62] add explicit double initialization for copy_from; add SQLite cmake module --- cmake/modules/FindSQLite3.cmake | 16 ++++++++++++++++ include/utilities/EndianCopy.hpp | 24 +++++++++++++++++++++--- src/geopackage/CMakeLists.txt | 2 +- src/geopackage/read.cpp | 2 ++ 4 files changed, 40 insertions(+), 4 deletions(-) create mode 100644 cmake/modules/FindSQLite3.cmake diff --git a/cmake/modules/FindSQLite3.cmake b/cmake/modules/FindSQLite3.cmake new file mode 100644 index 0000000000..ffed8ef65e --- /dev/null +++ b/cmake/modules/FindSQLite3.cmake @@ -0,0 +1,16 @@ +message("Looking for SQLite3...") +find_path(SQLITE3_INCLUDE NAMES sqlite3.h) +mark_as_advanced(SQLITE3_INCLUDE) +if(SQLITE3_INCLUDE) + include_directories(${SQLITE3_INCLUDE}) +endif() + +find_library(SQLITE3_LIBRARY NAMES sqlite3) +mark_as_advanced(SQLITE3_LIBRARY) +if(SQLITE3_LIBRARY) + add_library(sqlite3 SHARED IMPORTED) + set_property(TARGET sqlite3 PROPERTY IMPORTED_LOCATION "${SQLITE3_LIBRARY}") +endif() + +include(FindPackageHandleStandardArgs) +FIND_PACKAGE_HANDLE_STANDARD_ARGS(SQLite3 DEFAULT_MSG SQLITE3_LIBRARY SQLITE3_INCLUDE) \ No newline at end of file diff --git a/include/utilities/EndianCopy.hpp b/include/utilities/EndianCopy.hpp index 35e254e039..abcebe90d2 100644 --- a/include/utilities/EndianCopy.hpp +++ b/include/utilities/EndianCopy.hpp @@ -14,15 +14,14 @@ namespace utils { * and increments @param{index} by the number of bytes * used to store @tparam{S} * - * @tparam T an integral type * @tparam S a primitive type * @param src a vector of bytes * @param index an integral type tracking the starting position of @param{dst}'s memory * @param dst output primitive * @param order endianness value (0x01 == Little; 0x00 == Big) */ -template -void copy_from(std::vector src, T& index, S& dst, uint8_t order) +template +void copy_from(const std::vector& src, int& index, S& dst, uint8_t order) { std::memcpy(&dst, &src[index], sizeof(S)); @@ -35,6 +34,25 @@ void copy_from(std::vector src, T& index, S& dst, uint8_t order) index += sizeof(S); } +// boost::endian doesn't support using primitive doubles +// see: https://github.com/boostorg/endian/issues/36 +template<> +inline void copy_from(const std::vector& src, int& index, double& dst, uint8_t order) +{ + static_assert(sizeof(uint64_t) == sizeof(double), "sizeof(uint64_t) is not the same as sizeof(double)!"); + + uint64_t tmp; + + // copy into uint64_t + copy_from(src, index, tmp, order); + + // copy resolved endianness into double + std::memcpy(&dst, &tmp, sizeof(double)); + + // above call to copy_from handles index + // incrementing, so we don't need to. +} + } #endif // NGEN_ENDIANCOPY_H \ No newline at end of file diff --git a/src/geopackage/CMakeLists.txt b/src/geopackage/CMakeLists.txt index 3f146358f8..c06ddba4d4 100644 --- a/src/geopackage/CMakeLists.txt +++ b/src/geopackage/CMakeLists.txt @@ -9,4 +9,4 @@ add_library(geopackage STATIC geometry.cpp add_library(NGen::geopackage ALIAS geopackage) target_include_directories(geopackage PUBLIC ${PROJECT_SOURCE_DIR}/include/geopackage) target_include_directories(geopackage PRIVATE ${PROJECT_SOURCE_DIR}/include/utilities) -target_link_libraries(geopackage PUBLIC NGen::geojson Boost::boost libsqlite3) +target_link_libraries(geopackage PUBLIC NGen::geojson Boost::boost sqlite3) diff --git a/src/geopackage/read.cpp b/src/geopackage/read.cpp index 25903f61e8..535015fc2f 100644 --- a/src/geopackage/read.cpp +++ b/src/geopackage/read.cpp @@ -1,5 +1,7 @@ #include "GeoPackage.hpp" +#include + std::shared_ptr geopackage::read( const std::string& gpkg_path, const std::string& layer = "", From 4545dfe45803cae65b737a65e50a87fa618e43b3 Mon Sep 17 00:00:00 2001 From: program-- Date: Mon, 8 May 2023 13:31:15 -0700 Subject: [PATCH 24/62] cleanup formatting/tests --- src/geopackage/geometry.cpp | 1 - src/geopackage/properties.cpp | 3 ++- src/geopackage/sqlite/iterator.cpp | 14 ++++++++++++-- test/geopackage/GeoPackage_Test.cpp | 4 ++-- test/geopackage/SQLite_Test.cpp | 3 ++- 5 files changed, 18 insertions(+), 7 deletions(-) diff --git a/src/geopackage/geometry.cpp b/src/geopackage/geometry.cpp index 7be3ec241c..b21d8380ad 100644 --- a/src/geopackage/geometry.cpp +++ b/src/geopackage/geometry.cpp @@ -1,5 +1,4 @@ #include "GeoPackage.hpp" - #include "EndianCopy.hpp" #include "WKB.hpp" diff --git a/src/geopackage/properties.cpp b/src/geopackage/properties.cpp index da2ac8bf46..7fd6f7f6aa 100644 --- a/src/geopackage/properties.cpp +++ b/src/geopackage/properties.cpp @@ -27,6 +27,7 @@ geojson::PropertyMap geopackage::build_properties( std::map property_types; const auto data_cols = row.columns(); const auto data_types = row.types(); + std::transform( data_cols.begin(), data_cols.end(), @@ -43,7 +44,7 @@ geojson::PropertyMap geopackage::build_properties( continue; } - geojson::JSONProperty property = get_property(row, std::move(name), type); + geojson::JSONProperty property = get_property(row, name, type); properties.emplace(name, std::move(property)); } diff --git a/src/geopackage/sqlite/iterator.cpp b/src/geopackage/sqlite/iterator.cpp index 30bd1419ee..cb4d1dc309 100644 --- a/src/geopackage/sqlite/iterator.cpp +++ b/src/geopackage/sqlite/iterator.cpp @@ -7,6 +7,11 @@ using namespace geopackage; sqlite_iter::sqlite_iter(stmt_t stmt) : stmt(stmt) { + // sqlite3_column_type requires the last result code to be + // SQLITE_ROW, so we need to iterate on the first row. + // TODO: need to test how this functions if the last result code + // was SQLITE_DONE. + this->next(); this->column_count = sqlite3_column_count(this->ptr()); this->column_names = std::vector(); this->column_names.reserve(this->column_count); @@ -14,9 +19,11 @@ sqlite_iter::sqlite_iter(stmt_t stmt) this->column_types.reserve(this->column_count); for (int i = 0; i < this->column_count; i++) { - this->column_names.push_back(sqlite3_column_name(this->ptr(), i)); - this->column_types.push_back(sqlite3_column_type(this->ptr(), i)); + this->column_names.emplace_back(sqlite3_column_name(this->ptr(), i)); + this->column_types.emplace_back(sqlite3_column_type(this->ptr(), i)); } + + this->restart(); }; sqlite3_stmt* sqlite_iter::ptr() const noexcept @@ -35,6 +42,7 @@ void sqlite_iter::handle_get_index(int col) const } if (col < 0 || col >= this->column_count) { + throw std::out_of_range( "column " + std::to_string(col) + " out of range of " + std::to_string(this->column_count) + " columns" ); @@ -52,6 +60,8 @@ sqlite_iter& sqlite_iter::next() const int returncode = sqlite3_step(this->ptr()); if (returncode == SQLITE_DONE) { this->iteration_finished = true; + } else if (returncode != SQLITE_ROW) { + throw sqlite_error("sqlite3_step", returncode); } this->iteration_step++; } diff --git a/test/geopackage/GeoPackage_Test.cpp b/test/geopackage/GeoPackage_Test.cpp index e3483eba7e..ac11dc566f 100644 --- a/test/geopackage/GeoPackage_Test.cpp +++ b/test/geopackage/GeoPackage_Test.cpp @@ -16,7 +16,7 @@ class GeoPackage_Test : public ::testing::Test }); if (this->path.empty()) { - FAIL() << "can't find gauge_01073000.gpkg"; + FAIL() << "can't find test/data/geopackage/example.gpkg"; } } @@ -47,4 +47,4 @@ TEST_F(GeoPackage_Test, geopackage_read_test) EXPECT_EQ(point.get<1>(), 0.5); ASSERT_TRUE(third == nullptr); -} \ No newline at end of file +} diff --git a/test/geopackage/SQLite_Test.cpp b/test/geopackage/SQLite_Test.cpp index 7695b93695..c1980fe58c 100644 --- a/test/geopackage/SQLite_Test.cpp +++ b/test/geopackage/SQLite_Test.cpp @@ -95,12 +95,13 @@ TEST_F(SQLite_Test, sqlite_query_test) ASSERT_ANY_THROW(iter.get(0)); iter.next(); EXPECT_EQ(iter.current_row(), 0); + ASSERT_EQ(iter.get(0), "flowpaths"); // finishing iter.next(); EXPECT_TRUE(iter.done()); EXPECT_EQ(iter.current_row(), 1); - ASSERT_THROW((iter.get(0)), std::runtime_error); + ASSERT_ANY_THROW(iter.get(0)); // next should be idempotent when iteration is done ASSERT_NO_THROW(iter.next()); From e2878349a5593fa9576dcce350437c396588bf12 Mon Sep 17 00:00:00 2001 From: program-- Date: Mon, 8 May 2023 13:32:34 -0700 Subject: [PATCH 25/62] fix OOR error; better errors for geopackage::read --- include/geopackage/SQLite.hpp | 37 ++++++++----------- include/geopackage/WKB.hpp | 2 +- src/geopackage/read.cpp | 45 +++++++++++++++-------- src/geopackage/sqlite/database.cpp | 57 +++++++++++++++++++++++++----- 4 files changed, 95 insertions(+), 46 deletions(-) diff --git a/include/geopackage/SQLite.hpp b/include/geopackage/SQLite.hpp index 1d1a5a3be2..b0ed9a498c 100644 --- a/include/geopackage/SQLite.hpp +++ b/include/geopackage/SQLite.hpp @@ -43,9 +43,19 @@ using stmt_t = std::shared_ptr; * @param code sqlite3 result code * @return std::runtime_error */ -inline std::runtime_error sqlite_error(const std::string& f, int code) +inline std::runtime_error sqlite_error(const std::string& f, int code, const std::string& extra = "") { - std::string errmsg = f + " returned code " + std::to_string(code); + std::string errmsg = f + " returned code " + + std::to_string(code) + + " (msg: " + + std::string(sqlite3_errstr(code)) + + ")"; + + if (!extra.empty()) { + errmsg += " "; + errmsg += extra; + } + return std::runtime_error(errmsg); } @@ -159,6 +169,8 @@ class sqlite */ sqlite_iter query(const std::string& statement); + sqlite_iter query(const std::string& statement, const std::vector& binds); + /** * Query the SQLite Database with a bound statement and get the result * @param statement String query with parameters @@ -169,27 +181,6 @@ class sqlite sqlite_iter query(const std::string& statement, T const&... params); }; -template -inline sqlite_iter sqlite::query(const std::string& statement, T const&... params) -{ - sqlite3_stmt* stmt; - const auto cstmt = statement.c_str(); - const int code = sqlite3_prepare_v2(this->connection(), cstmt, statement.length() + 1, &stmt, NULL); - - if (code != SQLITE_OK) { - - throw sqlite_error("[with statement: " + std::string(sqlite3_errmsg(this->conn.get())) + "] " + "sqlite3_prepare_v2", code); - } - - std::vector binds{ { params... } }; - for (size_t i = 0; i < binds.size(); i++) { - sqlite3_bind_text(stmt, i + 1, binds[i].c_str(), -1, SQLITE_STATIC); - } - - this->stmt = stmt_t(stmt, sqlite_deleter{}); - return sqlite_iter(this->stmt); -} - } // namespace geopackage #endif // NGEN_GEOPACKAGE_SQLITE_H \ No newline at end of file diff --git a/include/geopackage/WKB.hpp b/include/geopackage/WKB.hpp index 1020d50e2f..2bf7ae71f2 100644 --- a/include/geopackage/WKB.hpp +++ b/include/geopackage/WKB.hpp @@ -83,7 +83,7 @@ inline typename wkb::polygon_t wkb::read_polygon(const byte_vector& buffer, int& polygon_t polygon; if (count > 1) { - polygon.inners().resize(count - 1); + polygon.inners().resize(count); } auto outer = read_linestring(buffer, index, order); diff --git a/src/geopackage/read.cpp b/src/geopackage/read.cpp index 535015fc2f..4242f75036 100644 --- a/src/geopackage/read.cpp +++ b/src/geopackage/read.cpp @@ -12,36 +12,53 @@ std::shared_ptr geopackage::read( // Check if layer exists if (!db.has_table(layer)) { - throw std::runtime_error("table " + layer + " does not exist"); + std::string errmsg = "[" + std::string(sqlite3_errmsg(db.connection())) + "] " + + "table " + layer + " does not exist.\n\tTables: "; + + auto errquery = db.query("SELECT name FROM sqlite_master WHERE type='table'").next(); + while(!errquery.done()) { + errmsg += errquery.get(0); + errmsg += ", "; + errquery.next(); + } + + throw std::runtime_error(errmsg); } // Layer exists, getting statement for it std::string joined_ids = ""; if (!ids.empty()) { - std::accumulate( - ids.begin(), - ids.end(), - joined_ids, - [](const std::string& origin, const std::string& append) { - return origin.empty() ? append : origin + "," + append; - } - ); - - joined_ids = " WHERE id (" + joined_ids + ")"; + joined_ids = " WHERE id IN (?"; + for (size_t i = 1; i < ids.size(); i++) { + joined_ids += ", ?"; + } + joined_ids += ")"; } // Get number of features - sqlite_iter query_get_layer_count = db.query("SELECT COUNT(*) FROM " + layer); + sqlite_iter query_get_layer_count = db.query("SELECT COUNT(*) FROM " + layer + joined_ids, ids); query_get_layer_count.next(); const int layer_feature_count = query_get_layer_count.get(0); + #ifndef NGEN_QUIET + std::cout << "Reading " << layer_feature_count << " features in layer " << layer; + if (!ids.empty()) { + std::cout << " (id subset:"; + for (auto& id : ids) { + std::cout << " " << id; + } + std::cout << ")"; + } + std::cout << std::endl; + #endif + // Get layer feature metadata (geometry column name + type) sqlite_iter query_get_layer_geom_meta = db.query("SELECT column_name FROM gpkg_geometry_columns WHERE table_name = ?", layer); query_get_layer_geom_meta.next(); const std::string layer_geometry_column = query_get_layer_geom_meta.get(0); // Get layer - sqlite_iter query_get_layer = db.query("SELECT * FROM " + layer); + sqlite_iter query_get_layer = db.query("SELECT * FROM " + layer + joined_ids, ids); query_get_layer.next(); std::vector features; @@ -52,7 +69,7 @@ std::shared_ptr geopackage::read( layer_geometry_column ); - features.emplace_back(feature); + features.push_back(feature); query_get_layer.next(); } diff --git a/src/geopackage/sqlite/database.cpp b/src/geopackage/sqlite/database.cpp index 6131c36010..933579bf51 100644 --- a/src/geopackage/sqlite/database.cpp +++ b/src/geopackage/sqlite/database.cpp @@ -45,14 +45,9 @@ sqlite3* sqlite::connection() const noexcept bool sqlite::has_table(const std::string& table) noexcept { - auto q = this->query("SELECT 1 from sqlite_master WHERE type='table' AND name = ?", table); - + auto q = this->query("SELECT EXISTS(SELECT 1 from sqlite_master WHERE type='table' AND name=?)", table); q.next(); - if (q.done()) { - return false; - } else { - return static_cast(q.get(0)); - } + return q.get(0); }; sqlite_iter sqlite::query(const std::string& statement) @@ -64,7 +59,53 @@ sqlite_iter sqlite::query(const std::string& statement) if (code != SQLITE_OK) { // something happened, can probably switch on result codes // https://www.sqlite.org/rescode.html - throw sqlite_error("[with statement: " + statement + "] " + "sqlite3_prepare_v2", code); + throw sqlite_error("sqlite3_prepare_v2", code, "(query: " + statement + ")"); + } + + this->stmt = stmt_t(stmt, sqlite_deleter{}); + return sqlite_iter(this->stmt); +} + +sqlite_iter sqlite::query(const std::string& statement, const std::vector& binds) +{ + sqlite3_stmt* stmt; + const auto cstmt = statement.c_str(); + const int code = sqlite3_prepare_v2(this->connection(), cstmt, statement.length() + 1, &stmt, NULL); + + if (code != SQLITE_OK) { + throw sqlite_error("sqlite3_prepare_v2", code, "(query: " + statement + ")"); + } + + if (!binds.empty()) { + for (size_t i = 0; i < binds.size(); i++) { + const int code = sqlite3_bind_text(stmt, i + 1, binds[i].c_str(), -1, SQLITE_TRANSIENT); + if (code != SQLITE_OK) { + throw sqlite_error("sqlite3_bind_text", code); + } + } + } + + this->stmt = stmt_t(stmt, sqlite_deleter{}); + return sqlite_iter(this->stmt); +} + +template +inline sqlite_iter sqlite::query(const std::string& statement, T const&... params) +{ + sqlite3_stmt* stmt; + const auto cstmt = statement.c_str(); + const int code = sqlite3_prepare_v2(this->connection(), cstmt, statement.length() + 1, &stmt, NULL); + + if (code != SQLITE_OK) { + throw sqlite_error("sqlite3_prepare_v2", code, "(query: " + statement + ")"); + } + + std::vector binds{ { params... } }; + for (size_t i = 0; i < binds.size(); i++) { + const int code = sqlite3_bind_text(stmt, i + 1, binds[i].c_str(), -1, SQLITE_TRANSIENT); + if (code != SQLITE_OK) { + throw sqlite_error("sqlite3_bind_text", code); + } } this->stmt = stmt_t(stmt, sqlite_deleter{}); From 88885a6cc502c8685dfa0352c1d322a77d5400d0 Mon Sep 17 00:00:00 2001 From: program-- Date: Mon, 8 May 2023 13:33:02 -0700 Subject: [PATCH 26/62] allow read from gpkg or geojson when running ngen --- src/NGen.cpp | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/NGen.cpp b/src/NGen.cpp index 5734820960..48da38a5fa 100644 --- a/src/NGen.cpp +++ b/src/NGen.cpp @@ -6,6 +6,7 @@ #include "realizations/catchment/Formulation_Manager.hpp" #include #include +#include #include "NGenConfig.h" #include "tshirt_params.h" @@ -259,11 +260,21 @@ int main(int argc, char *argv[]) { #endif // NGEN_MPI_ACTIVE // TODO: Instead of iterating through a collection of FeatureBase objects mapping to nexi, we instead want to iterate through HY_HydroLocation objects - geojson::GeoJSON nexus_collection = geojson::read(nexusDataFile, nexus_subset_ids); + geojson::GeoJSON nexus_collection; + if (boost::algorithm::ends_with(nexusDataFile, "gpkg")) { + nexus_collection = geopackage::read(nexusDataFile, "nexus", nexus_subset_ids); + } else { + nexus_collection = geojson::read(nexusDataFile, nexus_subset_ids); + } std::cout << "Building Catchment collection" << std::endl; // TODO: Instead of iterating through a collection of FeatureBase objects mapping to catchments, we instead want to iterate through HY_Catchment objects - geojson::GeoJSON catchment_collection = geojson::read(catchmentDataFile, catchment_subset_ids); + geojson::GeoJSON catchment_collection; + if (boost::algorithm::ends_with(catchmentDataFile, "gpkg")) { + catchment_collection = geopackage::read(catchmentDataFile, "divides", catchment_subset_ids); + } else { + catchment_collection = geojson::read(catchmentDataFile, catchment_subset_ids); + } for(auto& feature: *catchment_collection) { From 6abf62a6df59c6c04e5fddb4c2f425461e8797ea Mon Sep 17 00:00:00 2001 From: program-- Date: Tue, 9 May 2023 13:03:26 -0700 Subject: [PATCH 27/62] added more docs; working on issue w/ projections for Release build type --- include/geopackage/SQLite.hpp | 135 ++++++++++++++++++++----- include/geopackage/WKB.hpp | 99 +++++++++++++----- src/geopackage/geometry.cpp | 43 ++++++-- test/data/geopackage/example_3857.gpkg | Bin 0 -> 98304 bytes test/geopackage/GeoPackage_Test.cpp | 37 +++++++ 5 files changed, 257 insertions(+), 57 deletions(-) create mode 100644 test/data/geopackage/example_3857.gpkg diff --git a/include/geopackage/SQLite.hpp b/include/geopackage/SQLite.hpp index b0ed9a498c..5e163c3f28 100644 --- a/include/geopackage/SQLite.hpp +++ b/include/geopackage/SQLite.hpp @@ -4,21 +4,28 @@ #include #include #include +#include #include namespace geopackage { +/** + * Runtime error for iterations that haven't started + */ const auto sqlite_get_notstarted_error = std::runtime_error( "sqlite iteration is has not started, get() is not callable (call sqlite_iter::next() before)" ); +/** + * Runtime error for iterations that are finished + */ const auto sqlite_get_done_error = std::runtime_error( "sqlite iteration is done, get() is not callable" ); /** - * @brief Deleter used to provide smart pointer support for sqlite3 structs. + * Deleter used to provide smart pointer support for sqlite3 structs. */ struct sqlite_deleter { @@ -27,20 +34,21 @@ struct sqlite_deleter }; /** - * @brief Smart pointer (unique) type for sqlite3 database + * Smart pointer (unique) type for sqlite3 database */ using sqlite_t = std::unique_ptr; /** - * @brief Smart pointer (shared) type for sqlite3 prepared statements + * Smart pointer (shared) type for sqlite3 prepared statements */ using stmt_t = std::shared_ptr; /** - * @brief Get a runtime error based on a function and code. + * Get a runtime error based on a function and code. * * @param f String denoting the function where the error originated * @param code sqlite3 result code + * @param extra additional messages to add to the end of the error * @return std::runtime_error */ inline std::runtime_error sqlite_error(const std::string& f, int code, const std::string& extra = "") @@ -60,7 +68,7 @@ inline std::runtime_error sqlite_error(const std::string& f, int code, const std } /** - * @brief SQLite3 row iterator + * SQLite3 row iterator * * Provides a simple iterator-like implementation * over rows of a SQLite3 query. @@ -69,13 +77,14 @@ class sqlite_iter { private: stmt_t stmt; - int iteration_step = -1; - bool iteration_finished = false; + int iteration_step = -1; + bool iteration_finished = false; - // column metadata - int column_count; + int column_count; std::vector column_names; - std::vector column_types; + + // vector of SQLITE data types, see: https://www.sqlite.org/datatype3.html + std::vector column_types; // returns the raw pointer to the sqlite statement sqlite3_stmt* ptr() const noexcept; @@ -85,19 +94,90 @@ class sqlite_iter public: sqlite_iter(stmt_t stmt); - bool done() const noexcept; - sqlite_iter& next(); - sqlite_iter& restart(); - void close(); - int current_row() const noexcept; - int num_columns() const noexcept; - int column_index(const std::string& name) const noexcept; + + /** + * Check if a row iterator is finished + * + * @return true if next() returned SQLITE_DONE + * @return false if there is more rows available + */ + bool done() const noexcept; + + /** + * Step into the next row of a SQLite query + * + * If the query is finished, next() acts idempotently, + * but will change done() to return true. + * @return sqlite_iter& returns itself + */ + sqlite_iter& next(); + + /** + * Restart an iteration to its initial state. + * next() must be called after calling this. + * + * @return sqlite_iter& returns itself + */ + sqlite_iter& restart(); + + /** + * Call the row iterator destructor + */ + void close(); + + /** + * Get the current row index for the iterator + * + * @return int the current row index, or -1 if next() hasn't been called + */ + int current_row() const noexcept; + + /** + * Get the number of columns within this iterator + * @return int number of columns in query + */ + int num_columns() const noexcept; + + /** + * Return the column index for a named column + * + * @param name column name to search for + * @return int index of given column name, or -1 if not found. + */ + int column_index(const std::string& name) const noexcept; + + /** + * Get a vector of column names + * + * @return const std::vector& column names as a vector of strings + */ const std::vector& columns() const noexcept { return this->column_names; } - const std::vector& types() const noexcept { return this->column_types; } + /** + * Get a vector of column types + * + * See https://www.sqlite.org/datatype3.html for type values. The integers + * are the affinity for data types. + * @return const std::vector& column types as a vector of ints + */ + const std::vector& types() const noexcept { return this->column_types; } + + /** + * Get a column value from a row iterator by index + * + * @tparam T Type to parse value as, i.e. int + * @param col Column index to parse + * @return T value at column `col` + */ template T get(int col) const; + /** + * Get a column value from a row iterator by name + * + * @tparam T Type to parse value as, i.e. int + * @return T value at the named column + */ template T get(const std::string&) const; }; @@ -110,7 +190,7 @@ inline T sqlite_iter::get(const std::string& name) const } /** - * @brief Wrapper around a SQLite3 database + * Wrapper around a SQLite3 database */ class sqlite { @@ -122,7 +202,7 @@ class sqlite sqlite() = default; /** - * @brief Construct a new sqlite object from a path to database + * Construct a new sqlite object from a path to database * * @param path File path to sqlite3 database */ @@ -132,14 +212,14 @@ class sqlite sqlite& operator=(sqlite& db); /** - * @brief Take ownership of a sqlite3 database + * Take ownership of a sqlite3 database * * @param db sqlite3 database object */ sqlite(sqlite&& db); /** - * @brief Move assignment operator + * Move assignment operator * * @param db sqlite3 database object * @return sqlite& reference to sqlite3 database @@ -147,14 +227,14 @@ class sqlite sqlite& operator=(sqlite&& db); /** - * @brief Return the originating sqlite3 database pointer + * Return the originating sqlite3 database pointer * * @return sqlite3* */ sqlite3* connection() const noexcept; /** - * @brief Check if SQLite database contains a given table + * Check if SQLite database contains a given table * * @param table name of table * @return true if table does exist @@ -169,6 +249,13 @@ class sqlite */ sqlite_iter query(const std::string& statement); + /** + * Query the SQLite Database with multiple boundable text parameters. + * + * @param statement String query with parameters + * @param binds text parameters to bind to statement + * @return sqlite_iter SQLite row iterator + */ sqlite_iter query(const std::string& statement, const std::vector& binds); /** diff --git a/include/geopackage/WKB.hpp b/include/geopackage/WKB.hpp index 2bf7ae71f2..7e0bdc4b8a 100644 --- a/include/geopackage/WKB.hpp +++ b/include/geopackage/WKB.hpp @@ -11,7 +11,7 @@ namespace bg = boost::geometry; namespace geopackage { /** - * WKB reader struct + * A recursive WKB reader */ struct wkb { using point_t = bg::model::point; @@ -32,8 +32,13 @@ struct wkb { using byte_t = uint8_t; using byte_vector = std::vector; + /** + * projection visitor. applied with boost to project from + * cartesian coordinates to WGS84. + */ struct wgs84; + // prevent instatiation of this struct wkb() = delete; /** @@ -43,16 +48,46 @@ struct wkb { */ static geometry read(const byte_vector& buffer); + static bg::srs::dpar::parameters<> get_prj(uint32_t srid); + private: + /** + * Read a WKB point into a cartesian model. + * @return point_t + */ static point_t read_point(const byte_vector&, int&, uint8_t); + + /** + * Read a WKB linestring into a cartesian model. + * @return linestring_t + */ static linestring_t read_linestring(const byte_vector&, int&, uint8_t); + + /** + * Read a WKB polygon into a cartesian model. + * @return polygon_t + */ static polygon_t read_polygon(const byte_vector&, int&, uint8_t); + + /** + * Read a WKB multipoint into a cartesian model. + * @return multipoint_t + */ static multipoint_t read_multipoint(const byte_vector&, int&, uint8_t); + + /** + * Read a WKB multilinestring into a cartesian model. + * @return multilinestring_t + */ static multilinestring_t read_multilinestring(const byte_vector&, int&, uint8_t); + + /** + * Read a WKB multipolygon into a cartesian model. + * @return multipolygon_t + */ static multipolygon_t read_multipolygon(const byte_vector&, int&, uint8_t); }; - inline typename wkb::point_t wkb::read_point(const byte_vector& buffer, int& index, uint8_t order) { double x, y; @@ -83,6 +118,9 @@ inline typename wkb::polygon_t wkb::read_polygon(const byte_vector& buffer, int& polygon_t polygon; if (count > 1) { + // polygons only have 1 outer ring, + // so any extra vectors are considered to be + // inner rings. polygon.inners().resize(count); } @@ -208,26 +246,33 @@ const auto epsg5070 = bg::srs::dpar::parameters<>(bg::srs::dpar::proj_aea) (bg::srs::dpar::x_0, 0) (bg::srs::dpar::y_0, 0); +const auto epsg3857 = bg::srs::dpar::parameters<>(bg::srs::dpar::proj_merc) + (bg::srs::dpar::units_m) + (bg::srs::dpar::no_defs) + (bg::srs::dpar::a, 6378137) + (bg::srs::dpar::b, 6378137) + (bg::srs::dpar::lat_ts, 0) + (bg::srs::dpar::lon_0, 0) + (bg::srs::dpar::x_0, 0) + (bg::srs::dpar::y_0, 0) + (bg::srs::dpar::k, 1); + +inline bg::srs::dpar::parameters<> wkb::get_prj(uint32_t srid) { + switch(srid) { + case 5070: + return epsg5070; + case 3857: + return epsg3857; + default: + return bg::projections::detail::epsg_to_parameters(srid); + } +} + struct wkb::wgs84 : public boost::static_visitor { - wgs84(uint32_t srs) : srs(srs) - { - if (srs == 5070) { - this->tr = std::make_unique>( - bg::srs::transformation{ - epsg5070, - bg::srs::dpar::parameters<>(bg::srs::dpar::proj_longlat)(bg::srs::dpar::datum_wgs84)(bg::srs::dpar::no_defs) - } - ); - } else { - this->tr = std::make_unique>( - bg::srs::transformation{ - bg::srs::epsg(srs), - bg::srs::epsg(4326) - } - ); - } - }; + wgs84(uint32_t srs, const bg::srs::transformation<>& tr) + : srs(srs) + , tr(tr) {}; geojson::geometry operator()(point_t& g) { @@ -236,7 +281,7 @@ struct wkb::wgs84 : public boost::static_visitor } geojson::coordinate_t h; - this->tr->forward(g, h); + this->tr.forward(g, h); return h; } @@ -253,7 +298,7 @@ struct wkb::wgs84 : public boost::static_visitor ); } } else { - this->tr->forward(g, h); + this->tr.forward(g, h); } return h; } @@ -284,7 +329,7 @@ struct wkb::wgs84 : public boost::static_visitor } } } else { - this->tr->forward(g, h); + this->tr.forward(g, h); } return h; } @@ -302,7 +347,7 @@ struct wkb::wgs84 : public boost::static_visitor ); } } else { - this->tr->forward(g, h); + this->tr.forward(g, h); } return h; @@ -322,7 +367,7 @@ struct wkb::wgs84 : public boost::static_visitor ); } } else { - this->tr->forward(g, h); + this->tr.forward(g, h); } return h; @@ -342,7 +387,7 @@ struct wkb::wgs84 : public boost::static_visitor ); } } else { - this->tr->forward(g, h); + this->tr.forward(g, h); } return h; @@ -350,7 +395,7 @@ struct wkb::wgs84 : public boost::static_visitor private: uint32_t srs; - std::unique_ptr> tr; + const bg::srs::transformation<>& tr; }; } // namespace geopackage diff --git a/src/geopackage/geometry.cpp b/src/geopackage/geometry.cpp index b21d8380ad..7949242355 100644 --- a/src/geopackage/geometry.cpp +++ b/src/geopackage/geometry.cpp @@ -25,15 +25,46 @@ geojson::geometry geopackage::build_geometry( // Read srs_id uint32_t srs_id; utils::copy_from(geometry_blob, index, srs_id, endian); - + + const auto epsg = wkb::get_prj(srs_id); + const auto prj = bg::srs::transformation<>(epsg, wkb::get_prj(4326)); + wkb::wgs84 pvisitor{srs_id, prj}; + if (indicator > 0 & indicator < 5) { // not an empty envelope + + double min_x = 0, max_x = 0, min_y = 0, max_y = 0; + utils::copy_from(geometry_blob, index, min_x, endian); // min_x + utils::copy_from(geometry_blob, index, max_x, endian); // max_x + utils::copy_from(geometry_blob, index, min_y, endian); // min_y + utils::copy_from(geometry_blob, index, max_y, endian); // max_y + + // we need to transform the bounding box from its initial SRS + // to EPSG: 4326 -- so, we construct a temporary WKB linestring_t type + // which will get projected to a geojson::geometry (aka geojson::linestring_t) type. + + // create a wkb::linestring_t bbox object + wkb::point_t max{max_x, max_y}; + wkb::point_t min{min_x, min_y}; + geojson::coordinate_t max_prj{}; + geojson::coordinate_t min_prj{}; + + // project the raw bounding box + if (srs_id == 4326) { + max_prj = geojson::coordinate_t{max.get<0>(), max.get<1>()}; + min_prj = geojson::coordinate_t{min.get<0>(), min.get<1>()}; + } else { + prj.forward(max, max_prj); + prj.forward(min, min_prj); + } + + // assign the projected values to the bounding_box parameter bounding_box.clear(); bounding_box.resize(4); // only 4, not supporting Z or M dims - utils::copy_from(geometry_blob, index, bounding_box[0], endian); // min_x - utils::copy_from(geometry_blob, index, bounding_box[2], endian); // max_x - utils::copy_from(geometry_blob, index, bounding_box[1], endian); // min_y - utils::copy_from(geometry_blob, index, bounding_box[3], endian); // max_y + bounding_box[0] = min_prj.get<0>(); // min_x + bounding_box[1] = min_prj.get<1>(); // min_y + bounding_box[2] = max_prj.get<0>(); // max_x + bounding_box[3] = max_prj.get<1>(); // max_y // ensure `index` is at beginning of data if (indicator == 2 || indicator == 3) { @@ -46,9 +77,9 @@ geojson::geometry geopackage::build_geometry( if (!is_empty) { const std::vector geometry_data(geometry_blob.begin() + index, geometry_blob.end()); auto wkb_geometry = wkb::read(geometry_data); - wkb::wgs84 pvisitor{srs_id}; geojson::geometry geometry = boost::apply_visitor(pvisitor, wkb_geometry); return geometry; + return geojson::geometry{}; } else { return geojson::geometry{}; } diff --git a/test/data/geopackage/example_3857.gpkg b/test/data/geopackage/example_3857.gpkg new file mode 100644 index 0000000000000000000000000000000000000000..d4bd67e5e531b8b544af805b07d7d8241f389a84 GIT binary patch literal 98304 zcmeI*-)|eqVF&PCNfssB66M4;FLit#GhrxJlq~9pZP};Rw6vmmmPnag#@0E;VoeUI zt;t<^cNs~Cpr~wLiuR!|Y4T760RpsW(1&{{+N6DG|AiKLx(2yE^r0wf;Ls+;J@kO2 zvp+~Kf627$i!Jv{NFq5iJ3I54AG5d24=(=T#Lyyls_%~{x!Z{>xjaWMkadGr3)B4$8;XkpSNx!?4y&LS7 zDi}=&=|m$%FuEKUb8GenQf%S!TriqlzaeDO$>mxm!RW==OP6D_m!dNZm#1dtFU`!% z&R?93#&hD0bS5co+znd$4n~8Qre~*v_wIFwxi~v>(TZ8mB-aGCw}eiqx~=3lgVE^@ z$uBQV$E@VJR8qVfEYK25&QeZI$1csy&(18&%*PfM=4LyUVIej%)42?D^)jU26_&(g zI(0W#!;@m!^&l%=JQH6NL?LrGn5U|#%VnuTb)}%lD%*Wiypfm7R4U4OMjwpEn&OIb z*`N}Y4O3A|ts6R#s-~NpwOFT`6>93NmMp9Iog`~Qp|-sc&x*CcRDHWtIwRh=7ZvV^ zLMm%l#%g=ke!+xk)Kv0T z#~=Im|1U$z?v|^vshHIQ>#1%Jbb9Nmrk31rx4*Vc5AFbDUCv2)fptTB6VlK(h@z4L z%_|kT%wn(=LyrC61_1~_00Izz00bZa0SG_<0uX?} zs}i`!9}ArgjV$_xe7-)qBUiTAD`wW~LPaW@Rh=5%u$k$Z*~#ho$>{|#cJa#G?3KCs zsp*Be_u}+NSNDF&|82Z@bM@BPKm6+2XOlloPA$#(PaI|S;{E@{u^-$Z009U<00Izz z00bZa0SG_<0uXpZ1pNJ>VecyjSpUBv4U1|(00Izz00bZa0SG_<0uX=z1Uv$*>wo9- z|DSOFPdth65CRZ@00bZa0SG_<0uX=z1Rwx`Lo6`H_lJ1z+y8xd|Njs-CyEXM2tWV= z5P$##AOHafKmY;|=pxYh{r`_S|HoZ6KrjR#009U<00Izz00bZa0SG_<0qA{4IP<{+V%gQa{i~UcU4e22tWV=5P$##AOHafKmY;|fB*z~ z5;!>!I@NUTKkJnMeEz>DJ0yYt1Rwwb2tWV=5P$##AOHaf94diMpa18^4pkLUSO`D> z0uX=z1Rwwb2tWV=5P$##o>Rbj{~zoB=R`wt2tWV=5P$##AOHafKmY;|fWRRWu)hBf z>;FU6XecfOAOHafKmY;|fB*y_009U<;5h~G{{M4=Avpvf009U<00Izz00bZa0SG|g z5DJWp{(|%OJ>^FK_xQ)7KN1y|id3fGq-CkP zRgg{Enqny<#6^LKnPeg%WJtI*zCDL9xh|}vGXlx2FI$^xMbg*{NvGQL2osi`2=S#G zB$K|)Hcuo|BrB{6OCrg{lUZRrem$KLqa^HK~A+`MIolyt# zNDr8bYEa#5#d;o0Cx<#_GL;oF;tTRQdA=IH-yInYWG8t}Q44g(cvx0UD#=w-v!6?? zQc|onWc2aUabI9^lK=LC-S(~Vt5fw&9kt!^Y>1WGL@bvYgIY6{;7> ziQMfRxs*<2MK**|BH4N%nP!EoXOe62%m%qBYK40L{CH@c3HO8xT{bg$E zyHZ{&S1YPvH-Bs3PCFs^S-X~%*wP6jS`lze42YFhW zjWug$RxJ&0@ADQr$_`JKfnAWMAy;Kv=e@lomrCBsu_IGs5H_V}L)1#IHaEN{aZe+& zwKwIZ7w=2S;laSYOZ)bvdZV~KLYnE=CypkEo_VdWf#C3FxcK9z0sa_*=OP@QrWEPlKu6@@xj4B?!vxY-u=4c z&Ku-={N+AhAROjD*lyIVsa~}w{q1$CMR_~gJ{4kTi?U=5rRFqIrNAbArKnKdiEw&v zv_Or#u56hKJ9)DYxV5T>!_l&Am{Ojdhn3jzjh!=!$+gB2j?@y4vy-`^sZ{8AIJ7Yt zs!WCoV(7+|(At$y@x5>~tZLigNF?f=$)m(OgGVc#iJL`vChV5uLi=FQt>IuDISw+r z)+g76fx*C9c;6PTwSqekYEO?W^!L80)A4JY4zQKSkhS`F)X&?;-p7$f=V&<&*nPhx ztgpSRw4`v39v#l5y3DBFIWel~C0SM8XNRL&F`-Y7P3G`+wMRUHrC1zJ>8#m=Io zJGS<_$jHz?al@RzjsE?}eczM8@qwTB|3Tjz_Yo)fOCvuUxqke^kquvmAWM*Hnl~ z)v%X*C7sQ~Y-(y*^;uocUimd`^gH#nU-GnLd!{Gb1FrtO`<7m>^`EzBGg1A+cx7?)Hn}FL>be=F$DQ z>AdXzs?PA-x!x&6?^C^#zH!i89&=bv^u+pN$4rOy^;(3y97gN^|NH#!asH3kAKV}S z0SG_<0uX=z1Rwwb2tWV=5O_|3Z}lDF_S7#W*17%@evbPI|I59rKcaE|AIEO3-Ymvz zxm{hFO8#{6vuk`G3$Z?&P)k@#@Xz1<%ZK0lR~oM+{Ex^t|KI2Q-+v=2h)O^J0uX=z1Rwwb2tWV=5P$##4xPY#elT=u z^5J>!g5CO$cl)vgx~Wsj{)>-YKWfu7x2{~cu)V!K#iF<5`~$f}r`Y8JE*M)hADfxJ z@D;klE7lD43Jd`VKmY;|fB*y_009U<00Izzz&--~ef`{U zKhOTypZ`BL$*~{YAOHafKmY;|fB*y_00FPSAFZ>iYJKOcEXcZ8mv!TBPR4)Eem_oZ zly&xNI>B6@9X|73-g=(+7xhowaNI!v0uXp@1g!NxH}={T2L*uu1Rwwb2tWV=5P$## zAOHafK;Ux_u-5;6{#Ts;!_mJPt&aTL$h*gXeEjs$zaEYZ@q@yVw+5#B11uW1H&Woy z>9FsN6dHOw>wbx*dnXlWnVM88QrWEPRLW~r)s(#dOX7XJlo8^hK*UTkkq|N@*mEX9 z5?>M77R$n_z%pQ8NE-C!8zk3-L^4I(@BOwfd!dzdRvMLE&B8QDRuD;35y@i9^GOmT zw{Hj;ft0mvs*juU{W6tQxk4i>trHgZ?lbIuCHLvCb%t@i1M~%9o|#Dvrt$spFB@j! zfG~SB zByunwHVVhoT^Mg=lr3mrB;uqiO>=Lo&kp zYJ5o`$&{Ghr(=FSzM2!VZqr)5GRg|Krl@zkjTyB{ye9872RaEiIwfohbZ5Fy(CO&3 zr?#6pc-zgiw%rPGhi?1C-_qq@L62{0J~kmd9z1V5eGu-+DRywD*yrQ)e4YI@?yxek zk3dhPXEcO!ls!m0I9t}5p+?bpULIHJcH1GSw+W?|@#{A!?43R53nY{L-n@On)aCFg zZOJ-258FRBP&@3$udfPSCpZ~u2Nbq}+eaob4?91f470Ql0)my3T zw83%vuA`S#Y*mZwjeXe{xO#PHZ(+Z3bt#csl&>cp zR1|fG{grpDzlyqRKi-Wjo*4|>n&3J6>UGA$GP@X^Bv(z%elFSP`SlCd8B(kz;N;%+ zZ}|cf6Z~VR^;@D_o_?`Ovn7&v76e2T?ugO6R<2f5dxhWx)Cpath.empty()) { FAIL() << "can't find test/data/geopackage/example.gpkg"; } + + this->path2 = utils::FileChecker::find_first_readable({ + "test/data/geopackage/example_3857.gpkg", + "../test/data/geopackage/example_3857.gpkg", + "../../test/data/geopackage/example_3857.gpkg" + }); + + if (this->path2.empty()) { + FAIL() << "can't find test/data/geopackage/example_3857.gpkg"; + } } void TearDown() override {}; std::string path; + std::string path2; }; TEST_F(GeoPackage_Test, geopackage_read_test) @@ -48,3 +59,29 @@ TEST_F(GeoPackage_Test, geopackage_read_test) ASSERT_TRUE(third == nullptr); } + +// this test is essentially the same as the above, however, the coordinates +// are stored in EPSG:3857. When read in, they should convert to EPSG:4326. +TEST_F(GeoPackage_Test, geopackage_projection_test) +{ + const auto gpkg = geopackage::read(this->path2, "example_3857", {}); + EXPECT_NE(gpkg->find("First"), -1); + EXPECT_NE(gpkg->find("Second"), -1); + const auto bbox = gpkg->get_bounding_box(); + EXPECT_EQ(bbox.size(), 4); + EXPECT_NEAR(bbox[0], 102.0, 0.0001); + EXPECT_NEAR(bbox[1], 0.0, 0.0001); + EXPECT_NEAR(bbox[2], 105.0, 0.0001); + EXPECT_NEAR(bbox[3], 1.0, 0.0001); + EXPECT_EQ(2, gpkg->get_size()); + + const auto& first = gpkg->get_feature(0); + const auto& third = gpkg->get_feature(2); + EXPECT_EQ(first->get_id(), "First"); + + const auto& point = boost::get(first->geometry()); + EXPECT_NEAR(point.get<0>(), 102.0, 0.0001); + EXPECT_NEAR(point.get<1>(), 0.5, 0.0001); + + ASSERT_TRUE(third == nullptr); +} \ No newline at end of file From 9eee5d4ba945e815eff20cff202b82ab5f702f2c Mon Sep 17 00:00:00 2001 From: program-- Date: Tue, 9 May 2023 13:21:56 -0700 Subject: [PATCH 28/62] more docs to source files --- src/geopackage/feature.cpp | 7 +++++-- src/geopackage/read.cpp | 19 ++++++++++++++++++- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/src/geopackage/feature.cpp b/src/geopackage/feature.cpp index 6edfcfe362..b2e4ba094d 100644 --- a/src/geopackage/feature.cpp +++ b/src/geopackage/feature.cpp @@ -7,10 +7,13 @@ geojson::Feature geopackage::build_feature( { std::vector bounding_box(4); const auto id = row.get("id"); - geojson::PropertyMap properties = std::move(build_properties(row, geom_col)); - geojson::geometry geometry = std::move(build_geometry(row, geom_col, bounding_box)); + geojson::PropertyMap properties = build_properties(row, geom_col); + geojson::geometry geometry = build_geometry(row, geom_col, bounding_box); + // Convert variant type (0-based) to FeatureType const auto wkb_type = geojson::FeatureType(geometry.which() + 1); + + // Points don't have a bounding box, so we can say its bbox is itself if (wkb_type == geojson::FeatureType::Point) { const auto& pt = boost::get(geometry); bounding_box[0] = pt.get<0>(); diff --git a/src/geopackage/read.cpp b/src/geopackage/read.cpp index 4242f75036..e52b898184 100644 --- a/src/geopackage/read.cpp +++ b/src/geopackage/read.cpp @@ -12,6 +12,10 @@ std::shared_ptr geopackage::read( // Check if layer exists if (!db.has_table(layer)) { + // Since the layer doesn't exist, we need to output some additional + // debug information with the error. In this case, we add ALL the tables + // available in the GPKG, so that if the user sees this error, then it + // might've been either a typo or a bad data input, and they can correct. std::string errmsg = "[" + std::string(sqlite3_errmsg(db.connection())) + "] " + "table " + layer + " does not exist.\n\tTables: "; @@ -26,6 +30,12 @@ std::shared_ptr geopackage::read( } // Layer exists, getting statement for it + // + // this creates a string in the form: + // WHERE id IN (?, ?, ?, ...) + // so that it can be bound by SQLite. + // This is safer than trying to concatenate + // the IDs together. std::string joined_ids = ""; if (!ids.empty()) { joined_ids = " WHERE id IN (?"; @@ -41,6 +51,7 @@ std::shared_ptr geopackage::read( const int layer_feature_count = query_get_layer_count.get(0); #ifndef NGEN_QUIET + // output debug info on what is read exactly std::cout << "Reading " << layer_feature_count << " features in layer " << layer; if (!ids.empty()) { std::cout << " (id subset:"; @@ -61,6 +72,7 @@ std::shared_ptr geopackage::read( sqlite_iter query_get_layer = db.query("SELECT * FROM " + layer + joined_ids, ids); query_get_layer.next(); + // build features out of layer query std::vector features; features.reserve(layer_feature_count); while(!query_get_layer.done()) { @@ -73,7 +85,12 @@ std::shared_ptr geopackage::read( query_get_layer.next(); } - // get layer bounding box + // get layer bounding box from features + // + // GeoPackage contains a bounding box in the SQLite DB, + // however, it is in the SRS of the GPKG. By creating + // the bbox after the features are built, the projection + // is already done. This also should be fairly cheap to do. double min_x = std::numeric_limits::infinity(); double min_y = std::numeric_limits::infinity(); double max_x = -std::numeric_limits::infinity(); From d2de692cb42c58e78108cd3cece82530dba3426b Mon Sep 17 00:00:00 2001 From: program-- Date: Tue, 9 May 2023 13:40:20 -0700 Subject: [PATCH 29/62] add conditional SQLite support --- CMakeLists.txt | 18 +++++++++++++++--- src/NGen.cpp | 8 ++++++++ test/CMakeLists.txt | 5 ++++- 3 files changed, 27 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b5d5156377..6c1b285989 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -113,7 +113,15 @@ endif() find_package(Boost 1.72.0 REQUIRED) -find_package(SQLite3 REQUIRED) # required for now + +find_package(SQLite3) +if(SQLITE3_LIBRARY AND SQLITE3_INCLUDE) + set(NGEN_WITH_SQLITE3 ON) +else() + set(NGEN_WITH_SQLITE3 OFF) +endif() +add_compile_definitions(NGEN_WITH_SQLITE3) +message(INFO " SQLite3 support is ${NGEN_WITH_SQLITE3}") # UDUNITS # Since UDUNITS is currently not really optional (yet) let's make it default to on... @@ -261,7 +269,12 @@ endif() add_subdirectory("src/core") add_dependencies(core libudunits2) add_subdirectory("src/geojson") -add_subdirectory("src/geopackage") + +if(NGEN_WITH_SQLITE3) + add_subdirectory("src/geopackage") + target_link_libraries(ngen PUBLIC NGen::geopackage) +endif() + add_subdirectory("src/realizations/catchment") add_subdirectory("src/models/tshirt") add_subdirectory("src/models/kernels/reservoir") @@ -274,7 +287,6 @@ target_link_libraries(ngen PUBLIC #NGen::core_catchment_giuh NGen::core_nexus NGen::geojson - NGen::geopackage NGen::models_tshirt NGen::realizations_catchment NGen::kernels_reservoir diff --git a/src/NGen.cpp b/src/NGen.cpp index 48da38a5fa..ef9efdcd33 100644 --- a/src/NGen.cpp +++ b/src/NGen.cpp @@ -262,7 +262,11 @@ int main(int argc, char *argv[]) { // TODO: Instead of iterating through a collection of FeatureBase objects mapping to nexi, we instead want to iterate through HY_HydroLocation objects geojson::GeoJSON nexus_collection; if (boost::algorithm::ends_with(nexusDataFile, "gpkg")) { + #ifdef NGEN_WITH_SQLITE3 nexus_collection = geopackage::read(nexusDataFile, "nexus", nexus_subset_ids); + #else + throw std::runtime_error("SQLite3 support required to read GeoPackage files.") + #endif } else { nexus_collection = geojson::read(nexusDataFile, nexus_subset_ids); } @@ -271,7 +275,11 @@ int main(int argc, char *argv[]) { // TODO: Instead of iterating through a collection of FeatureBase objects mapping to catchments, we instead want to iterate through HY_Catchment objects geojson::GeoJSON catchment_collection; if (boost::algorithm::ends_with(catchmentDataFile, "gpkg")) { + #ifdef NGEN_WITH_SQLITE3 catchment_collection = geopackage::read(catchmentDataFile, "divides", catchment_subset_ids); + #else + throw std::runtime_error("SQLite3 support required to read GeoPackage files.") + #endif } else { catchment_collection = geojson::read(catchmentDataFile, catchment_subset_ids); } diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 29a48ccf57..d5bc656388 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -1,5 +1,6 @@ project(test) +include(CTest) add_subdirectory(googletest) include_directories(${gtest_SOURCE_DIR}/include ${gtest_SOURCE_DIR}) include_directories(${gmock_SOURCE_DIR}/include ${gmock_SOURCE_DIR}) @@ -85,13 +86,15 @@ add_test(test_geojson ) ########################## GeoPackage Unit Tests +if(NGEN_WITH_SQLITE3) add_test(test_geopackage 3 geopackage/WKB_Test.cpp geopackage/SQLite_Test.cpp geopackage/GeoPackage_Test.cpp - NGen::core + NGen::core # needed for FileChecker.h NGen::geopackage) +endif() ########################## Realization Config Unit Tests add_test(test_realization_config From 8aab30a733bff95ed56f89a66a6fed871e2bd560 Mon Sep 17 00:00:00 2001 From: program-- Date: Tue, 9 May 2023 13:48:22 -0700 Subject: [PATCH 30/62] add ifdef guard around GeoPackage.hpp include --- src/NGen.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/NGen.cpp b/src/NGen.cpp index ef9efdcd33..f5ecb413ad 100644 --- a/src/NGen.cpp +++ b/src/NGen.cpp @@ -6,7 +6,10 @@ #include "realizations/catchment/Formulation_Manager.hpp" #include #include + +#ifdef NGEN_WITH_SQLITE3 #include +#endif #include "NGenConfig.h" #include "tshirt_params.h" From 8701bfe4b3655eabac6bf5bc00b8a2b5027226a4 Mon Sep 17 00:00:00 2001 From: program-- Date: Wed, 10 May 2023 06:49:54 -0700 Subject: [PATCH 31/62] fix ifdef -> if --- src/NGen.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/NGen.cpp b/src/NGen.cpp index f5ecb413ad..c6e5807f77 100644 --- a/src/NGen.cpp +++ b/src/NGen.cpp @@ -7,7 +7,7 @@ #include #include -#ifdef NGEN_WITH_SQLITE3 +#if NGEN_WITH_SQLITE3 #include #endif From 71824c9b4e526affcee09cbc1dcedbddc36d7303 Mon Sep 17 00:00:00 2001 From: program-- Date: Wed, 10 May 2023 07:03:22 -0700 Subject: [PATCH 32/62] more ifdef -> if fixes --- src/NGen.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/NGen.cpp b/src/NGen.cpp index c6e5807f77..7ed7d52ced 100644 --- a/src/NGen.cpp +++ b/src/NGen.cpp @@ -265,7 +265,7 @@ int main(int argc, char *argv[]) { // TODO: Instead of iterating through a collection of FeatureBase objects mapping to nexi, we instead want to iterate through HY_HydroLocation objects geojson::GeoJSON nexus_collection; if (boost::algorithm::ends_with(nexusDataFile, "gpkg")) { - #ifdef NGEN_WITH_SQLITE3 + #if NGEN_WITH_SQLITE3 nexus_collection = geopackage::read(nexusDataFile, "nexus", nexus_subset_ids); #else throw std::runtime_error("SQLite3 support required to read GeoPackage files.") @@ -278,7 +278,7 @@ int main(int argc, char *argv[]) { // TODO: Instead of iterating through a collection of FeatureBase objects mapping to catchments, we instead want to iterate through HY_Catchment objects geojson::GeoJSON catchment_collection; if (boost::algorithm::ends_with(catchmentDataFile, "gpkg")) { - #ifdef NGEN_WITH_SQLITE3 + #if NGEN_WITH_SQLITE3 catchment_collection = geopackage::read(catchmentDataFile, "divides", catchment_subset_ids); #else throw std::runtime_error("SQLite3 support required to read GeoPackage files.") From 999c523ab05b0e3c0d4be34245fe9ae746952428 Mon Sep 17 00:00:00 2001 From: program-- Date: Wed, 10 May 2023 07:14:06 -0700 Subject: [PATCH 33/62] revert ifdef changes; fix adding compile defs for sqlite --- CMakeLists.txt | 4 ++-- src/NGen.cpp | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 6c1b285989..0ea53b7d41 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -116,11 +116,11 @@ find_package(Boost 1.72.0 REQUIRED) find_package(SQLite3) if(SQLITE3_LIBRARY AND SQLITE3_INCLUDE) - set(NGEN_WITH_SQLITE3 ON) + set(NGEN_WITH_SQLITE3 ON) + add_compile_definitions(NGEN_WITH_SQLITE3) else() set(NGEN_WITH_SQLITE3 OFF) endif() -add_compile_definitions(NGEN_WITH_SQLITE3) message(INFO " SQLite3 support is ${NGEN_WITH_SQLITE3}") # UDUNITS diff --git a/src/NGen.cpp b/src/NGen.cpp index 7ed7d52ced..07e5b5827c 100644 --- a/src/NGen.cpp +++ b/src/NGen.cpp @@ -7,7 +7,7 @@ #include #include -#if NGEN_WITH_SQLITE3 +#ifdef NGEN_WITH_SQLITE3 #include #endif @@ -265,10 +265,10 @@ int main(int argc, char *argv[]) { // TODO: Instead of iterating through a collection of FeatureBase objects mapping to nexi, we instead want to iterate through HY_HydroLocation objects geojson::GeoJSON nexus_collection; if (boost::algorithm::ends_with(nexusDataFile, "gpkg")) { - #if NGEN_WITH_SQLITE3 + #ifdef NGEN_WITH_SQLITE3 nexus_collection = geopackage::read(nexusDataFile, "nexus", nexus_subset_ids); #else - throw std::runtime_error("SQLite3 support required to read GeoPackage files.") + throw std::runtime_error("SQLite3 support required to read GeoPackage files."); #endif } else { nexus_collection = geojson::read(nexusDataFile, nexus_subset_ids); @@ -278,10 +278,10 @@ int main(int argc, char *argv[]) { // TODO: Instead of iterating through a collection of FeatureBase objects mapping to catchments, we instead want to iterate through HY_Catchment objects geojson::GeoJSON catchment_collection; if (boost::algorithm::ends_with(catchmentDataFile, "gpkg")) { - #if NGEN_WITH_SQLITE3 + #ifdef NGEN_WITH_SQLITE3 catchment_collection = geopackage::read(catchmentDataFile, "divides", catchment_subset_ids); #else - throw std::runtime_error("SQLite3 support required to read GeoPackage files.") + throw std::runtime_error("SQLite3 support required to read GeoPackage files."); #endif } else { catchment_collection = geojson::read(catchmentDataFile, catchment_subset_ids); From 97c6db4a810f294e41f2536daad3f15e65a7c652 Mon Sep 17 00:00:00 2001 From: program-- Date: Wed, 10 May 2023 11:47:56 -0700 Subject: [PATCH 34/62] disable optimizations for geometry.cpp if compiler is not intel --- src/geopackage/CMakeLists.txt | 12 ++++++++++++ src/geopackage/geometry.cpp | 2 +- src/geopackage/sqlite/iterator.cpp | 2 +- 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/src/geopackage/CMakeLists.txt b/src/geopackage/CMakeLists.txt index c06ddba4d4..2aead82963 100644 --- a/src/geopackage/CMakeLists.txt +++ b/src/geopackage/CMakeLists.txt @@ -1,4 +1,16 @@ cmake_minimum_required(VERSION 3.10) + + +string(COMPARE EQUAL "${CMAKE_CXX_COMPILER_ID}" "IntelLLVM" _cmp) +if (NOT _cmp) + message(WARNING "[NGen::geopackage] geometry.cpp cannot be optimized with " + "${CMAKE_CXX_COMPILER_ID} due to a suspected compiler/boost issue. " + "Use IntelLLVM if optimization for this source file is required.") + # !! Required due to optimizer issue with either clang or + # !! boost::geometry::srs::transformation + set_source_files_properties(geometry.cpp PROPERTIES COMPILE_FLAGS -O0) +endif() + add_library(geopackage STATIC geometry.cpp properties.cpp feature.cpp diff --git a/src/geopackage/geometry.cpp b/src/geopackage/geometry.cpp index 7949242355..af89b51b07 100644 --- a/src/geopackage/geometry.cpp +++ b/src/geopackage/geometry.cpp @@ -27,7 +27,7 @@ geojson::geometry geopackage::build_geometry( utils::copy_from(geometry_blob, index, srs_id, endian); const auto epsg = wkb::get_prj(srs_id); - const auto prj = bg::srs::transformation<>(epsg, wkb::get_prj(4326)); + const bg::srs::transformation<> prj{epsg, wkb::get_prj(4326)}; wkb::wgs84 pvisitor{srs_id, prj}; if (indicator > 0 & indicator < 5) { diff --git a/src/geopackage/sqlite/iterator.cpp b/src/geopackage/sqlite/iterator.cpp index cb4d1dc309..704bb7a476 100644 --- a/src/geopackage/sqlite/iterator.cpp +++ b/src/geopackage/sqlite/iterator.cpp @@ -94,7 +94,7 @@ int sqlite_iter::num_columns() const noexcept int sqlite_iter::column_index(const std::string& name) const noexcept { - const ptrdiff_t pos = + const size_t pos = std::distance(this->column_names.begin(), std::find(this->column_names.begin(), this->column_names.end(), name)); return pos >= this->column_names.size() ? -1 : pos; From aef9c5665c412d19e1cb052b0a417950bfea7f45 Mon Sep 17 00:00:00 2001 From: program-- Date: Mon, 10 Jul 2023 08:00:27 -0700 Subject: [PATCH 35/62] rev: remove default construction for sqlite object changes per review at https://github.com/NOAA-OWP/ngen/pull/522#discussion_r1256229575 --- include/geopackage/SQLite.hpp | 2 +- test/geopackage/SQLite_Test.cpp | 15 ++++++++------- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/include/geopackage/SQLite.hpp b/include/geopackage/SQLite.hpp index 5e163c3f28..eeccdf103f 100644 --- a/include/geopackage/SQLite.hpp +++ b/include/geopackage/SQLite.hpp @@ -199,7 +199,7 @@ class sqlite stmt_t stmt = nullptr; public: - sqlite() = default; + sqlite() = delete; /** * Construct a new sqlite object from a path to database diff --git a/test/geopackage/SQLite_Test.cpp b/test/geopackage/SQLite_Test.cpp index c1980fe58c..56216ea5a0 100644 --- a/test/geopackage/SQLite_Test.cpp +++ b/test/geopackage/SQLite_Test.cpp @@ -19,32 +19,33 @@ class SQLite_Test : public ::testing::Test if (this->path.empty()) { FAIL() << "can't find gauge_01073000.gpkg"; } - - ASSERT_NO_THROW(this->db = sqlite(this->path)); } void TearDown() override {}; std::string path; - sqlite db; + }; TEST_F(SQLite_Test, sqlite_access_test) { + sqlite db {this->path}; // user wants metadata - EXPECT_TRUE(this->db.has_table("gpkg_contents")); - EXPECT_FALSE(this->db.has_table("some_fake_table")); + EXPECT_TRUE(db.has_table("gpkg_contents")); + EXPECT_FALSE(db.has_table("some_fake_table")); } TEST_F(SQLite_Test, sqlite_query_test) { - if (this->db.connection() == nullptr) { + sqlite db {this->path}; + + if (db.connection() == nullptr) { FAIL() << "database is not loaded"; } // user provides a query const std::string query = "SELECT * FROM gpkg_contents LIMIT 1"; - sqlite_iter iter = this->db.query(query); + sqlite_iter iter = db.query(query); EXPECT_EQ(iter.num_columns(), 10); EXPECT_EQ(iter.columns(), std::vector({ From ec3421d3fdb592fb999ba40fc04e86f5a78a3c94 Mon Sep 17 00:00:00 2001 From: program-- Date: Mon, 10 Jul 2023 08:52:44 -0700 Subject: [PATCH 36/62] rev: fix off-by-one in WKB reader; reorg WKB impl Review changes per https://github.com/NOAA-OWP/ngen/pull/522#discussion_r1256246141. This commit also reorganizes the WKB implementation by moving inline definitions to a source file, since there is no need to inline those functions. --- include/geopackage/WKB.hpp | 283 ++----------------------------- src/geopackage/CMakeLists.txt | 1 + src/geopackage/wkb.cpp | 306 ++++++++++++++++++++++++++++++++++ test/geopackage/WKB_Test.cpp | 64 +++++++ 4 files changed, 383 insertions(+), 271 deletions(-) create mode 100644 src/geopackage/wkb.cpp diff --git a/include/geopackage/WKB.hpp b/include/geopackage/WKB.hpp index 7e0bdc4b8a..d7386324ba 100644 --- a/include/geopackage/WKB.hpp +++ b/include/geopackage/WKB.hpp @@ -88,148 +88,6 @@ struct wkb { static multipolygon_t read_multipolygon(const byte_vector&, int&, uint8_t); }; -inline typename wkb::point_t wkb::read_point(const byte_vector& buffer, int& index, uint8_t order) -{ - double x, y; - utils::copy_from(buffer, index, x, order); - utils::copy_from(buffer, index, y, order); - return point_t{x, y}; -} - -inline typename wkb::linestring_t wkb::read_linestring(const byte_vector& buffer, int& index, uint8_t order) -{ - uint32_t count; - utils::copy_from(buffer, index, count, order); - - linestring_t linestring; - linestring.resize(count); - for (auto& child : linestring) { - child = read_point(buffer, index, order); - } - - return linestring; -} - -inline typename wkb::polygon_t wkb::read_polygon(const byte_vector& buffer, int& index, uint8_t order) -{ - uint32_t count; - utils::copy_from(buffer, index, count, order); - - polygon_t polygon; - - if (count > 1) { - // polygons only have 1 outer ring, - // so any extra vectors are considered to be - // inner rings. - polygon.inners().resize(count); - } - - auto outer = read_linestring(buffer, index, order); - polygon.outer().reserve(outer.size()); - for (auto& p : outer) { - polygon.outer().push_back(p); - } - - for (uint32_t i = 1; i < count; i++) { - auto inner = read_linestring(buffer, index, order); - polygon.inners().at(i).reserve(inner.size()); - for (auto& p : inner) { - polygon.inners().at(i).push_back(p); - } - } - - return polygon; -} - -inline typename wkb::multipoint_t wkb::read_multipoint(const byte_vector& buffer, int& index, uint8_t order) -{ - uint32_t count; - utils::copy_from(buffer, index, count, order); - - multipoint_t mp; - mp.resize(count); - - for (auto& point : mp) { - const byte_t new_order = buffer[index]; - index++; - - uint32_t type; - utils::copy_from(buffer, index, type, new_order); - - point = read_point(buffer, index, new_order); - } - - return mp; -} - -inline typename wkb::multilinestring_t wkb::read_multilinestring(const byte_vector& buffer, int& index, uint8_t order) -{ - uint32_t count; - utils::copy_from(buffer, index, count, order); - - multilinestring_t ml; - ml.resize(count); - for (auto& line : ml) { - const byte_t new_order = buffer[index]; - index++; - - uint32_t type; - utils::copy_from(buffer, index, type, new_order); - - line = read_linestring(buffer, index, new_order); - } - - return ml; -} - -inline typename wkb::multipolygon_t wkb::read_multipolygon(const byte_vector& buffer, int& index, uint8_t order) -{ - uint32_t count; - utils::copy_from(buffer, index, count, order); - - multipolygon_t mpl; - mpl.resize(count); - for (auto& polygon : mpl) { - const byte_t new_order = buffer[index]; - index++; - - uint32_t type; - utils::copy_from(buffer, index, type, new_order); - - polygon = read_polygon(buffer, index, new_order); - } - - return mpl; -} - - -inline typename wkb::geometry wkb::read(const byte_vector& buffer) -{ - if (buffer.size() < 5) { - throw std::runtime_error("buffer reached end before encountering WKB"); - } - - int index = 0; - const byte_t order = buffer[index]; - index++; - - uint32_t type; - utils::copy_from(buffer, index, type, order); - - geometry g; - switch(type) { - case 1: g = read_point(buffer, index, order); break; - case 2: g = read_linestring(buffer, index, order); break; - case 3: g = read_polygon(buffer, index, order); break; - case 4: g = read_multipoint(buffer, index, order); break; - case 5: g = read_multilinestring(buffer, index, order); break; - case 6: g = read_multipolygon(buffer, index, order); break; - default: g = point_t{std::nan("0"), std::nan("0")}; break; - } - - return g; -} - /** * EPSG 5070 projection definition for use with boost::geometry. * @@ -246,6 +104,12 @@ const auto epsg5070 = bg::srs::dpar::parameters<>(bg::srs::dpar::proj_aea) (bg::srs::dpar::x_0, 0) (bg::srs::dpar::y_0, 0); +/** + * EPSG 3857 projection definition for use with boost::geometry. + * + * @note this is required because boost 1.72.0 does not + * have an EPSG definition for 3857 in boost::srs::epsg. + */ const auto epsg3857 = bg::srs::dpar::parameters<>(bg::srs::dpar::proj_merc) (bg::srs::dpar::units_m) (bg::srs::dpar::no_defs) @@ -257,141 +121,18 @@ const auto epsg3857 = bg::srs::dpar::parameters<>(bg::srs::dpar::proj_merc) (bg::srs::dpar::y_0, 0) (bg::srs::dpar::k, 1); -inline bg::srs::dpar::parameters<> wkb::get_prj(uint32_t srid) { - switch(srid) { - case 5070: - return epsg5070; - case 3857: - return epsg3857; - default: - return bg::projections::detail::epsg_to_parameters(srid); - } -} - struct wkb::wgs84 : public boost::static_visitor { wgs84(uint32_t srs, const bg::srs::transformation<>& tr) : srs(srs) , tr(tr) {}; - geojson::geometry operator()(point_t& g) - { - if (this->srs == 4326) { - return geojson::coordinate_t(g.get<0>(), g.get<1>()); - } - - geojson::coordinate_t h; - this->tr.forward(g, h); - return h; - } - - geojson::geometry operator()(linestring_t& g) - { - geojson::linestring_t h; - - if (this->srs == 4326) { - h.reserve(g.size()); - for (auto&& gg : g) { - h.emplace_back( - std::move(gg.get<0>()), - std::move(gg.get<1>()) - ); - } - } else { - this->tr.forward(g, h); - } - return h; - } - - geojson::geometry operator()(polygon_t& g) - { - geojson::polygon_t h; - - if(this->srs == 4326) { - h.outer().reserve(g.outer().size()); - for (auto&& gg : g.outer()) { - h.outer().emplace_back( - std::move(gg.get<0>()), - std::move(gg.get<1>()) - ); - } - - h.inners().resize(g.inners().size()); - auto&& inner_g = g.inners().begin(); - auto&& inner_h = h.inners().begin(); - for (; inner_g != g.inners().end(); inner_g++, inner_h++) { - inner_h->reserve(inner_g->size()); - for (auto&& gg : *inner_g) { - inner_h->emplace_back( - std::move(gg.get<0>()), - std::move(gg.get<1>()) - ); - } - } - } else { - this->tr.forward(g, h); - } - return h; - } - - geojson::geometry operator()(multipoint_t& g) - { - geojson::multipoint_t h; - - if (this->srs == 4326) { - h.reserve(g.size()); - for (auto&& gg : g) { - h.emplace_back( - std::move(gg.get<0>()), - std::move(gg.get<1>()) - ); - } - } else { - this->tr.forward(g, h); - } - - return h; - } - - geojson::geometry operator()(multilinestring_t& g) - { - geojson::multilinestring_t h; - - if (this->srs == 4326) { - h.resize(g.size()); - auto&& line_g = g.begin(); - auto&& line_h = h.begin(); - for(; line_g != g.end(); line_g++, line_h++) { - *line_h = std::move( - boost::get(this->operator()(*line_g)) - ); - } - } else { - this->tr.forward(g, h); - } - - return h; - } - - geojson::geometry operator()(multipolygon_t& g) - { - geojson::multipolygon_t h; - - if (this->srs == 4326) { - h.resize(g.size()); - auto&& polygon_g = g.begin(); - auto&& polygon_h = h.begin(); - for (; polygon_g != g.end(); polygon_g++, polygon_h++) { - *polygon_h = std::move( - boost::get(this->operator()(*polygon_g)) - ); - } - } else { - this->tr.forward(g, h); - } - - return h; - } + geojson::geometry operator()(point_t& g); + geojson::geometry operator()(linestring_t& g); + geojson::geometry operator()(polygon_t& g); + geojson::geometry operator()(multipoint_t& g); + geojson::geometry operator()(multilinestring_t& g); + geojson::geometry operator()(multipolygon_t& g); private: uint32_t srs; diff --git a/src/geopackage/CMakeLists.txt b/src/geopackage/CMakeLists.txt index 2aead82963..5a61c54658 100644 --- a/src/geopackage/CMakeLists.txt +++ b/src/geopackage/CMakeLists.txt @@ -15,6 +15,7 @@ add_library(geopackage STATIC geometry.cpp properties.cpp feature.cpp read.cpp + wkb.cpp sqlite/iterator.cpp sqlite/database.cpp ) diff --git a/src/geopackage/wkb.cpp b/src/geopackage/wkb.cpp new file mode 100644 index 0000000000..c2cabc1d47 --- /dev/null +++ b/src/geopackage/wkb.cpp @@ -0,0 +1,306 @@ +#include "WKB.hpp" + +using namespace geopackage; + +// ---------------------------------------------------------------------------- +// WKB Readers +// ---------------------------------------------------------------------------- + +typename wkb::point_t wkb::read_point(const byte_vector& buffer, int& index, uint8_t order) +{ + double x, y; + utils::copy_from(buffer, index, x, order); + utils::copy_from(buffer, index, y, order); + return point_t{x, y}; +} + +// ---------------------------------------------------------------------------- + +typename wkb::linestring_t wkb::read_linestring(const byte_vector& buffer, int& index, uint8_t order) +{ + uint32_t count; + utils::copy_from(buffer, index, count, order); + + linestring_t linestring; + linestring.resize(count); + for (auto& child : linestring) { + child = read_point(buffer, index, order); + } + + return linestring; +} + +// ---------------------------------------------------------------------------- + +typename wkb::polygon_t wkb::read_polygon(const byte_vector& buffer, int& index, uint8_t order) +{ + uint32_t count; + utils::copy_from(buffer, index, count, order); + + polygon_t polygon; + + if (count > 1) { + // polygons only have 1 outer ring, + // so any extra vectors are considered to be + // inner rings. + polygon.inners().resize(count - 1); + } + + auto outer = read_linestring(buffer, index, order); + polygon.outer().reserve(outer.size()); + for (auto& p : outer) { + polygon.outer().push_back(p); + } + + for (uint32_t i = 0; i < count - 1; i++) { + auto inner = read_linestring(buffer, index, order); + polygon.inners().at(i).reserve(inner.size()); + for (auto& p : inner) { + polygon.inners().at(i).push_back(p); + } + } + + return polygon; +} + +// ---------------------------------------------------------------------------- + +typename wkb::multipoint_t wkb::read_multipoint(const byte_vector& buffer, int& index, uint8_t order) +{ + uint32_t count; + utils::copy_from(buffer, index, count, order); + + multipoint_t mp; + mp.resize(count); + + for (auto& point : mp) { + const byte_t new_order = buffer[index]; + index++; + + uint32_t type; + utils::copy_from(buffer, index, type, new_order); + + point = read_point(buffer, index, new_order); + } + + return mp; +} + +// ---------------------------------------------------------------------------- + +typename wkb::multilinestring_t wkb::read_multilinestring(const byte_vector& buffer, int& index, uint8_t order) +{ + uint32_t count; + utils::copy_from(buffer, index, count, order); + + multilinestring_t ml; + ml.resize(count); + for (auto& line : ml) { + const byte_t new_order = buffer[index]; + index++; + + uint32_t type; + utils::copy_from(buffer, index, type, new_order); + + line = read_linestring(buffer, index, new_order); + } + + return ml; +} + +// ---------------------------------------------------------------------------- + +typename wkb::multipolygon_t wkb::read_multipolygon(const byte_vector& buffer, int& index, uint8_t order) +{ + uint32_t count; + utils::copy_from(buffer, index, count, order); + + multipolygon_t mpl; + mpl.resize(count); + for (auto& polygon : mpl) { + const byte_t new_order = buffer[index]; + index++; + + uint32_t type; + utils::copy_from(buffer, index, type, new_order); + + polygon = read_polygon(buffer, index, new_order); + } + + return mpl; +} + +// ---------------------------------------------------------------------------- + +typename wkb::geometry wkb::read(const byte_vector& buffer) +{ + if (buffer.size() < 5) { + throw std::runtime_error("buffer reached end before encountering WKB"); + } + + int index = 0; + const byte_t order = buffer[index]; + index++; + + uint32_t type; + utils::copy_from(buffer, index, type, order); + + geometry g; + switch(type) { + case 1: g = read_point(buffer, index, order); break; + case 2: g = read_linestring(buffer, index, order); break; + case 3: g = read_polygon(buffer, index, order); break; + case 4: g = read_multipoint(buffer, index, order); break; + case 5: g = read_multilinestring(buffer, index, order); break; + case 6: g = read_multipolygon(buffer, index, order); break; + default: g = point_t{std::nan("0"), std::nan("0")}; break; + } + + return g; +} + +// ---------------------------------------------------------------------------- +// WKB Projection Visitor +// ---------------------------------------------------------------------------- + +bg::srs::dpar::parameters<> wkb::get_prj(uint32_t srid) { + switch(srid) { + case 5070: + return epsg5070; + case 3857: + return epsg3857; + default: + return bg::projections::detail::epsg_to_parameters(srid); + } +} + +// ---------------------------------------------------------------------------- + +geojson::geometry wkb::wgs84::operator()(point_t& g) +{ + if (this->srs == 4326) { + return geojson::coordinate_t(g.get<0>(), g.get<1>()); + } + + geojson::coordinate_t h; + this->tr.forward(g, h); + return h; +} + +// ---------------------------------------------------------------------------- + +geojson::geometry wkb::wgs84::operator()(linestring_t& g) +{ + geojson::linestring_t h; + + if (this->srs == 4326) { + h.reserve(g.size()); + for (auto&& gg : g) { + h.emplace_back( + std::move(gg.get<0>()), + std::move(gg.get<1>()) + ); + } + } else { + this->tr.forward(g, h); + } + return h; +} + +// ---------------------------------------------------------------------------- + +geojson::geometry wkb::wgs84::operator()(polygon_t& g) +{ + geojson::polygon_t h; + + if(this->srs == 4326) { + h.outer().reserve(g.outer().size()); + for (auto&& gg : g.outer()) { + h.outer().emplace_back( + std::move(gg.get<0>()), + std::move(gg.get<1>()) + ); + } + + h.inners().resize(g.inners().size()); + auto&& inner_g = g.inners().begin(); + auto&& inner_h = h.inners().begin(); + for (; inner_g != g.inners().end(); inner_g++, inner_h++) { + inner_h->reserve(inner_g->size()); + for (auto&& gg : *inner_g) { + inner_h->emplace_back( + std::move(gg.get<0>()), + std::move(gg.get<1>()) + ); + } + } + } else { + this->tr.forward(g, h); + } + return h; +} + +// ---------------------------------------------------------------------------- + +geojson::geometry wkb::wgs84::operator()(multipoint_t& g) +{ + geojson::multipoint_t h; + + if (this->srs == 4326) { + h.reserve(g.size()); + for (auto&& gg : g) { + h.emplace_back( + std::move(gg.get<0>()), + std::move(gg.get<1>()) + ); + } + } else { + this->tr.forward(g, h); + } + + return h; +} + +// ---------------------------------------------------------------------------- + +geojson::geometry wkb::wgs84::operator()(multilinestring_t& g) +{ + geojson::multilinestring_t h; + + if (this->srs == 4326) { + h.resize(g.size()); + auto&& line_g = g.begin(); + auto&& line_h = h.begin(); + for(; line_g != g.end(); line_g++, line_h++) { + *line_h = std::move( + boost::get(this->operator()(*line_g)) + ); + } + } else { + this->tr.forward(g, h); + } + + return h; +} + +// ---------------------------------------------------------------------------- + +geojson::geometry wkb::wgs84::operator()(multipolygon_t& g) +{ + geojson::multipolygon_t h; + + if (this->srs == 4326) { + h.resize(g.size()); + auto&& polygon_g = g.begin(); + auto&& polygon_h = h.begin(); + for (; polygon_g != g.end(); polygon_g++, polygon_h++) { + *polygon_h = std::move( + boost::get(this->operator()(*polygon_g)) + ); + } + } else { + this->tr.forward(g, h); + } + + return h; +} diff --git a/test/geopackage/WKB_Test.cpp b/test/geopackage/WKB_Test.cpp index b03b5cf370..f0732a979c 100644 --- a/test/geopackage/WKB_Test.cpp +++ b/test/geopackage/WKB_Test.cpp @@ -107,6 +107,35 @@ class WKB_Test : public ::testing::Test 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3e, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x24, 0x40 }; + + // POLYGON ((0 0, 5 0, 5 5, 0 5, 0 0), (2 2, 3 2, 3 3, 2 3, 2 2)) + this->wkb_bytes["polygon_with_holes"] = { + 0x01, + 0x03, 0x00, 0x00, 0x00, + 0x02, 0x00, 0x00, 0x00, + 0x05, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x14, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x14, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x14, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x14, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x05, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40 + }; } void TearDown() override {} @@ -162,6 +191,41 @@ TEST_F(WKB_Test, wkb_polygon_test) } } +TEST_F(WKB_Test, wkb_polygon_with_holes_test) +{ + const wkb::geometry geom = wkb::read(this->wkb_bytes["polygon_with_holes"]); + EXPECT_EQ(geom.which() + 1, 3); + + const wkb::polygon_t& poly = boost::get(geom); + ASSERT_EQ(poly.inners().size(), 1); + + const std::vector> expected_outer = { + {0, 0}, + {5, 0}, + {5, 5}, + {0, 5}, + {0, 0} + }; + + for (int i = 0; i < expected_outer.size(); i++) { + EXPECT_NEAR(poly.outer()[i].get<0>(), expected_outer[i].first, 0.0001); + EXPECT_NEAR(poly.outer()[i].get<1>(), expected_outer[i].second, 0.0001); + } + + const std::vector> expected_inner = { + {2, 2}, + {3, 2}, + {3, 3}, + {2, 3}, + {2, 2} + }; + + for (int i = 0; i < expected_inner.size(); i++) { + EXPECT_NEAR(poly.inners().at(0)[i].get<0>(), expected_inner[i].first, 0.0001); + EXPECT_NEAR(poly.inners().at(0)[i].get<1>(), expected_inner[i].second, 0.0001); + } +} + TEST_F(WKB_Test, wkb_multipoint_test) { const wkb::geometry geom = wkb::read(this->wkb_bytes["multipoint"]); From d114363a707e802aed82f783021c9593e38c1c21 Mon Sep 17 00:00:00 2001 From: program-- Date: Mon, 10 Jul 2023 09:07:00 -0700 Subject: [PATCH 37/62] rev: remove extra return changes per https://github.com/NOAA-OWP/ngen/pull/522#discussion_r1256293048 --- src/geopackage/geometry.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/geopackage/geometry.cpp b/src/geopackage/geometry.cpp index af89b51b07..11f22edd19 100644 --- a/src/geopackage/geometry.cpp +++ b/src/geopackage/geometry.cpp @@ -79,7 +79,6 @@ geojson::geometry geopackage::build_geometry( auto wkb_geometry = wkb::read(geometry_data); geojson::geometry geometry = boost::apply_visitor(pvisitor, wkb_geometry); return geometry; - return geojson::geometry{}; } else { return geojson::geometry{}; } From a78e9b7c179290743bb8d356ffa4712a7fd566f5 Mon Sep 17 00:00:00 2001 From: program-- Date: Mon, 10 Jul 2023 09:10:22 -0700 Subject: [PATCH 38/62] rev: get_property takes in reference to string changes per https://github.com/NOAA-OWP/ngen/pull/522#discussion_r1256304118 --- src/geopackage/properties.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/geopackage/properties.cpp b/src/geopackage/properties.cpp index 7fd6f7f6aa..a124ebd55b 100644 --- a/src/geopackage/properties.cpp +++ b/src/geopackage/properties.cpp @@ -1,7 +1,7 @@ #include "GeoPackage.hpp" #include "JSONProperty.hpp" -geojson::JSONProperty get_property(const geopackage::sqlite_iter& row, std::string name, int type) +geojson::JSONProperty get_property(const geopackage::sqlite_iter& row, const std::string& name, int type) { if (type == SQLITE_INTEGER) { auto val = row.get(name); From 79148ba1ea1da2a80d06cedeb344365cda9768d3 Mon Sep 17 00:00:00 2001 From: program-- Date: Mon, 10 Jul 2023 09:32:45 -0700 Subject: [PATCH 39/62] rev: add test for geopackage subsetting on read changes per https://github.com/NOAA-OWP/ngen/pull/522#discussion_r1256341575 --- test/geopackage/GeoPackage_Test.cpp | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/test/geopackage/GeoPackage_Test.cpp b/test/geopackage/GeoPackage_Test.cpp index 48516a02cb..391e3cd150 100644 --- a/test/geopackage/GeoPackage_Test.cpp +++ b/test/geopackage/GeoPackage_Test.cpp @@ -60,6 +60,21 @@ TEST_F(GeoPackage_Test, geopackage_read_test) ASSERT_TRUE(third == nullptr); } +TEST_F(GeoPackage_Test, geopackage_idsubset_test) +{ + const auto gpkg = geopackage::read(this->path, "test", { "First" }); + EXPECT_NE(gpkg->find("First"), -1); + EXPECT_EQ(gpkg->find("Second"), -1); + + const auto& first = gpkg->get_feature(0); + EXPECT_EQ(first->get_id(), "First"); + const auto& point = boost::get(first->geometry()); + EXPECT_EQ(point.get<0>(), 102.0); + EXPECT_EQ(point.get<1>(), 0.5); + + ASSERT_TRUE(gpkg->get_feature(1) == nullptr); +} + // this test is essentially the same as the above, however, the coordinates // are stored in EPSG:3857. When read in, they should convert to EPSG:4326. TEST_F(GeoPackage_Test, geopackage_projection_test) @@ -84,4 +99,4 @@ TEST_F(GeoPackage_Test, geopackage_projection_test) EXPECT_NEAR(point.get<1>(), 0.5, 0.0001); ASSERT_TRUE(third == nullptr); -} \ No newline at end of file +} From 4b82a22a52f87578da3c46a37b8335dc5be3a91c Mon Sep 17 00:00:00 2001 From: program-- Date: Mon, 10 Jul 2023 09:34:37 -0700 Subject: [PATCH 40/62] rev: insert newlines to prevent git issues changes per https://github.com/NOAA-OWP/ngen/pull/522#discussion_r1258556140 --- include/geopackage/GeoPackage.hpp | 2 +- include/geopackage/SQLite.hpp | 2 +- src/geopackage/feature.cpp | 2 +- src/geopackage/geometry.cpp | 2 +- src/geopackage/read.cpp | 2 +- src/geopackage/sqlite/database.cpp | 2 +- src/geopackage/sqlite/iterator.cpp | 2 +- test/geopackage/SQLite_Test.cpp | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/include/geopackage/GeoPackage.hpp b/include/geopackage/GeoPackage.hpp index 0b0cd9971d..e0b4d6e650 100644 --- a/include/geopackage/GeoPackage.hpp +++ b/include/geopackage/GeoPackage.hpp @@ -59,4 +59,4 @@ std::shared_ptr read( ); } // namespace geopackage -#endif // NGEN_GEOPACKAGE_H \ No newline at end of file +#endif // NGEN_GEOPACKAGE_H diff --git a/include/geopackage/SQLite.hpp b/include/geopackage/SQLite.hpp index eeccdf103f..1dceea19f6 100644 --- a/include/geopackage/SQLite.hpp +++ b/include/geopackage/SQLite.hpp @@ -270,4 +270,4 @@ class sqlite } // namespace geopackage -#endif // NGEN_GEOPACKAGE_SQLITE_H \ No newline at end of file +#endif // NGEN_GEOPACKAGE_SQLITE_H diff --git a/src/geopackage/feature.cpp b/src/geopackage/feature.cpp index b2e4ba094d..06d516e673 100644 --- a/src/geopackage/feature.cpp +++ b/src/geopackage/feature.cpp @@ -94,4 +94,4 @@ geojson::Feature geopackage::build_feature( {} )); } -} \ No newline at end of file +} diff --git a/src/geopackage/geometry.cpp b/src/geopackage/geometry.cpp index 11f22edd19..f304b761e8 100644 --- a/src/geopackage/geometry.cpp +++ b/src/geopackage/geometry.cpp @@ -82,4 +82,4 @@ geojson::geometry geopackage::build_geometry( } else { return geojson::geometry{}; } -} \ No newline at end of file +} diff --git a/src/geopackage/read.cpp b/src/geopackage/read.cpp index e52b898184..e53797d947 100644 --- a/src/geopackage/read.cpp +++ b/src/geopackage/read.cpp @@ -105,4 +105,4 @@ std::shared_ptr geopackage::read( const auto fc = geojson::FeatureCollection(std::move(features), {min_x, min_y, max_x, max_y}); return std::make_shared(fc); -} \ No newline at end of file +} diff --git a/src/geopackage/sqlite/database.cpp b/src/geopackage/sqlite/database.cpp index 933579bf51..02e3736f76 100644 --- a/src/geopackage/sqlite/database.cpp +++ b/src/geopackage/sqlite/database.cpp @@ -110,4 +110,4 @@ inline sqlite_iter sqlite::query(const std::string& statement, T const&... param this->stmt = stmt_t(stmt, sqlite_deleter{}); return sqlite_iter(this->stmt); -} \ No newline at end of file +} diff --git a/src/geopackage/sqlite/iterator.cpp b/src/geopackage/sqlite/iterator.cpp index 704bb7a476..6982d4e488 100644 --- a/src/geopackage/sqlite/iterator.cpp +++ b/src/geopackage/sqlite/iterator.cpp @@ -132,4 +132,4 @@ std::string sqlite_iter::get(int col) const int size = sqlite3_column_bytes(this->ptr(), col); const unsigned char* ptr = sqlite3_column_text(this->ptr(), col); return std::string(ptr, ptr+size); -} \ No newline at end of file +} diff --git a/test/geopackage/SQLite_Test.cpp b/test/geopackage/SQLite_Test.cpp index 56216ea5a0..b09171c4b3 100644 --- a/test/geopackage/SQLite_Test.cpp +++ b/test/geopackage/SQLite_Test.cpp @@ -108,4 +108,4 @@ TEST_F(SQLite_Test, sqlite_query_test) ASSERT_NO_THROW(iter.next()); EXPECT_TRUE(iter.done()); EXPECT_EQ(iter.current_row(), 1); -} \ No newline at end of file +} From 64b3cfd02328cb96f127ad78220091d159959782 Mon Sep 17 00:00:00 2001 From: program-- Date: Mon, 10 Jul 2023 09:45:30 -0700 Subject: [PATCH 41/62] rev: rm sqlite3 cmake module in favor of built-in changes per https://github.com/NOAA-OWP/ngen/pull/522#discussion_r1258565667 --- CMakeLists.txt | 2 +- cmake/modules/FindSQLite3.cmake | 16 ---------------- 2 files changed, 1 insertion(+), 17 deletions(-) delete mode 100644 cmake/modules/FindSQLite3.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 0ea53b7d41..cd6f9fa1dc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -115,7 +115,7 @@ find_package(Boost 1.72.0 REQUIRED) find_package(SQLite3) -if(SQLITE3_LIBRARY AND SQLITE3_INCLUDE) +if(SQLite3_FOUND) set(NGEN_WITH_SQLITE3 ON) add_compile_definitions(NGEN_WITH_SQLITE3) else() diff --git a/cmake/modules/FindSQLite3.cmake b/cmake/modules/FindSQLite3.cmake deleted file mode 100644 index ffed8ef65e..0000000000 --- a/cmake/modules/FindSQLite3.cmake +++ /dev/null @@ -1,16 +0,0 @@ -message("Looking for SQLite3...") -find_path(SQLITE3_INCLUDE NAMES sqlite3.h) -mark_as_advanced(SQLITE3_INCLUDE) -if(SQLITE3_INCLUDE) - include_directories(${SQLITE3_INCLUDE}) -endif() - -find_library(SQLITE3_LIBRARY NAMES sqlite3) -mark_as_advanced(SQLITE3_LIBRARY) -if(SQLITE3_LIBRARY) - add_library(sqlite3 SHARED IMPORTED) - set_property(TARGET sqlite3 PROPERTY IMPORTED_LOCATION "${SQLITE3_LIBRARY}") -endif() - -include(FindPackageHandleStandardArgs) -FIND_PACKAGE_HANDLE_STANDARD_ARGS(SQLite3 DEFAULT_MSG SQLITE3_LIBRARY SQLITE3_INCLUDE) \ No newline at end of file From 0730d2f0eda078cb706404d4c2feb295e2b492ce Mon Sep 17 00:00:00 2001 From: program-- Date: Mon, 10 Jul 2023 10:02:36 -0700 Subject: [PATCH 42/62] cmake: remove test linking to NGen::core; make includes public --- src/geopackage/CMakeLists.txt | 2 +- test/CMakeLists.txt | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/geopackage/CMakeLists.txt b/src/geopackage/CMakeLists.txt index 5a61c54658..97f25cdd98 100644 --- a/src/geopackage/CMakeLists.txt +++ b/src/geopackage/CMakeLists.txt @@ -21,5 +21,5 @@ add_library(geopackage STATIC geometry.cpp ) add_library(NGen::geopackage ALIAS geopackage) target_include_directories(geopackage PUBLIC ${PROJECT_SOURCE_DIR}/include/geopackage) -target_include_directories(geopackage PRIVATE ${PROJECT_SOURCE_DIR}/include/utilities) +target_include_directories(geopackage PUBLIC ${PROJECT_SOURCE_DIR}/include/utilities) target_link_libraries(geopackage PUBLIC NGen::geojson Boost::boost sqlite3) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index d5bc656388..a7934beb44 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -92,7 +92,6 @@ add_test(test_geopackage geopackage/WKB_Test.cpp geopackage/SQLite_Test.cpp geopackage/GeoPackage_Test.cpp - NGen::core # needed for FileChecker.h NGen::geopackage) endif() From fd548c9ad0e748b12880af830d2e119613733887 Mon Sep 17 00:00:00 2001 From: program-- Date: Mon, 10 Jul 2023 10:52:32 -0700 Subject: [PATCH 43/62] rev: rename SQLite.hpp; move sqlite_error def changes per https://github.com/NOAA-OWP/ngen/pull/522#discussion_r1258593990, and https://github.com/NOAA-OWP/ngen/pull/522#discussion_r1258584308. --- include/geopackage/GeoPackage.hpp | 2 +- .../{SQLite.hpp => NGen_SQLite.hpp} | 38 ------------------- src/geopackage/sqlite/database.cpp | 26 ++++++++++++- src/geopackage/sqlite/iterator.cpp | 19 +++++++++- test/geopackage/SQLite_Test.cpp | 2 +- 5 files changed, 45 insertions(+), 42 deletions(-) rename include/geopackage/{SQLite.hpp => NGen_SQLite.hpp} (84%) diff --git a/include/geopackage/GeoPackage.hpp b/include/geopackage/GeoPackage.hpp index e0b4d6e650..71203f4bcb 100644 --- a/include/geopackage/GeoPackage.hpp +++ b/include/geopackage/GeoPackage.hpp @@ -2,7 +2,7 @@ #define NGEN_GEOPACKAGE_H #include "FeatureCollection.hpp" -#include "SQLite.hpp" +#include "NGen_SQLite.hpp" namespace geopackage { diff --git a/include/geopackage/SQLite.hpp b/include/geopackage/NGen_SQLite.hpp similarity index 84% rename from include/geopackage/SQLite.hpp rename to include/geopackage/NGen_SQLite.hpp index 1dceea19f6..4c9b07b50d 100644 --- a/include/geopackage/SQLite.hpp +++ b/include/geopackage/NGen_SQLite.hpp @@ -10,20 +10,6 @@ namespace geopackage { -/** - * Runtime error for iterations that haven't started - */ -const auto sqlite_get_notstarted_error = std::runtime_error( - "sqlite iteration is has not started, get() is not callable (call sqlite_iter::next() before)" -); - -/** - * Runtime error for iterations that are finished - */ -const auto sqlite_get_done_error = std::runtime_error( - "sqlite iteration is done, get() is not callable" -); - /** * Deleter used to provide smart pointer support for sqlite3 structs. */ @@ -43,30 +29,6 @@ using sqlite_t = std::unique_ptr; */ using stmt_t = std::shared_ptr; -/** - * Get a runtime error based on a function and code. - * - * @param f String denoting the function where the error originated - * @param code sqlite3 result code - * @param extra additional messages to add to the end of the error - * @return std::runtime_error - */ -inline std::runtime_error sqlite_error(const std::string& f, int code, const std::string& extra = "") -{ - std::string errmsg = f + " returned code " - + std::to_string(code) - + " (msg: " - + std::string(sqlite3_errstr(code)) - + ")"; - - if (!extra.empty()) { - errmsg += " "; - errmsg += extra; - } - - return std::runtime_error(errmsg); -} - /** * SQLite3 row iterator * diff --git a/src/geopackage/sqlite/database.cpp b/src/geopackage/sqlite/database.cpp index 02e3736f76..7145f35642 100644 --- a/src/geopackage/sqlite/database.cpp +++ b/src/geopackage/sqlite/database.cpp @@ -1,7 +1,31 @@ -#include "SQLite.hpp" +#include "NGen_SQLite.hpp" using namespace geopackage; +/** + * Get a runtime error based on a function and code. + * + * @param f String denoting the function where the error originated + * @param code sqlite3 result code + * @param extra additional messages to add to the end of the error + * @return std::runtime_error + */ +std::runtime_error sqlite_error(const std::string& f, int code, const std::string& extra = "") +{ + std::string errmsg = f + " returned code " + + std::to_string(code) + + " (msg: " + + std::string(sqlite3_errstr(code)) + + ")"; + + if (!extra.empty()) { + errmsg += " "; + errmsg += extra; + } + + return std::runtime_error(errmsg); +} + sqlite::sqlite(const std::string& path) { sqlite3* conn; diff --git a/src/geopackage/sqlite/iterator.cpp b/src/geopackage/sqlite/iterator.cpp index 6982d4e488..9b0a7ec9e7 100644 --- a/src/geopackage/sqlite/iterator.cpp +++ b/src/geopackage/sqlite/iterator.cpp @@ -1,9 +1,26 @@ #include -#include "SQLite.hpp" +#include "NGen_SQLite.hpp" using namespace geopackage; +// Defined in database.cpp +extern std::runtime_error sqlite_error(const std::string& f, int code, const std::string& extra = ""); + +/** + * Runtime error for iterations that haven't started + */ +const auto sqlite_get_notstarted_error = std::runtime_error( + "sqlite iteration is has not started, get() is not callable (call sqlite_iter::next() before)" +); + +/** + * Runtime error for iterations that are finished + */ +const auto sqlite_get_done_error = std::runtime_error( + "sqlite iteration is done, get() is not callable" +); + sqlite_iter::sqlite_iter(stmt_t stmt) : stmt(stmt) { diff --git a/test/geopackage/SQLite_Test.cpp b/test/geopackage/SQLite_Test.cpp index b09171c4b3..6bba79c2cb 100644 --- a/test/geopackage/SQLite_Test.cpp +++ b/test/geopackage/SQLite_Test.cpp @@ -1,6 +1,6 @@ #include -#include "SQLite.hpp" +#include "NGen_SQLite.hpp" #include "FileChecker.h" using namespace geopackage; From 864a859081856b73349e024935ff4877b84a085a Mon Sep 17 00:00:00 2001 From: program-- Date: Mon, 10 Jul 2023 10:55:30 -0700 Subject: [PATCH 44/62] rev: move sqlite_t type alias into class changes per https://github.com/NOAA-OWP/ngen/pull/522#discussion_r1258599947 --- include/geopackage/NGen_SQLite.hpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/include/geopackage/NGen_SQLite.hpp b/include/geopackage/NGen_SQLite.hpp index 4c9b07b50d..1bc0b53038 100644 --- a/include/geopackage/NGen_SQLite.hpp +++ b/include/geopackage/NGen_SQLite.hpp @@ -19,11 +19,6 @@ struct sqlite_deleter void operator()(sqlite3_stmt* stmt) { sqlite3_finalize(stmt); } }; -/** - * Smart pointer (unique) type for sqlite3 database - */ -using sqlite_t = std::unique_ptr; - /** * Smart pointer (shared) type for sqlite3 prepared statements */ @@ -157,6 +152,11 @@ inline T sqlite_iter::get(const std::string& name) const class sqlite { private: + /** + * Smart pointer (unique) type for sqlite3 database + */ + using sqlite_t = std::unique_ptr; + sqlite_t conn = nullptr; stmt_t stmt = nullptr; From 87618ca069049c6631f8494c7beb2ab964c6c6c7 Mon Sep 17 00:00:00 2001 From: program-- Date: Mon, 10 Jul 2023 11:22:32 -0700 Subject: [PATCH 45/62] rev: delete sqlite copy constructor/assignment Copy constructor/assignment operator are not used for the `sqlite` class, and should subsequently be deleted. This change is implemented per https://github.com/NOAA-OWP/ngen/pull/522#discussion_r1258676455. --- include/geopackage/NGen_SQLite.hpp | 5 +++-- src/geopackage/sqlite/database.cpp | 13 ------------- 2 files changed, 3 insertions(+), 15 deletions(-) diff --git a/include/geopackage/NGen_SQLite.hpp b/include/geopackage/NGen_SQLite.hpp index 1bc0b53038..191522eef5 100644 --- a/include/geopackage/NGen_SQLite.hpp +++ b/include/geopackage/NGen_SQLite.hpp @@ -170,8 +170,9 @@ class sqlite */ sqlite(const std::string& path); - sqlite(sqlite& db); - sqlite& operator=(sqlite& db); + sqlite(sqlite& db) = delete; + + sqlite& operator=(sqlite& db) = delete; /** * Take ownership of a sqlite3 database diff --git a/src/geopackage/sqlite/database.cpp b/src/geopackage/sqlite/database.cpp index 7145f35642..4fdb0f6cd1 100644 --- a/src/geopackage/sqlite/database.cpp +++ b/src/geopackage/sqlite/database.cpp @@ -36,19 +36,6 @@ sqlite::sqlite(const std::string& path) this->conn = sqlite_t(conn); } -sqlite::sqlite(sqlite& db) -{ - this->conn = std::move(db.conn); - this->stmt = db.stmt; -} - -sqlite& sqlite::operator=(sqlite& db) -{ - this->conn = std::move(db.conn); - this->stmt = db.stmt; - return *this; -} - sqlite::sqlite(sqlite&& db) { this->conn = std::move(db.conn); From e99953138885f527ef12f3a1aad38a74d3c93aa8 Mon Sep 17 00:00:00 2001 From: program-- Date: Mon, 10 Jul 2023 11:58:48 -0700 Subject: [PATCH 46/62] rev: rm reinitialization in constructor changes per https://github.com/NOAA-OWP/ngen/pull/522/files/071e8f87cd8110bb6fee7167aa8398843141e82c#r1258698308. --- src/geopackage/sqlite/iterator.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/geopackage/sqlite/iterator.cpp b/src/geopackage/sqlite/iterator.cpp index 9b0a7ec9e7..befffbab27 100644 --- a/src/geopackage/sqlite/iterator.cpp +++ b/src/geopackage/sqlite/iterator.cpp @@ -30,9 +30,7 @@ sqlite_iter::sqlite_iter(stmt_t stmt) // was SQLITE_DONE. this->next(); this->column_count = sqlite3_column_count(this->ptr()); - this->column_names = std::vector(); this->column_names.reserve(this->column_count); - this->column_types = std::vector(); this->column_types.reserve(this->column_count); for (int i = 0; i < this->column_count; i++) { From fcd9d6d02a96060afd8696fa86fd8fdc8159df3b Mon Sep 17 00:00:00 2001 From: program-- Date: Mon, 10 Jul 2023 12:02:22 -0700 Subject: [PATCH 47/62] rev: set sqlite move constructor/assignment to default changes per https://github.com/NOAA-OWP/ngen/pull/522#discussion_r1258687723. --- include/geopackage/NGen_SQLite.hpp | 4 ++-- src/geopackage/sqlite/database.cpp | 13 ------------- 2 files changed, 2 insertions(+), 15 deletions(-) diff --git a/include/geopackage/NGen_SQLite.hpp b/include/geopackage/NGen_SQLite.hpp index 191522eef5..eecc999748 100644 --- a/include/geopackage/NGen_SQLite.hpp +++ b/include/geopackage/NGen_SQLite.hpp @@ -179,7 +179,7 @@ class sqlite * * @param db sqlite3 database object */ - sqlite(sqlite&& db); + sqlite(sqlite&& db) = default; /** * Move assignment operator @@ -187,7 +187,7 @@ class sqlite * @param db sqlite3 database object * @return sqlite& reference to sqlite3 database */ - sqlite& operator=(sqlite&& db); + sqlite& operator=(sqlite&& db) = default; /** * Return the originating sqlite3 database pointer diff --git a/src/geopackage/sqlite/database.cpp b/src/geopackage/sqlite/database.cpp index 4fdb0f6cd1..20bc07e5ee 100644 --- a/src/geopackage/sqlite/database.cpp +++ b/src/geopackage/sqlite/database.cpp @@ -36,19 +36,6 @@ sqlite::sqlite(const std::string& path) this->conn = sqlite_t(conn); } -sqlite::sqlite(sqlite&& db) -{ - this->conn = std::move(db.conn); - this->stmt = db.stmt; -} - -sqlite& sqlite::operator=(sqlite&& db) -{ - this->conn = std::move(db.conn); - this->stmt = db.stmt; - return *this; -} - sqlite3* sqlite::connection() const noexcept { return this->conn.get(); From 9dcecbf43ab5b9f110804c687f86808325e08923 Mon Sep 17 00:00:00 2001 From: program-- Date: Tue, 11 Jul 2023 08:34:54 -0700 Subject: [PATCH 48/62] rev: remove `stmt` from sqlite; shared -> unique changes per https://github.com/NOAA-OWP/ngen/pull/522#discussion_r1258715285 and https://github.com/NOAA-OWP/ngen/pull/522#discussion_r1258717997. --- include/geopackage/NGen_SQLite.hpp | 8 +------- src/geopackage/read.cpp | 3 ++- src/geopackage/sqlite/database.cpp | 9 +++------ src/geopackage/sqlite/iterator.cpp | 10 +++------- 4 files changed, 9 insertions(+), 21 deletions(-) diff --git a/include/geopackage/NGen_SQLite.hpp b/include/geopackage/NGen_SQLite.hpp index eecc999748..9aaf55d3a7 100644 --- a/include/geopackage/NGen_SQLite.hpp +++ b/include/geopackage/NGen_SQLite.hpp @@ -22,7 +22,7 @@ struct sqlite_deleter /** * Smart pointer (shared) type for sqlite3 prepared statements */ -using stmt_t = std::shared_ptr; +using stmt_t = std::unique_ptr; /** * SQLite3 row iterator @@ -77,11 +77,6 @@ class sqlite_iter */ sqlite_iter& restart(); - /** - * Call the row iterator destructor - */ - void close(); - /** * Get the current row index for the iterator * @@ -158,7 +153,6 @@ class sqlite using sqlite_t = std::unique_ptr; sqlite_t conn = nullptr; - stmt_t stmt = nullptr; public: sqlite() = delete; diff --git a/src/geopackage/read.cpp b/src/geopackage/read.cpp index e53797d947..218d817761 100644 --- a/src/geopackage/read.cpp +++ b/src/geopackage/read.cpp @@ -19,7 +19,8 @@ std::shared_ptr geopackage::read( std::string errmsg = "[" + std::string(sqlite3_errmsg(db.connection())) + "] " + "table " + layer + " does not exist.\n\tTables: "; - auto errquery = db.query("SELECT name FROM sqlite_master WHERE type='table'").next(); + auto errquery = db.query("SELECT name FROM sqlite_master WHERE type='table'"); + errquery.next(); while(!errquery.done()) { errmsg += errquery.get(0); errmsg += ", "; diff --git a/src/geopackage/sqlite/database.cpp b/src/geopackage/sqlite/database.cpp index 20bc07e5ee..b768aa1882 100644 --- a/src/geopackage/sqlite/database.cpp +++ b/src/geopackage/sqlite/database.cpp @@ -60,8 +60,7 @@ sqlite_iter sqlite::query(const std::string& statement) throw sqlite_error("sqlite3_prepare_v2", code, "(query: " + statement + ")"); } - this->stmt = stmt_t(stmt, sqlite_deleter{}); - return sqlite_iter(this->stmt); + return sqlite_iter(std::move(stmt_t(stmt))); } sqlite_iter sqlite::query(const std::string& statement, const std::vector& binds) @@ -83,8 +82,7 @@ sqlite_iter sqlite::query(const std::string& statement, const std::vectorstmt = stmt_t(stmt, sqlite_deleter{}); - return sqlite_iter(this->stmt); + return sqlite_iter(std::move(stmt_t(stmt))); } template @@ -106,6 +104,5 @@ inline sqlite_iter sqlite::query(const std::string& statement, T const&... param } } - this->stmt = stmt_t(stmt, sqlite_deleter{}); - return sqlite_iter(this->stmt); + return sqlite_iter(std::move(stmt_t(stmt))); } diff --git a/src/geopackage/sqlite/iterator.cpp b/src/geopackage/sqlite/iterator.cpp index befffbab27..4cd322533f 100644 --- a/src/geopackage/sqlite/iterator.cpp +++ b/src/geopackage/sqlite/iterator.cpp @@ -1,4 +1,5 @@ #include +#include #include "NGen_SQLite.hpp" @@ -22,7 +23,7 @@ const auto sqlite_get_done_error = std::runtime_error( ); sqlite_iter::sqlite_iter(stmt_t stmt) - : stmt(stmt) + : stmt(std::move(stmt)) { // sqlite3_column_type requires the last result code to be // SQLITE_ROW, so we need to iterate on the first row. @@ -39,7 +40,7 @@ sqlite_iter::sqlite_iter(stmt_t stmt) } this->restart(); -}; +} sqlite3_stmt* sqlite_iter::ptr() const noexcept { @@ -92,11 +93,6 @@ sqlite_iter& sqlite_iter::restart() return *this; } -void sqlite_iter::close() -{ - this->~sqlite_iter(); -} - int sqlite_iter::current_row() const noexcept { return this->iteration_step; From 4e18ca2be2c45eebaaf9900885853e40a8f95b0f Mon Sep 17 00:00:00 2001 From: program-- Date: Tue, 11 Jul 2023 08:39:08 -0700 Subject: [PATCH 49/62] rev: add comments regarding WKB implementation changes per https://github.com/NOAA-OWP/ngen/pull/522#discussion_r1258944119 --- include/geopackage/WKB.hpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/include/geopackage/WKB.hpp b/include/geopackage/WKB.hpp index d7386324ba..a4927c017b 100644 --- a/include/geopackage/WKB.hpp +++ b/include/geopackage/WKB.hpp @@ -11,7 +11,12 @@ namespace bg = boost::geometry; namespace geopackage { /** - * A recursive WKB reader + * A recursive WKB reader. + * + * @note This WKB implementation follows a subset of the + * OGC Specification found in https://www.ogc.org/standard/sfa/. + * This is a strict WKB implementation and does not support + * Extended WKB (EWKB) or Tiny WKB (TWKB). */ struct wkb { using point_t = bg::model::point; From 9e44d426df9afd83be84f710c358020178133fb9 Mon Sep 17 00:00:00 2001 From: program-- Date: Tue, 11 Jul 2023 08:52:27 -0700 Subject: [PATCH 50/62] rev: add geometry type guards in wkb reader changes per https://github.com/NOAA-OWP/ngen/pull/522#discussion_r1258951431. --- src/geopackage/wkb.cpp | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/src/geopackage/wkb.cpp b/src/geopackage/wkb.cpp index c2cabc1d47..33efd5b90e 100644 --- a/src/geopackage/wkb.cpp +++ b/src/geopackage/wkb.cpp @@ -2,6 +2,29 @@ using namespace geopackage; +enum wkb_geom_t { + geometry = 0, + point = 1, + linestring = 2, + polygon = 3, + multipoint = 4, + multilinestring = 5, + multipolygon = 6, + geometry_collection = 7 +}; + +void throw_if_not_type(uint32_t given, wkb_geom_t expected) +{ + if (given != expected) { + throw std::runtime_error( + "expected WKB geometry type " + + std::to_string(expected) + + ", but received " + + std::to_string(given) + ); + } +} + // ---------------------------------------------------------------------------- // WKB Readers // ---------------------------------------------------------------------------- @@ -79,6 +102,7 @@ typename wkb::multipoint_t wkb::read_multipoint(const byte_vector& buffer, int& uint32_t type; utils::copy_from(buffer, index, type, new_order); + throw_if_not_type(type, wkb_geom_t::point); point = read_point(buffer, index, new_order); } @@ -101,6 +125,7 @@ typename wkb::multilinestring_t wkb::read_multilinestring(const byte_vector& buf uint32_t type; utils::copy_from(buffer, index, type, new_order); + throw_if_not_type(type, wkb_geom_t::linestring); line = read_linestring(buffer, index, new_order); } @@ -123,6 +148,7 @@ typename wkb::multipolygon_t wkb::read_multipolygon(const byte_vector& buffer, i uint32_t type; utils::copy_from(buffer, index, type, new_order); + throw_if_not_type(type, wkb_geom_t::polygon); polygon = read_polygon(buffer, index, new_order); } From f244bb42039479097352a2c8971665a9e342f4b4 Mon Sep 17 00:00:00 2001 From: program-- Date: Tue, 11 Jul 2023 08:56:42 -0700 Subject: [PATCH 51/62] rev: throw exception if invalid wkb type changes per https://github.com/NOAA-OWP/ngen/pull/522#discussion_r1258952435 also modifies switch to return instead of assign to `g`. --- src/geopackage/wkb.cpp | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/src/geopackage/wkb.cpp b/src/geopackage/wkb.cpp index 33efd5b90e..6c7b663690 100644 --- a/src/geopackage/wkb.cpp +++ b/src/geopackage/wkb.cpp @@ -171,18 +171,25 @@ typename wkb::geometry wkb::read(const byte_vector& buffer) uint32_t type; utils::copy_from(buffer, index, type, order); - geometry g; switch(type) { - case 1: g = read_point(buffer, index, order); break; - case 2: g = read_linestring(buffer, index, order); break; - case 3: g = read_polygon(buffer, index, order); break; - case 4: g = read_multipoint(buffer, index, order); break; - case 5: g = read_multilinestring(buffer, index, order); break; - case 6: g = read_multipolygon(buffer, index, order); break; - default: g = point_t{std::nan("0"), std::nan("0")}; break; + case 1: + return read_point(buffer, index, order); + case 2: + return read_linestring(buffer, index, order); + case 3: + return read_polygon(buffer, index, order); + case 4: + return read_multipoint(buffer, index, order); + case 5: + return read_multilinestring(buffer, index, order); + case 6: + return read_multipolygon(buffer, index, order); + default: + throw std::runtime_error( + "this reader only implements OGC geometry types 1-6, " + "but received type " + std::to_string(type) + ); } - - return g; } // ---------------------------------------------------------------------------- From 74018aafb05cd7f31fecf42e77f1c34471fb63bd Mon Sep 17 00:00:00 2001 From: "Justin Singh-M. - NOAA" Date: Tue, 11 Jul 2023 09:01:18 -0700 Subject: [PATCH 52/62] rev: rewrite `std::memcpy` call in `copy_from` Co-authored-by: Phil Miller - NOAA --- include/utilities/EndianCopy.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/utilities/EndianCopy.hpp b/include/utilities/EndianCopy.hpp index abcebe90d2..70cc25355a 100644 --- a/include/utilities/EndianCopy.hpp +++ b/include/utilities/EndianCopy.hpp @@ -23,7 +23,7 @@ namespace utils { template void copy_from(const std::vector& src, int& index, S& dst, uint8_t order) { - std::memcpy(&dst, &src[index], sizeof(S)); + std::memcpy(&dst, src.data() + index, sizeof(dst)); if (order == 0x01) { boost::endian::little_to_native_inplace(dst); From 9ceed0c353bbe8819e1bd97e7958fb782ea0678c Mon Sep 17 00:00:00 2001 From: program-- Date: Tue, 11 Jul 2023 09:02:45 -0700 Subject: [PATCH 53/62] rev: add newline to --- include/utilities/EndianCopy.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/utilities/EndianCopy.hpp b/include/utilities/EndianCopy.hpp index 70cc25355a..8bce187052 100644 --- a/include/utilities/EndianCopy.hpp +++ b/include/utilities/EndianCopy.hpp @@ -55,4 +55,4 @@ inline void copy_from(const std::vector& src, int& index, doubl } -#endif // NGEN_ENDIANCOPY_H \ No newline at end of file +#endif // NGEN_ENDIANCOPY_H From a562388db242af34a59e40322a6d76159fc62a6c Mon Sep 17 00:00:00 2001 From: program-- Date: Tue, 11 Jul 2023 09:06:14 -0700 Subject: [PATCH 54/62] rev: handle FeatureCollection `shared_ptr` creation changes per https://github.com/NOAA-OWP/ngen/pull/522#discussion_r1258985474. --- src/geopackage/read.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/geopackage/read.cpp b/src/geopackage/read.cpp index 218d817761..7302a7d3b1 100644 --- a/src/geopackage/read.cpp +++ b/src/geopackage/read.cpp @@ -104,6 +104,8 @@ std::shared_ptr geopackage::read( max_y = bbox[3] > max_y ? bbox[3] : max_y; } - const auto fc = geojson::FeatureCollection(std::move(features), {min_x, min_y, max_x, max_y}); - return std::make_shared(fc); + return std::make_shared( + std::move(features), + std::vector({min_x, min_y, max_x, max_y}) + ); } From 19849fcdd112498f283ad9c4470298cb07005183 Mon Sep 17 00:00:00 2001 From: program-- Date: Tue, 11 Jul 2023 09:08:59 -0700 Subject: [PATCH 55/62] rev: remove statement output from sqlite_error calls changes per https://github.com/NOAA-OWP/ngen/pull/522#discussion_r1258986962 --- src/geopackage/sqlite/database.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/geopackage/sqlite/database.cpp b/src/geopackage/sqlite/database.cpp index b768aa1882..41a7d418a4 100644 --- a/src/geopackage/sqlite/database.cpp +++ b/src/geopackage/sqlite/database.cpp @@ -57,7 +57,7 @@ sqlite_iter sqlite::query(const std::string& statement) if (code != SQLITE_OK) { // something happened, can probably switch on result codes // https://www.sqlite.org/rescode.html - throw sqlite_error("sqlite3_prepare_v2", code, "(query: " + statement + ")"); + throw sqlite_error("sqlite3_prepare_v2", code); } return sqlite_iter(std::move(stmt_t(stmt))); @@ -70,7 +70,7 @@ sqlite_iter sqlite::query(const std::string& statement, const std::vectorconnection(), cstmt, statement.length() + 1, &stmt, NULL); if (code != SQLITE_OK) { - throw sqlite_error("sqlite3_prepare_v2", code, "(query: " + statement + ")"); + throw sqlite_error("sqlite3_prepare_v2", code); } if (!binds.empty()) { @@ -93,7 +93,7 @@ inline sqlite_iter sqlite::query(const std::string& statement, T const&... param const int code = sqlite3_prepare_v2(this->connection(), cstmt, statement.length() + 1, &stmt, NULL); if (code != SQLITE_OK) { - throw sqlite_error("sqlite3_prepare_v2", code, "(query: " + statement + ")"); + throw sqlite_error("sqlite3_prepare_v2", code); } std::vector binds{ { params... } }; From 5698854fab3919bef1d04c2b10502d130588dc79 Mon Sep 17 00:00:00 2001 From: program-- Date: Tue, 11 Jul 2023 09:18:13 -0700 Subject: [PATCH 56/62] rev: move point bbox into switch changes per https://github.com/NOAA-OWP/ngen/pull/522#discussion_r1256264544 and also modifies `std::make_shared` calls within the switch statement to reflect forwarding, instead of construction before passing. --- src/geopackage/feature.cpp | 83 +++++++++++++++----------------------- 1 file changed, 32 insertions(+), 51 deletions(-) diff --git a/src/geopackage/feature.cpp b/src/geopackage/feature.cpp index 06d516e673..76b95378de 100644 --- a/src/geopackage/feature.cpp +++ b/src/geopackage/feature.cpp @@ -1,5 +1,15 @@ #include "GeoPackage.hpp" +// Points don't have a bounding box, so we can say its bbox is itself +inline void build_point_bbox(const geojson::geometry& geom, std::vector& bbox) +{ + const auto& pt = boost::get(geom); + bbox[0] = pt.get<0>(); + bbox[1] = pt.get<1>(); + bbox[2] = pt.get<0>(); + bbox[3] = pt.get<1>(); +} + geojson::Feature geopackage::build_feature( const sqlite_iter& row, const std::string& geom_col @@ -13,85 +23,56 @@ geojson::Feature geopackage::build_feature( // Convert variant type (0-based) to FeatureType const auto wkb_type = geojson::FeatureType(geometry.which() + 1); - // Points don't have a bounding box, so we can say its bbox is itself - if (wkb_type == geojson::FeatureType::Point) { - const auto& pt = boost::get(geometry); - bounding_box[0] = pt.get<0>(); - bounding_box[1] = pt.get<1>(); - bounding_box[2] = pt.get<0>(); - bounding_box[3] = pt.get<1>(); - } - switch(wkb_type) { case geojson::FeatureType::Point: - return std::make_shared(geojson::PointFeature( + build_point_bbox(geometry, bounding_box); + return std::make_shared( boost::get(geometry), id, properties, - bounding_box, - std::vector(), - std::vector(), - {} - )); + bounding_box + ); case geojson::FeatureType::LineString: - return std::make_shared(geojson::LineStringFeature( + return std::make_shared( boost::get(geometry), id, properties, - bounding_box, - std::vector(), - std::vector(), - {} - )); + bounding_box + ); case geojson::FeatureType::Polygon: - return std::make_shared(geojson::PolygonFeature( + return std::make_shared( boost::get(geometry), id, properties, - bounding_box, - std::vector(), - std::vector(), - {} - )); + bounding_box + ); case geojson::FeatureType::MultiPoint: - return std::make_shared(geojson::MultiPointFeature( + return std::make_shared( boost::get(geometry), id, properties, - bounding_box, - std::vector(), - std::vector(), - {} - )); + bounding_box + ); case geojson::FeatureType::MultiLineString: - return std::make_shared(geojson::MultiLineStringFeature( + return std::make_shared( boost::get(geometry), id, properties, - bounding_box, - std::vector(), - std::vector(), - {} - )); + bounding_box + ); case geojson::FeatureType::MultiPolygon: - return std::make_shared(geojson::MultiPolygonFeature( + return std::make_shared( boost::get(geometry), id, properties, - bounding_box, - std::vector(), - std::vector(), - {} - )); + bounding_box + ); default: - return std::make_shared(geojson::CollectionFeature( + return std::make_shared( std::vector{geometry}, id, properties, - bounding_box, - std::vector(), - std::vector(), - {} - )); + bounding_box + ); } } From 894e8ab9633cd9fbbd355c85a80c1b6bc2d327e8 Mon Sep 17 00:00:00 2001 From: program-- Date: Tue, 11 Jul 2023 10:39:48 -0700 Subject: [PATCH 57/62] rev: use wkb_geom_t enum for switch changes per https://github.com/NOAA-OWP/ngen/pull/522#discussion_r1260056159 --- src/geopackage/wkb.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/geopackage/wkb.cpp b/src/geopackage/wkb.cpp index 6c7b663690..a47974799b 100644 --- a/src/geopackage/wkb.cpp +++ b/src/geopackage/wkb.cpp @@ -172,17 +172,17 @@ typename wkb::geometry wkb::read(const byte_vector& buffer) utils::copy_from(buffer, index, type, order); switch(type) { - case 1: + case wkb_geom_t::point: return read_point(buffer, index, order); - case 2: + case wkb_geom_t::linestring: return read_linestring(buffer, index, order); - case 3: + case wkb_geom_t::polygon: return read_polygon(buffer, index, order); - case 4: + case wkb_geom_t::multipoint: return read_multipoint(buffer, index, order); - case 5: + case wkb_geom_t::multilinestring: return read_multilinestring(buffer, index, order); - case 6: + case wkb_geom_t::multipolygon: return read_multipolygon(buffer, index, order); default: throw std::runtime_error( From 54197982c02b8d4d270a0fc30290e73632fcc3a8 Mon Sep 17 00:00:00 2001 From: program-- Date: Wed, 12 Jul 2023 13:41:42 -0700 Subject: [PATCH 58/62] rev: scientific notation; cmake: update optimizer warning changes per https://github.com/NOAA-OWP/ngen/pull/522#discussion_r1260136475 --- src/geopackage/CMakeLists.txt | 7 ++++--- test/geopackage/WKB_Test.cpp | 36 +++++++++++++++++------------------ 2 files changed, 22 insertions(+), 21 deletions(-) diff --git a/src/geopackage/CMakeLists.txt b/src/geopackage/CMakeLists.txt index 97f25cdd98..14a8f1324d 100644 --- a/src/geopackage/CMakeLists.txt +++ b/src/geopackage/CMakeLists.txt @@ -3,12 +3,13 @@ cmake_minimum_required(VERSION 3.10) string(COMPARE EQUAL "${CMAKE_CXX_COMPILER_ID}" "IntelLLVM" _cmp) if (NOT _cmp) - message(WARNING "[NGen::geopackage] geometry.cpp cannot be optimized with " - "${CMAKE_CXX_COMPILER_ID} due to a suspected compiler/boost issue. " + message(WARNING "[NGen::geopackage] wkb.cpp cannot be optimized with " + "${CMAKE_CXX_COMPILER_ID} due to a suspected optimizer/boost issue:\n" + "https://github.com/NOAA-OWP/ngen/issues/567.\n" "Use IntelLLVM if optimization for this source file is required.") # !! Required due to optimizer issue with either clang or # !! boost::geometry::srs::transformation - set_source_files_properties(geometry.cpp PROPERTIES COMPILE_FLAGS -O0) + set_source_files_properties(wkb.cpp PROPERTIES COMPILE_FLAGS -O0) endif() add_library(geopackage STATIC geometry.cpp diff --git a/test/geopackage/WKB_Test.cpp b/test/geopackage/WKB_Test.cpp index f0732a979c..13d2819c80 100644 --- a/test/geopackage/WKB_Test.cpp +++ b/test/geopackage/WKB_Test.cpp @@ -147,12 +147,12 @@ TEST_F(WKB_Test, wkb_point_test) // also tests endianness { const auto geom_big = boost::get(wkb::read(this->wkb_bytes["point_big"])); const auto geom_little = boost::get(wkb::read(this->wkb_bytes["point"])); - EXPECT_NEAR(geom_big.get<0>(), geom_little.get<0>(), 0.000001); - EXPECT_NEAR(geom_big.get<1>(), geom_little.get<1>(), 0.000001); - EXPECT_NEAR(geom_big.get<0>(), 2.0, 0.000001); - EXPECT_NEAR(geom_big.get<1>(), 4.0, 0.000001); - EXPECT_NEAR(geom_big.get<0>(), 2.0, 0.000001); - EXPECT_NEAR(geom_big.get<1>(), 4.0, 0.000001); + EXPECT_NEAR(geom_big.get<0>(), geom_little.get<0>(), 1e-6); + EXPECT_NEAR(geom_big.get<1>(), geom_little.get<1>(), 1e-6); + EXPECT_NEAR(geom_big.get<0>(), 2.0, 1e-6); + EXPECT_NEAR(geom_big.get<1>(), 4.0, 1e-6); + EXPECT_NEAR(geom_big.get<0>(), 2.0, 1e-6); + EXPECT_NEAR(geom_big.get<1>(), 4.0, 1e-6); } TEST_F(WKB_Test, wkb_linestring_test) @@ -166,8 +166,8 @@ TEST_F(WKB_Test, wkb_linestring_test) }; for (int i = 0; i < expected_coordinates.size(); i++) { - EXPECT_NEAR(line[i].get<0>(), expected_coordinates[i].first, 0.0001); - EXPECT_NEAR(line[i].get<1>(), expected_coordinates[i].second, 0.0001); + EXPECT_NEAR(line[i].get<0>(), expected_coordinates[i].first, 1e-4); + EXPECT_NEAR(line[i].get<1>(), expected_coordinates[i].second, 1e-4); } } @@ -186,8 +186,8 @@ TEST_F(WKB_Test, wkb_polygon_test) }; for (int i = 0; i < expected_coordinates.size(); i++) { - EXPECT_NEAR(poly.outer().at(i).get<0>(), expected_coordinates[i].first, 0.0001); - EXPECT_NEAR(poly.outer().at(i).get<1>(), expected_coordinates[i].second, 0.0001); + EXPECT_NEAR(poly.outer().at(i).get<0>(), expected_coordinates[i].first, 1e-4); + EXPECT_NEAR(poly.outer().at(i).get<1>(), expected_coordinates[i].second, 1e-4); } } @@ -208,8 +208,8 @@ TEST_F(WKB_Test, wkb_polygon_with_holes_test) }; for (int i = 0; i < expected_outer.size(); i++) { - EXPECT_NEAR(poly.outer()[i].get<0>(), expected_outer[i].first, 0.0001); - EXPECT_NEAR(poly.outer()[i].get<1>(), expected_outer[i].second, 0.0001); + EXPECT_NEAR(poly.outer()[i].get<0>(), expected_outer[i].first, 1e-4); + EXPECT_NEAR(poly.outer()[i].get<1>(), expected_outer[i].second, 1e-4); } const std::vector> expected_inner = { @@ -221,8 +221,8 @@ TEST_F(WKB_Test, wkb_polygon_with_holes_test) }; for (int i = 0; i < expected_inner.size(); i++) { - EXPECT_NEAR(poly.inners().at(0)[i].get<0>(), expected_inner[i].first, 0.0001); - EXPECT_NEAR(poly.inners().at(0)[i].get<1>(), expected_inner[i].second, 0.0001); + EXPECT_NEAR(poly.inners().at(0)[i].get<0>(), expected_inner[i].first, 1e-4); + EXPECT_NEAR(poly.inners().at(0)[i].get<1>(), expected_inner[i].second, 1e-4); } } @@ -237,8 +237,8 @@ TEST_F(WKB_Test, wkb_multipoint_test) }; for (int i = 0; i < expected_coordinates.size(); i++) { - EXPECT_NEAR(mp[i].get<0>(), expected_coordinates[i].first, 0.0001); - EXPECT_NEAR(mp[i].get<1>(), expected_coordinates[i].second, 0.0001); + EXPECT_NEAR(mp[i].get<0>(), expected_coordinates[i].first, 1e-4); + EXPECT_NEAR(mp[i].get<1>(), expected_coordinates[i].second, 1e-4); } } @@ -255,8 +255,8 @@ TEST_F(WKB_Test, wkb_multilinestring_test) for (int i = 0; i < expected_coordinates.size(); i++) { for (int j = 0; j < expected_coordinates[i].size(); j++) { - EXPECT_NEAR(mp[i][j].get<0>(), expected_coordinates[i][j].first, 0.0001); - EXPECT_NEAR(mp[i][j].get<1>(), expected_coordinates[i][j].second, 0.0001); + EXPECT_NEAR(mp[i][j].get<0>(), expected_coordinates[i][j].first, 1e-4); + EXPECT_NEAR(mp[i][j].get<1>(), expected_coordinates[i][j].second, 1e-4); } } } From 5531a89badcdfe2de89a5dc6d1fbf112c342a760 Mon Sep 17 00:00:00 2001 From: program-- Date: Wed, 12 Jul 2023 13:44:21 -0700 Subject: [PATCH 59/62] rev: change literal byte keys for clarity changes per https://github.com/NOAA-OWP/ngen/pull/522#discussion_r1260138439 and https://github.com/NOAA-OWP/ngen/pull/522#discussion_r1260197844 Co-authored-by: Phil Miller - NOAA --- test/geopackage/WKB_Test.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/geopackage/WKB_Test.cpp b/test/geopackage/WKB_Test.cpp index 13d2819c80..74bcdbf1fb 100644 --- a/test/geopackage/WKB_Test.cpp +++ b/test/geopackage/WKB_Test.cpp @@ -13,14 +13,14 @@ class WKB_Test : public ::testing::Test ~WKB_Test() override {} void SetUp() override { - this->wkb_bytes["point"] = { + this->wkb_bytes["point_little_endian"] = { 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x40 }; - this->wkb_bytes["point_big"] = { + this->wkb_bytes["point_big_endian"] = { 0x00, 0x00, 0x00, 0x00, 0x01, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, @@ -145,8 +145,8 @@ class WKB_Test : public ::testing::Test TEST_F(WKB_Test, wkb_point_test) // also tests endianness { - const auto geom_big = boost::get(wkb::read(this->wkb_bytes["point_big"])); - const auto geom_little = boost::get(wkb::read(this->wkb_bytes["point"])); + const auto geom_big = boost::get(wkb::read(this->wkb_bytes["point_big_endian"])); + const auto geom_little = boost::get(wkb::read(this->wkb_bytes["point_little_endian"])); EXPECT_NEAR(geom_big.get<0>(), geom_little.get<0>(), 1e-6); EXPECT_NEAR(geom_big.get<1>(), geom_little.get<1>(), 1e-6); EXPECT_NEAR(geom_big.get<0>(), 2.0, 1e-6); From 8cbd25372df854199013385fe7fca6e26f266d21 Mon Sep 17 00:00:00 2001 From: program-- Date: Wed, 12 Jul 2023 14:02:37 -0700 Subject: [PATCH 60/62] rev: white-list check for layer names changes per https://github.com/NOAA-OWP/ngen/pull/522#discussion_r1260116989 --- src/geopackage/read.cpp | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/geopackage/read.cpp b/src/geopackage/read.cpp index 7302a7d3b1..4cb7dad988 100644 --- a/src/geopackage/read.cpp +++ b/src/geopackage/read.cpp @@ -1,6 +1,17 @@ #include "GeoPackage.hpp" #include +#include + +void check_table_name(const std::string& table) +{ + if (boost::algorithm::starts_with(table, "sqlite_")) + throw std::runtime_error("table `" + table + "` is not queryable"); + + std::regex allowed("[^-A-Za-z0-9_ ]+"); + if (std::regex_match(table, allowed)) + throw std::runtime_error("table `" + table + "` contains invalid characters"); +} std::shared_ptr geopackage::read( const std::string& gpkg_path, @@ -8,6 +19,9 @@ std::shared_ptr geopackage::read( const std::vector& ids = {} ) { + // Check for malicious/invalid layer input + check_table_name(layer); + sqlite db(gpkg_path); // Check if layer exists From cf1d17f60453c696accc1503e9da50127018fa03 Mon Sep 17 00:00:00 2001 From: program-- Date: Thu, 13 Jul 2023 07:29:56 -0700 Subject: [PATCH 61/62] rev: update cmakelists; feature exhaustive case changes per https://github.com/NOAA-OWP/ngen/pull/522#discussion_r1258973951 and https://github.com/NOAA-OWP/ngen/pull/522#discussion_r1261903562 --- src/geopackage/CMakeLists.txt | 14 +++++++------- src/geopackage/feature.cpp | 4 +++- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/geopackage/CMakeLists.txt b/src/geopackage/CMakeLists.txt index 14a8f1324d..acc592bac1 100644 --- a/src/geopackage/CMakeLists.txt +++ b/src/geopackage/CMakeLists.txt @@ -12,13 +12,13 @@ if (NOT _cmp) set_source_files_properties(wkb.cpp PROPERTIES COMPILE_FLAGS -O0) endif() -add_library(geopackage STATIC geometry.cpp - properties.cpp - feature.cpp - read.cpp - wkb.cpp - sqlite/iterator.cpp - sqlite/database.cpp +add_library(geopackage geometry.cpp + properties.cpp + feature.cpp + read.cpp + wkb.cpp + sqlite/iterator.cpp + sqlite/database.cpp ) add_library(NGen::geopackage ALIAS geopackage) target_include_directories(geopackage PUBLIC ${PROJECT_SOURCE_DIR}/include/geopackage) diff --git a/src/geopackage/feature.cpp b/src/geopackage/feature.cpp index 76b95378de..8b51e4d495 100644 --- a/src/geopackage/feature.cpp +++ b/src/geopackage/feature.cpp @@ -67,12 +67,14 @@ geojson::Feature geopackage::build_feature( properties, bounding_box ); - default: + case geojson::FeatureType::GeometryCollection: return std::make_shared( std::vector{geometry}, id, properties, bounding_box ); + default: + throw std::runtime_error("invalid WKB feature type. Received: " + std::to_string(geometry.which() + 1)); } } From bd30664f5c129268ad5b09547ca60e565a278f27 Mon Sep 17 00:00:00 2001 From: program-- Date: Fri, 14 Jul 2023 14:22:18 -0700 Subject: [PATCH 62/62] rev: update wkb test changes per https://github.com/NOAA-OWP/ngen/pull/522#discussion_r1260142200 --- test/geopackage/WKB_Test.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/geopackage/WKB_Test.cpp b/test/geopackage/WKB_Test.cpp index 74bcdbf1fb..05a6a9b3cf 100644 --- a/test/geopackage/WKB_Test.cpp +++ b/test/geopackage/WKB_Test.cpp @@ -214,9 +214,9 @@ TEST_F(WKB_Test, wkb_polygon_with_holes_test) const std::vector> expected_inner = { {2, 2}, - {3, 2}, - {3, 3}, {2, 3}, + {3, 3}, + {3, 2}, {2, 2} };