From 06e2bfdd402706368b4e0858fee793441b8cadf9 Mon Sep 17 00:00:00 2001 From: Egor Date: Fri, 5 Jul 2024 11:21:06 +0200 Subject: [PATCH] editoast: app health and layers refactor --- .../editoast_models/src/db_connection_pool.rs | 10 +++- editoast/editoast_models/src/error.rs | 2 + editoast/editoast_models/src/lib.rs | 1 + .../layers/geo_json_and_data.rs} | 52 ++++++++++++++++--- editoast/src/modelsv2/layers/mod.rs | 5 ++ editoast/src/modelsv2/mod.rs | 1 + editoast/src/views/layers/mod.rs | 25 +++------ editoast/src/views/mod.rs | 29 ++++++++--- 8 files changed, 90 insertions(+), 35 deletions(-) rename editoast/src/{views/layers/mvt_utils.rs => modelsv2/layers/geo_json_and_data.rs} (92%) create mode 100644 editoast/src/modelsv2/layers/mod.rs diff --git a/editoast/editoast_models/src/db_connection_pool.rs b/editoast/editoast_models/src/db_connection_pool.rs index 9bc32b66d51..907463e37d0 100644 --- a/editoast/editoast_models/src/db_connection_pool.rs +++ b/editoast/editoast_models/src/db_connection_pool.rs @@ -1,18 +1,20 @@ +use std::sync::Arc; + +use diesel::sql_query; use diesel::ConnectionError; use diesel::ConnectionResult; use diesel_async::pooled_connection::deadpool::Object; use diesel_async::pooled_connection::deadpool::Pool; - use diesel_async::pooled_connection::AsyncDieselConnectionManager; use diesel_async::pooled_connection::ManagerConfig; use diesel_async::AsyncPgConnection; +use diesel_async::RunQueryDsl; use futures::future::BoxFuture; use futures::Future; use futures_util::FutureExt as _; use openssl::ssl::SslConnector; use openssl::ssl::SslMethod; use openssl::ssl::SslVerifyMode; -use std::sync::Arc; use url::Url; #[cfg(feature = "testing")] @@ -320,6 +322,10 @@ impl DbConnectionPoolV2 { } } +pub async fn ping_database(conn: &mut DbConnection) -> Result { + Ok(sql_query("SELECT 1").execute(conn).await?) +} + pub fn create_connection_pool( url: Url, max_size: usize, diff --git a/editoast/editoast_models/src/error.rs b/editoast/editoast_models/src/error.rs index 09762129793..f13c8adf005 100644 --- a/editoast/editoast_models/src/error.rs +++ b/editoast/editoast_models/src/error.rs @@ -12,4 +12,6 @@ pub enum EditoastModelsError { #[allow(dead_code)] #[error("Test connection not initialized")] TestConnection, + #[error("Timeout error")] + TimeoutError, } diff --git a/editoast/editoast_models/src/lib.rs b/editoast/editoast_models/src/lib.rs index cb07797edd9..d46f846dfff 100644 --- a/editoast/editoast_models/src/lib.rs +++ b/editoast/editoast_models/src/lib.rs @@ -4,6 +4,7 @@ mod db_connection_pool; mod error; pub use db_connection_pool::create_connection_pool; +pub use db_connection_pool::ping_database; pub use db_connection_pool::DbConnectionPoolV2; pub use error::EditoastModelsError; diff --git a/editoast/src/views/layers/mvt_utils.rs b/editoast/src/modelsv2/layers/geo_json_and_data.rs similarity index 92% rename from editoast/src/views/layers/mvt_utils.rs rename to editoast/src/modelsv2/layers/geo_json_and_data.rs index 9d980b36efb..7ddf2f915fc 100644 --- a/editoast/src/views/layers/mvt_utils.rs +++ b/editoast/src/modelsv2/layers/geo_json_and_data.rs @@ -1,5 +1,10 @@ +use diesel::sql_query; +use diesel::sql_types::Integer; use diesel::sql_types::Jsonb; use diesel::sql_types::Text; +use diesel_async::RunQueryDsl; +use editoast_models::DbConnection; +use editoast_models::EditoastModelsError; use geos::geojson::Geometry; use geos::geojson::Value as GeoJsonValue; use mvt::Feature; @@ -11,6 +16,7 @@ use serde::Deserialize; use serde::Serialize; use serde_json::Value as JsonValue; +use crate::map::Layer; use crate::map::View; #[derive(Clone, QueryableByName, Queryable, Debug, Serialize, Deserialize)] @@ -21,17 +27,39 @@ pub struct GeoJsonAndData { pub data: JsonValue, } -fn geometry_into_mvt_geom_type(geometry: &Geometry) -> GeomType { - match geometry.value { - GeoJsonValue::Point { .. } => GeomType::Point, - GeoJsonValue::MultiPoint { .. } => GeomType::Point, - GeoJsonValue::LineString { .. } => GeomType::Linestring, - GeoJsonValue::MultiLineString { .. } => GeomType::Linestring, - _ => panic!("geometry type unsupported by editoast tiling system"), +#[derive(Debug)] +pub struct GeoPoint { + x: u64, + y: u64, + z: u64, +} + +impl GeoPoint { + pub fn new(x: u64, y: u64, z: u64) -> Self { + Self { x, y, z } } } impl GeoJsonAndData { + pub async fn get_records( + conn: &mut DbConnection, + layer: &Layer, + view: &View, + infra: i64, + geo_point: &GeoPoint, + ) -> Result, EditoastModelsError> { + let geo_json_query = get_geo_json_sql_query(&layer.table_name, view); + let records = sql_query(geo_json_query) + .bind::(geo_point.z as i32) + .bind::(geo_point.x as i32) + .bind::(geo_point.y as i32) + .bind::(infra as i32) + .get_results::(conn) + .await?; + + Ok(records) + } + /// Converts GeoJsonAndData as mvt GeomData pub fn as_geom_data(&self) -> GeomData { let geo_json = serde_json::from_str::(&self.geo_json).unwrap(); @@ -65,6 +93,16 @@ impl GeoJsonAndData { } } +fn geometry_into_mvt_geom_type(geometry: &Geometry) -> GeomType { + match geometry.value { + GeoJsonValue::Point { .. } => GeomType::Point, + GeoJsonValue::MultiPoint { .. } => GeomType::Point, + GeoJsonValue::LineString { .. } => GeomType::Linestring, + GeoJsonValue::MultiLineString { .. } => GeomType::Linestring, + _ => panic!("geometry type unsupported by editoast tiling system"), + } +} + /// Adds tags to an MVT feature /// /// tags must be flattened as mvt tags are only one level depth diff --git a/editoast/src/modelsv2/layers/mod.rs b/editoast/src/modelsv2/layers/mod.rs new file mode 100644 index 00000000000..6b97e72f910 --- /dev/null +++ b/editoast/src/modelsv2/layers/mod.rs @@ -0,0 +1,5 @@ +mod geo_json_and_data; + +pub use geo_json_and_data::create_and_fill_mvt_tile; +pub use geo_json_and_data::GeoJsonAndData; +pub use geo_json_and_data::GeoPoint; diff --git a/editoast/src/modelsv2/mod.rs b/editoast/src/modelsv2/mod.rs index 6d29c507889..02f8576e6bd 100644 --- a/editoast/src/modelsv2/mod.rs +++ b/editoast/src/modelsv2/mod.rs @@ -7,6 +7,7 @@ pub mod infra; pub mod infra_objects; pub mod light_rolling_stock; // We allow unused until models is moved to a separate crate +pub mod layers; pub mod pagination; #[allow(unused)] pub mod prelude; diff --git a/editoast/src/views/layers/mod.rs b/editoast/src/views/layers/mod.rs index aa660a926aa..8eaa74760b4 100644 --- a/editoast/src/views/layers/mod.rs +++ b/editoast/src/views/layers/mod.rs @@ -1,5 +1,3 @@ -mod mvt_utils; - use std::collections::HashMap; use actix_web::get; @@ -8,13 +6,8 @@ use actix_web::web::Json; use actix_web::web::Path; use actix_web::web::Query; use actix_web::HttpResponse; -use diesel::sql_query; -use diesel::sql_types::Integer; -use diesel_async::RunQueryDsl; use editoast_derive::EditoastError; -use mvt_utils::create_and_fill_mvt_tile; -use mvt_utils::get_geo_json_sql_query; -use mvt_utils::GeoJsonAndData; +use editoast_models::DbConnectionPoolV2; use redis::AsyncCommands; use serde::Deserialize; use serde::Serialize; @@ -30,8 +23,10 @@ use crate::map::get_view_cache_prefix; use crate::map::Layer; use crate::map::MapLayers; use crate::map::Tile; +use crate::modelsv2::layers::create_and_fill_mvt_tile; +use crate::modelsv2::layers::GeoJsonAndData; +use crate::modelsv2::layers::GeoPoint; use crate::RedisClient; -use editoast_models::DbConnectionPoolV2; crate::routes! { "/layers" => { @@ -208,15 +203,9 @@ async fn cache_and_get_mvt_tile( .body(value)); } - let geo_json_query = get_geo_json_sql_query(&layer.table_name, view); - let mut conn = db_pool.get().await?; - let records = sql_query(geo_json_query) - .bind::(z as i32) - .bind::(x as i32) - .bind::(y as i32) - .bind::(infra as i32) - .get_results::(&mut conn) - .await?; + let conn = &mut db_pool.get().await?; + let records = + GeoJsonAndData::get_records(conn, layer, view, infra, &GeoPoint::new(z, x, y)).await?; let mvt_bytes: Vec = create_and_fill_mvt_tile(layer_slug, records) .to_bytes() diff --git a/editoast/src/views/mod.rs b/editoast/src/views/mod.rs index e03c633476d..c25dfa8f232 100644 --- a/editoast/src/views/mod.rs +++ b/editoast/src/views/mod.rs @@ -25,16 +25,21 @@ pub mod work_schedules; mod test_app; use std::ops::DerefMut as _; +use std::sync::Arc; +use std::time::Duration; pub use openapi::OpenApiRoot; use actix_web::get; use actix_web::web::Data; use actix_web::web::Json; -use diesel::sql_query; +use editoast_models::ping_database; +use editoast_models::DbConnectionPoolV2; +use editoast_models::EditoastModelsError; use redis::cmd; use serde_derive::Deserialize; use serde_derive::Serialize; +use tokio::time::timeout; use utoipa::ToSchema; use crate::client::get_app_version; @@ -49,7 +54,6 @@ use crate::infra_cache::operation; use crate::models; use crate::modelsv2; use crate::RedisClient; -use editoast_models::DbConnectionPoolV2; crate::routes! { (health, version, core_version), @@ -105,14 +109,23 @@ async fn health( db_pool: Data, redis_client: Data, ) -> Result<&'static str> { - use diesel_async::RunQueryDsl; - sql_query("SELECT 1") - .execute(db_pool.get().await?.deref_mut()) - .await?; + timeout( + Duration::from_millis(500), + check_health(db_pool.into_inner(), redis_client.into_inner()), + ) + .await + .map_err(|_| EditoastModelsError::TimeoutError)??; + Ok("ok") +} +async fn check_health( + db_pool: Arc, + redis_client: Arc, +) -> Result<()> { + ping_database(db_pool.get().await?.deref_mut()).await?; let mut conn = redis_client.get_connection().await?; - cmd("PING").query_async::<_, ()>(&mut conn).await.unwrap(); - Ok("ok") + cmd("PING").query_async::<_, ()>(&mut conn).await?; + Ok(()) } #[derive(ToSchema, Serialize, Deserialize)]