From aa03600ff54a1c98cd5f01c9e67281655bddf864 Mon Sep 17 00:00:00 2001 From: Egor Date: Thu, 11 Jul 2024 15:44:15 +0200 Subject: [PATCH] editoast: app health and layers refactor Update front/public/locales/en/errors.json Co-authored-by: Jean SIMARD --- .../editoast_models/src/db_connection_pool.rs | 11 +++- editoast/editoast_models/src/lib.rs | 1 + editoast/openapi.yaml | 60 +++++++++++++++++++ editoast/src/modelsv2/layers.rs | 1 + .../layers/geo_json_and_data.rs} | 52 +++++++++++++--- editoast/src/modelsv2/mod.rs | 1 + editoast/src/redis_utils.rs | 18 ++++-- .../src/views/{layers/mod.rs => layers.rs} | 25 +++----- editoast/src/views/mod.rs | 47 +++++++++++---- front/public/locales/en/errors.json | 5 ++ front/public/locales/fr/errors.json | 5 ++ 11 files changed, 185 insertions(+), 41 deletions(-) create mode 100644 editoast/src/modelsv2/layers.rs rename editoast/src/{views/layers/mvt_utils.rs => modelsv2/layers/geo_json_and_data.rs} (92%) rename editoast/src/views/{layers/mod.rs => layers.rs} (93%) diff --git a/editoast/editoast_models/src/db_connection_pool.rs b/editoast/editoast_models/src/db_connection_pool.rs index 55e2eb1acaf..16d3b649c31 100644 --- a/editoast/editoast_models/src/db_connection_pool.rs +++ b/editoast/editoast_models/src/db_connection_pool.rs @@ -1,18 +1,20 @@ +use std::sync::Arc; + +use diesel::sql_query; use diesel::ConnectionError; use diesel::ConnectionResult; use diesel_async::pooled_connection::deadpool::Object; use diesel_async::pooled_connection::deadpool::Pool; - use diesel_async::pooled_connection::AsyncDieselConnectionManager; use diesel_async::pooled_connection::ManagerConfig; use diesel_async::AsyncPgConnection; +use diesel_async::RunQueryDsl; use futures::future::BoxFuture; use futures::Future; use futures_util::FutureExt as _; use openssl::ssl::SslConnector; use openssl::ssl::SslMethod; use openssl::ssl::SslVerifyMode; -use std::sync::Arc; use url::Url; #[cfg(feature = "testing")] @@ -328,6 +330,11 @@ impl DbConnectionPoolV2 { } } +pub async fn ping_database(conn: &mut DbConnection) -> Result<(), EditoastModelsError> { + sql_query("SELECT 1").execute(conn).await?; + Ok(()) +} + pub fn create_connection_pool( url: Url, max_size: usize, diff --git a/editoast/editoast_models/src/lib.rs b/editoast/editoast_models/src/lib.rs index cb07797edd9..d46f846dfff 100644 --- a/editoast/editoast_models/src/lib.rs +++ b/editoast/editoast_models/src/lib.rs @@ -4,6 +4,7 @@ mod db_connection_pool; mod error; pub use db_connection_pool::create_connection_pool; +pub use db_connection_pool::ping_database; pub use db_connection_pool::DbConnectionPoolV2; pub use error::EditoastModelsError; diff --git a/editoast/openapi.yaml b/editoast/openapi.yaml index c7d40dcecc0..f03dcccf740 100644 --- a/editoast/openapi.yaml +++ b/editoast/openapi.yaml @@ -3794,6 +3794,63 @@ components: enum: - STANDARD - MARECO + EditoastAppHealthErrorDatabase: + type: object + required: + - type + - status + - message + properties: + context: + type: object + message: + type: string + status: + type: integer + enum: + - 400 + type: + type: string + enum: + - editoast:app_health:Database + EditoastAppHealthErrorRedis: + type: object + required: + - type + - status + - message + properties: + context: + type: object + message: + type: string + status: + type: integer + enum: + - 400 + type: + type: string + enum: + - editoast:app_health:Redis + EditoastAppHealthErrorTimeout: + type: object + required: + - type + - status + - message + properties: + context: + type: object + message: + type: string + status: + type: integer + enum: + - 400 + type: + type: string + enum: + - editoast:app_health:Timeout EditoastAttachedErrorTrackNotFound: type: object required: @@ -4248,6 +4305,9 @@ components: - editoast:electrical_profiles:NotFound EditoastError: oneOf: + - $ref: '#/components/schemas/EditoastAppHealthErrorDatabase' + - $ref: '#/components/schemas/EditoastAppHealthErrorRedis' + - $ref: '#/components/schemas/EditoastAppHealthErrorTimeout' - $ref: '#/components/schemas/EditoastAttachedErrorTrackNotFound' - $ref: '#/components/schemas/EditoastAutoFixesEditoastErrorConflictingFixesOnSameObject' - $ref: '#/components/schemas/EditoastAutoFixesEditoastErrorFixTrialFailure' diff --git a/editoast/src/modelsv2/layers.rs b/editoast/src/modelsv2/layers.rs new file mode 100644 index 00000000000..caa2d94dae3 --- /dev/null +++ b/editoast/src/modelsv2/layers.rs @@ -0,0 +1 @@ +pub mod geo_json_and_data; diff --git a/editoast/src/views/layers/mvt_utils.rs b/editoast/src/modelsv2/layers/geo_json_and_data.rs similarity index 92% rename from editoast/src/views/layers/mvt_utils.rs rename to editoast/src/modelsv2/layers/geo_json_and_data.rs index 9d980b36efb..7ddf2f915fc 100644 --- a/editoast/src/views/layers/mvt_utils.rs +++ b/editoast/src/modelsv2/layers/geo_json_and_data.rs @@ -1,5 +1,10 @@ +use diesel::sql_query; +use diesel::sql_types::Integer; use diesel::sql_types::Jsonb; use diesel::sql_types::Text; +use diesel_async::RunQueryDsl; +use editoast_models::DbConnection; +use editoast_models::EditoastModelsError; use geos::geojson::Geometry; use geos::geojson::Value as GeoJsonValue; use mvt::Feature; @@ -11,6 +16,7 @@ use serde::Deserialize; use serde::Serialize; use serde_json::Value as JsonValue; +use crate::map::Layer; use crate::map::View; #[derive(Clone, QueryableByName, Queryable, Debug, Serialize, Deserialize)] @@ -21,17 +27,39 @@ pub struct GeoJsonAndData { pub data: JsonValue, } -fn geometry_into_mvt_geom_type(geometry: &Geometry) -> GeomType { - match geometry.value { - GeoJsonValue::Point { .. } => GeomType::Point, - GeoJsonValue::MultiPoint { .. } => GeomType::Point, - GeoJsonValue::LineString { .. } => GeomType::Linestring, - GeoJsonValue::MultiLineString { .. } => GeomType::Linestring, - _ => panic!("geometry type unsupported by editoast tiling system"), +#[derive(Debug)] +pub struct GeoPoint { + x: u64, + y: u64, + z: u64, +} + +impl GeoPoint { + pub fn new(x: u64, y: u64, z: u64) -> Self { + Self { x, y, z } } } impl GeoJsonAndData { + pub async fn get_records( + conn: &mut DbConnection, + layer: &Layer, + view: &View, + infra: i64, + geo_point: &GeoPoint, + ) -> Result, EditoastModelsError> { + let geo_json_query = get_geo_json_sql_query(&layer.table_name, view); + let records = sql_query(geo_json_query) + .bind::(geo_point.z as i32) + .bind::(geo_point.x as i32) + .bind::(geo_point.y as i32) + .bind::(infra as i32) + .get_results::(conn) + .await?; + + Ok(records) + } + /// Converts GeoJsonAndData as mvt GeomData pub fn as_geom_data(&self) -> GeomData { let geo_json = serde_json::from_str::(&self.geo_json).unwrap(); @@ -65,6 +93,16 @@ impl GeoJsonAndData { } } +fn geometry_into_mvt_geom_type(geometry: &Geometry) -> GeomType { + match geometry.value { + GeoJsonValue::Point { .. } => GeomType::Point, + GeoJsonValue::MultiPoint { .. } => GeomType::Point, + GeoJsonValue::LineString { .. } => GeomType::Linestring, + GeoJsonValue::MultiLineString { .. } => GeomType::Linestring, + _ => panic!("geometry type unsupported by editoast tiling system"), + } +} + /// Adds tags to an MVT feature /// /// tags must be flattened as mvt tags are only one level depth diff --git a/editoast/src/modelsv2/mod.rs b/editoast/src/modelsv2/mod.rs index 6d29c507889..74384efc961 100644 --- a/editoast/src/modelsv2/mod.rs +++ b/editoast/src/modelsv2/mod.rs @@ -5,6 +5,7 @@ pub mod electrical_profiles; pub mod fixtures; pub mod infra; pub mod infra_objects; +pub mod layers; pub mod light_rolling_stock; // We allow unused until models is moved to a separate crate pub mod pagination; diff --git a/editoast/src/redis_utils.rs b/editoast/src/redis_utils.rs index 753057bf1c7..332ee0e4369 100644 --- a/editoast/src/redis_utils.rs +++ b/editoast/src/redis_utils.rs @@ -1,10 +1,12 @@ -use crate::client::RedisConfig; -use crate::error::Result; +use std::fmt::Debug; + use futures::future; use futures::FutureExt; -use redis::aio::{ConnectionLike, ConnectionManager}; +use redis::aio::ConnectionLike; +use redis::aio::ConnectionManager; use redis::cluster::ClusterClient; use redis::cluster_async::ClusterConnection; +use redis::cmd; use redis::AsyncCommands; use redis::Client; use redis::ErrorKind; @@ -14,7 +16,9 @@ use redis::RedisResult; use redis::ToRedisArgs; use serde::de::DeserializeOwned; use serde::Serialize; -use std::fmt::Debug; + +use crate::client::RedisConfig; +use crate::error::Result; pub enum RedisConnection { Cluster(ClusterConnection), @@ -187,4 +191,10 @@ impl RedisClient { RedisClient::NoCache => Ok(RedisConnection::NoCache), } } + + pub async fn ping_redis(&self) -> RedisResult<()> { + let mut conn = self.get_connection().await?; + cmd("PING").query_async::<_, ()>(&mut conn).await?; + Ok(()) + } } diff --git a/editoast/src/views/layers/mod.rs b/editoast/src/views/layers.rs similarity index 93% rename from editoast/src/views/layers/mod.rs rename to editoast/src/views/layers.rs index 6fa2be18340..ef70c2b5054 100644 --- a/editoast/src/views/layers/mod.rs +++ b/editoast/src/views/layers.rs @@ -1,5 +1,3 @@ -mod mvt_utils; - use std::collections::HashMap; use actix_web::get; @@ -8,13 +6,8 @@ use actix_web::web::Json; use actix_web::web::Path; use actix_web::web::Query; use actix_web::HttpResponse; -use diesel::sql_query; -use diesel::sql_types::Integer; -use diesel_async::RunQueryDsl; use editoast_derive::EditoastError; -use mvt_utils::create_and_fill_mvt_tile; -use mvt_utils::get_geo_json_sql_query; -use mvt_utils::GeoJsonAndData; +use editoast_models::DbConnectionPoolV2; use redis::AsyncCommands; use serde::Deserialize; use serde::Serialize; @@ -30,8 +23,10 @@ use crate::map::get_view_cache_prefix; use crate::map::Layer; use crate::map::MapLayers; use crate::map::Tile; +use crate::modelsv2::layers::geo_json_and_data::create_and_fill_mvt_tile; +use crate::modelsv2::layers::geo_json_and_data::GeoJsonAndData; +use crate::modelsv2::layers::geo_json_and_data::GeoPoint; use crate::RedisClient; -use editoast_models::DbConnectionPoolV2; crate::routes! { "/layers" => { @@ -206,15 +201,9 @@ async fn cache_and_get_mvt_tile( .body(value)); } - let geo_json_query = get_geo_json_sql_query(&layer.table_name, view); - let mut conn = db_pool.get().await?; - let records = sql_query(geo_json_query) - .bind::(z as i32) - .bind::(x as i32) - .bind::(y as i32) - .bind::(infra as i32) - .get_results::(&mut conn) - .await?; + let conn = &mut db_pool.get().await?; + let records = + GeoJsonAndData::get_records(conn, layer, view, infra, &GeoPoint::new(z, x, y)).await?; let mvt_bytes: Vec = create_and_fill_mvt_tile(layer_slug, records) .to_bytes() diff --git a/editoast/src/views/mod.rs b/editoast/src/views/mod.rs index e03c633476d..bf507c41e69 100644 --- a/editoast/src/views/mod.rs +++ b/editoast/src/views/mod.rs @@ -25,16 +25,22 @@ pub mod work_schedules; mod test_app; use std::ops::DerefMut as _; +use std::sync::Arc; +use std::time::Duration; +use futures::TryFutureExt; pub use openapi::OpenApiRoot; use actix_web::get; use actix_web::web::Data; use actix_web::web::Json; -use diesel::sql_query; -use redis::cmd; +use editoast_derive::EditoastError; +use editoast_models::ping_database; +use editoast_models::DbConnectionPoolV2; use serde_derive::Deserialize; use serde_derive::Serialize; +use thiserror::Error; +use tokio::time::timeout; use utoipa::ToSchema; use crate::client::get_app_version; @@ -49,7 +55,6 @@ use crate::infra_cache::operation; use crate::models; use crate::modelsv2; use crate::RedisClient; -use editoast_models::DbConnectionPoolV2; crate::routes! { (health, version, core_version), @@ -95,6 +100,17 @@ editoast_common::schemas! { work_schedules::schemas(), } +#[derive(Debug, Error, EditoastError)] +#[editoast_error(base_id = "app_health")] +pub enum AppHealthError { + #[error("Timeout error")] + Timeout, + #[error(transparent)] + Database(#[from] editoast_models::EditoastModelsError), + #[error(transparent)] + Redis(#[from] redis::RedisError), +} + #[utoipa::path( responses( (status = 200, description = "Check if Editoast is running correctly", body = String) @@ -105,16 +121,27 @@ async fn health( db_pool: Data, redis_client: Data, ) -> Result<&'static str> { - use diesel_async::RunQueryDsl; - sql_query("SELECT 1") - .execute(db_pool.get().await?.deref_mut()) - .await?; - - let mut conn = redis_client.get_connection().await?; - cmd("PING").query_async::<_, ()>(&mut conn).await.unwrap(); + timeout( + Duration::from_millis(500), + check_health(db_pool.into_inner(), redis_client.into_inner()), + ) + .await + .map_err(|_| AppHealthError::Timeout)??; Ok("ok") } +async fn check_health( + db_pool: Arc, + redis_client: Arc, +) -> Result<()> { + let mut db_connection = db_pool.clone().get().await?; + tokio::try_join!( + ping_database(db_connection.deref_mut()).map_err(AppHealthError::Database), + redis_client.ping_redis().map_err(|e| e.into()) + )?; + Ok(()) +} + #[derive(ToSchema, Serialize, Deserialize)] pub struct Version { #[schema(required)] // Options are by default not required, but this one is diff --git a/front/public/locales/en/errors.json b/front/public/locales/en/errors.json index 48d05934e23..664d19fe104 100644 --- a/front/public/locales/en/errors.json +++ b/front/public/locales/en/errors.json @@ -193,6 +193,11 @@ }, "work_schedule": { "NameAlreadyUsed": "A group of work schedules with '{{name}}' already exists" + }, + "app_health": { + "Timeout": "Service has not responded in time", + "Database": "Database is in error", + "Redis": "Redis is in error" } } } diff --git a/front/public/locales/fr/errors.json b/front/public/locales/fr/errors.json index 7c33123b3b6..64c5566f5d6 100644 --- a/front/public/locales/fr/errors.json +++ b/front/public/locales/fr/errors.json @@ -195,6 +195,11 @@ }, "work_schedule": { "NameAlreadyUsed": "Un groupe de planches travaux avec le nom '{{name}}' existe déjà" + }, + "app_health": { + "Timeout": "Le serveur n'a pas répondu à temps", + "Database": "Erreur de base de données", + "Redis": "Erreur de Redis" } } }