editoast: app health and layers refactor

Update front/public/locales/en/errors.json

Co-authored-by: Jean SIMARD <[email protected]>

editoast/editoast_models/src/db_connection_pool.rs

@@ -1,18 +1,20 @@
+use std::sync::Arc;
+
+use diesel::sql_query;
 use diesel::ConnectionError;
 use diesel::ConnectionResult;
 use diesel_async::pooled_connection::deadpool::Object;
 use diesel_async::pooled_connection::deadpool::Pool;
-
 use diesel_async::pooled_connection::AsyncDieselConnectionManager;
 use diesel_async::pooled_connection::ManagerConfig;
 use diesel_async::AsyncPgConnection;
+use diesel_async::RunQueryDsl;
 use futures::future::BoxFuture;
 use futures::Future;
 use futures_util::FutureExt as _;
 use openssl::ssl::SslConnector;
 use openssl::ssl::SslMethod;
 use openssl::ssl::SslVerifyMode;
-use std::sync::Arc;
 use url::Url;
 
 #[cfg(feature = "testing")]
@@ -328,6 +330,11 @@ impl DbConnectionPoolV2 {
     }
 }
 
+pub async fn ping_database(conn: &mut DbConnection) -> Result<(), EditoastModelsError> {
+    sql_query("SELECT 1").execute(conn).await?;
+    Ok(())
+}
+
 pub fn create_connection_pool(
     url: Url,
     max_size: usize,

editoast/editoast_models/src/lib.rs

@@ -4,6 +4,7 @@ mod db_connection_pool;
 mod error;
 
 pub use db_connection_pool::create_connection_pool;
+pub use db_connection_pool::ping_database;
 pub use db_connection_pool::DbConnectionPoolV2;
 pub use error::EditoastModelsError;
 

editoast/openapi.yaml

@@ -3794,6 +3794,63 @@ components:
       enum:
       - STANDARD
       - MARECO
+    EditoastAppHealthErrorDatabase:
+      type: object
+      required:
+      - type
+      - status
+      - message
+      properties:
+        context:
+          type: object
+        message:
+          type: string
+        status:
+          type: integer
+          enum:
+          - 400
+        type:
+          type: string
+          enum:
+          - editoast:app_health:Database
+    EditoastAppHealthErrorRedis:
+      type: object
+      required:
+      - type
+      - status
+      - message
+      properties:
+        context:
+          type: object
+        message:
+          type: string
+        status:
+          type: integer
+          enum:
+          - 400
+        type:
+          type: string
+          enum:
+          - editoast:app_health:Redis
+    EditoastAppHealthErrorTimeout:
+      type: object
+      required:
+      - type
+      - status
+      - message
+      properties:
+        context:
+          type: object
+        message:
+          type: string
+        status:
+          type: integer
+          enum:
+          - 400
+        type:
+          type: string
+          enum:
+          - editoast:app_health:Timeout
     EditoastAttachedErrorTrackNotFound:
       type: object
       required:
@@ -4248,6 +4305,9 @@ components:
           - editoast:electrical_profiles:NotFound
     EditoastError:
       oneOf:
+      - $ref: '#/components/schemas/EditoastAppHealthErrorDatabase'
+      - $ref: '#/components/schemas/EditoastAppHealthErrorRedis'
+      - $ref: '#/components/schemas/EditoastAppHealthErrorTimeout'
       - $ref: '#/components/schemas/EditoastAttachedErrorTrackNotFound'
       - $ref: '#/components/schemas/EditoastAutoFixesEditoastErrorConflictingFixesOnSameObject'
       - $ref: '#/components/schemas/EditoastAutoFixesEditoastErrorFixTrialFailure'

editoast/src/modelsv2/layers.rs

@@ -0,0 +1 @@
+pub mod geo_json_and_data;

editoast/src/views/layers/mvt_utils.rs → editoast/src/modelsv2/layers/geo_json_and_data.rs

@@ -1,5 +1,10 @@
+use diesel::sql_query;
+use diesel::sql_types::Integer;
 use diesel::sql_types::Jsonb;
 use diesel::sql_types::Text;
+use diesel_async::RunQueryDsl;
+use editoast_models::DbConnection;
+use editoast_models::EditoastModelsError;
 use geos::geojson::Geometry;
 use geos::geojson::Value as GeoJsonValue;
 use mvt::Feature;
@@ -11,6 +16,7 @@ use serde::Deserialize;
 use serde::Serialize;
 use serde_json::Value as JsonValue;
 
+use crate::map::Layer;
 use crate::map::View;
 
 #[derive(Clone, QueryableByName, Queryable, Debug, Serialize, Deserialize)]
@@ -21,17 +27,39 @@ pub struct GeoJsonAndData {
     pub data: JsonValue,
 }
 
-fn geometry_into_mvt_geom_type(geometry: &Geometry) -> GeomType {
-    match geometry.value {
-        GeoJsonValue::Point { .. } => GeomType::Point,
-        GeoJsonValue::MultiPoint { .. } => GeomType::Point,
-        GeoJsonValue::LineString { .. } => GeomType::Linestring,
-        GeoJsonValue::MultiLineString { .. } => GeomType::Linestring,
-        _ => panic!("geometry type unsupported by editoast tiling system"),
+#[derive(Debug)]
+pub struct GeoPoint {
+    x: u64,
+    y: u64,
+    z: u64,
+}
+
+impl GeoPoint {
+    pub fn new(x: u64, y: u64, z: u64) -> Self {
+        Self { x, y, z }
     }
 }
 
 impl GeoJsonAndData {
+    pub async fn get_records(
+        conn: &mut DbConnection,
+        layer: &Layer,
+        view: &View,
+        infra: i64,
+        geo_point: &GeoPoint,
+    ) -> Result<Vec<GeoJsonAndData>, EditoastModelsError> {
+        let geo_json_query = get_geo_json_sql_query(&layer.table_name, view);
+        let records = sql_query(geo_json_query)
+            .bind::<Integer, _>(geo_point.z as i32)
+            .bind::<Integer, _>(geo_point.x as i32)
+            .bind::<Integer, _>(geo_point.y as i32)
+            .bind::<Integer, _>(infra as i32)
+            .get_results::<GeoJsonAndData>(conn)
+            .await?;
+
+        Ok(records)
+    }
+
     /// Converts GeoJsonAndData as mvt GeomData
     pub fn as_geom_data(&self) -> GeomData {
         let geo_json = serde_json::from_str::<Geometry>(&self.geo_json).unwrap();
@@ -65,6 +93,16 @@ impl GeoJsonAndData {
     }
 }
 
+fn geometry_into_mvt_geom_type(geometry: &Geometry) -> GeomType {
+    match geometry.value {
+        GeoJsonValue::Point { .. } => GeomType::Point,
+        GeoJsonValue::MultiPoint { .. } => GeomType::Point,
+        GeoJsonValue::LineString { .. } => GeomType::Linestring,
+        GeoJsonValue::MultiLineString { .. } => GeomType::Linestring,
+        _ => panic!("geometry type unsupported by editoast tiling system"),
+    }
+}
+
 /// Adds tags to an MVT feature
 ///
 /// tags must be flattened as mvt tags are only one level depth

editoast/src/modelsv2/mod.rs

@@ -5,6 +5,7 @@ pub mod electrical_profiles;
 pub mod fixtures;
 pub mod infra;
 pub mod infra_objects;
+pub mod layers;
 pub mod light_rolling_stock;
 // We allow unused until models is moved to a separate crate
 pub mod pagination;

editoast/src/redis_utils.rs

@@ -1,10 +1,12 @@
-use crate::client::RedisConfig;
-use crate::error::Result;
+use std::fmt::Debug;
+
 use futures::future;
 use futures::FutureExt;
-use redis::aio::{ConnectionLike, ConnectionManager};
+use redis::aio::ConnectionLike;
+use redis::aio::ConnectionManager;
 use redis::cluster::ClusterClient;
 use redis::cluster_async::ClusterConnection;
+use redis::cmd;
 use redis::AsyncCommands;
 use redis::Client;
 use redis::ErrorKind;
@@ -14,7 +16,9 @@ use redis::RedisResult;
 use redis::ToRedisArgs;
 use serde::de::DeserializeOwned;
 use serde::Serialize;
-use std::fmt::Debug;
+
+use crate::client::RedisConfig;
+use crate::error::Result;
 
 pub enum RedisConnection {
     Cluster(ClusterConnection),
@@ -187,4 +191,10 @@ impl RedisClient {
             RedisClient::NoCache => Ok(RedisConnection::NoCache),
         }
     }
+
+    pub async fn ping_redis(&self) -> RedisResult<()> {
+        let mut conn = self.get_connection().await?;
+        cmd("PING").query_async::<_, ()>(&mut conn).await?;
+        Ok(())
+    }
 }

editoast/src/views/layers/mod.rs → editoast/src/views/layers.rs

@@ -1,5 +1,3 @@
-mod mvt_utils;
-
 use std::collections::HashMap;
 
 use actix_web::get;
@@ -8,13 +6,8 @@ use actix_web::web::Json;
 use actix_web::web::Path;
 use actix_web::web::Query;
 use actix_web::HttpResponse;
-use diesel::sql_query;
-use diesel::sql_types::Integer;
-use diesel_async::RunQueryDsl;
 use editoast_derive::EditoastError;
-use mvt_utils::create_and_fill_mvt_tile;
-use mvt_utils::get_geo_json_sql_query;
-use mvt_utils::GeoJsonAndData;
+use editoast_models::DbConnectionPoolV2;
 use redis::AsyncCommands;
 use serde::Deserialize;
 use serde::Serialize;
@@ -30,8 +23,10 @@ use crate::map::get_view_cache_prefix;
 use crate::map::Layer;
 use crate::map::MapLayers;
 use crate::map::Tile;
+use crate::modelsv2::layers::geo_json_and_data::create_and_fill_mvt_tile;
+use crate::modelsv2::layers::geo_json_and_data::GeoJsonAndData;
+use crate::modelsv2::layers::geo_json_and_data::GeoPoint;
 use crate::RedisClient;
-use editoast_models::DbConnectionPoolV2;
 
 crate::routes! {
      "/layers" => {
@@ -206,15 +201,9 @@ async fn cache_and_get_mvt_tile(
             .body(value));
     }
 
-    let geo_json_query = get_geo_json_sql_query(&layer.table_name, view);
-    let mut conn = db_pool.get().await?;
-    let records = sql_query(geo_json_query)
-        .bind::<Integer, _>(z as i32)
-        .bind::<Integer, _>(x as i32)
-        .bind::<Integer, _>(y as i32)
-        .bind::<Integer, _>(infra as i32)
-        .get_results::<GeoJsonAndData>(&mut conn)
-        .await?;
+    let conn = &mut db_pool.get().await?;
+    let records =
+        GeoJsonAndData::get_records(conn, layer, view, infra, &GeoPoint::new(z, x, y)).await?;
 
     let mvt_bytes: Vec<u8> = create_and_fill_mvt_tile(layer_slug, records)
         .to_bytes()

editoast/src/views/mod.rs

@@ -25,16 +25,22 @@ pub mod work_schedules;
 mod test_app;
 
 use std::ops::DerefMut as _;
+use std::sync::Arc;
+use std::time::Duration;
 
+use futures::TryFutureExt;
 pub use openapi::OpenApiRoot;
 
 use actix_web::get;
 use actix_web::web::Data;
 use actix_web::web::Json;
-use diesel::sql_query;
-use redis::cmd;
+use editoast_derive::EditoastError;
+use editoast_models::ping_database;
+use editoast_models::DbConnectionPoolV2;
 use serde_derive::Deserialize;
 use serde_derive::Serialize;
+use thiserror::Error;
+use tokio::time::timeout;
 use utoipa::ToSchema;
 
 use crate::client::get_app_version;
@@ -49,7 +55,6 @@ use crate::infra_cache::operation;
 use crate::models;
 use crate::modelsv2;
 use crate::RedisClient;
-use editoast_models::DbConnectionPoolV2;
 
 crate::routes! {
     (health, version, core_version),
@@ -95,6 +100,17 @@ editoast_common::schemas! {
     work_schedules::schemas(),
 }
 
+#[derive(Debug, Error, EditoastError)]
+#[editoast_error(base_id = "app_health")]
+pub enum AppHealthError {
+    #[error("Timeout error")]
+    Timeout,
+    #[error(transparent)]
+    Database(#[from] editoast_models::EditoastModelsError),
+    #[error(transparent)]
+    Redis(#[from] redis::RedisError),
+}
+
 #[utoipa::path(
     responses(
         (status = 200, description = "Check if Editoast is running correctly", body = String)
@@ -105,16 +121,27 @@ async fn health(
     db_pool: Data<DbConnectionPoolV2>,
     redis_client: Data<RedisClient>,
 ) -> Result<&'static str> {
-    use diesel_async::RunQueryDsl;
-    sql_query("SELECT 1")
-        .execute(db_pool.get().await?.deref_mut())
-        .await?;
-
-    let mut conn = redis_client.get_connection().await?;
-    cmd("PING").query_async::<_, ()>(&mut conn).await.unwrap();
+    timeout(
+        Duration::from_millis(500),
+        check_health(db_pool.into_inner(), redis_client.into_inner()),
+    )
+    .await
+    .map_err(|_| AppHealthError::Timeout)??;
     Ok("ok")
 }
 
+async fn check_health(
+    db_pool: Arc<DbConnectionPoolV2>,
+    redis_client: Arc<RedisClient>,
+) -> Result<()> {
+    let mut db_connection = db_pool.clone().get().await?;
+    tokio::try_join!(
+        ping_database(db_connection.deref_mut()).map_err(AppHealthError::Database),
+        redis_client.ping_redis().map_err(|e| e.into())
+    )?;
+    Ok(())
+}
+
 #[derive(ToSchema, Serialize, Deserialize)]
 pub struct Version {
     #[schema(required)] // Options are by default not required, but this one is

front/public/locales/en/errors.json

@@ -193,6 +193,11 @@
     },
     "work_schedule": {
       "NameAlreadyUsed": "A group of work schedules with '{{name}}' already exists"
+    },
+    "app_health": {
+      "Timeout": "Service has not responded in time",
+      "Database": "Database is in error",
+      "Redis": "Redis is in error"
     }
   }
 }