From 9e841e6a2fde1fdfa9ba421f465acbbfcbd0180e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Benjamin=20St=C3=BCrmer?= Date: Mon, 28 Aug 2023 23:32:45 +0200 Subject: [PATCH] clean up and extract database into crate --- Cargo.lock | 38 ++++++------- Cargo.toml | 56 ++++++++++++------- README.md | 31 +++++----- SECURITY.md | 2 + crates/accounts/Cargo.toml | 1 - crates/database/Cargo.toml | 20 +++++++ crates/database/README.md | 15 +++++ .../repository/mod.rs => database/src/lib.rs} | 3 +- crates/media/Cargo.toml | 3 - crates/media/src/api/router.rs | 2 +- crates/media/src/api/routes/get_media.rs | 2 +- crates/media/src/api/routes/post_media.rs | 3 +- crates/media/src/data/mod.rs | 22 -------- .../media/src/{repository => }/repository.rs | 5 +- crates/oauth_authentication/Cargo.toml | 20 +++---- crates/oauth_authorization_server/Cargo.toml | 22 ++++---- .../oauth_authorization_server/src/realm.rs | 12 ++-- src/lib.rs | 21 ++----- 18 files changed, 142 insertions(+), 136 deletions(-) create mode 100644 crates/database/Cargo.toml create mode 100644 crates/database/README.md rename crates/{media/src/repository/mod.rs => database/src/lib.rs} (87%) rename crates/media/src/{repository => }/repository.rs (93%) diff --git a/Cargo.lock b/Cargo.lock index ee17543..a1dfa12 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -55,7 +55,6 @@ name = "accounts" version = "0.6.0" dependencies = [ "axum", - "log", "mockall", "rstest", "sea-orm", @@ -796,6 +795,8 @@ dependencies = [ "anyhow", "axum", "common", + "core_extensions", + "database", "media", "oauth_authentication", "oauth_authorization_server", @@ -998,6 +999,14 @@ version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2e66c9d817f1720209181c316d28635c050fa304f9c79e47a520882661b7308" +[[package]] +name = "database" +version = "0.6.0" +dependencies = [ + "async-trait", + "sea-orm", +] + [[package]] name = "der" version = "0.7.8" @@ -1934,7 +1943,6 @@ dependencies = [ "axum", "common", "hyper", - "log", "mime", "mockall", "rstest", @@ -2194,7 +2202,7 @@ dependencies = [ "openidconnect", "rand", "serde_json", - "testdir 0.8.0", + "testdir", "thiserror", "tokio", "tower-http", @@ -2215,7 +2223,7 @@ dependencies = [ "rsa", "serde", "serde_json", - "testdir 0.7.3", + "testdir", "thiserror", "tokio", "tower-http", @@ -2868,9 +2876,9 @@ dependencies = [ [[package]] name = "rstest" -version = "0.18.1" +version = "0.18.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b96577ca10cb3eade7b337eb46520108a67ca2818a24d0b63f41fd62bc9651c" +checksum = "97eeab2f3c0a199bc4be135c36c924b6590b88c377d416494288c14f2db30199" dependencies = [ "futures", "futures-timer", @@ -2880,9 +2888,9 @@ dependencies = [ [[package]] name = "rstest_macros" -version = "0.18.1" +version = "0.18.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "225e674cf31712b8bb15fdbca3ec0c1b9d825c5a24407ff2b7e005fb6a29ba03" +checksum = "d428f8247852f894ee1be110b375111b586d4fa431f6c46e64ba5a0dcccbe605" dependencies = [ "cfg-if", "glob", @@ -3704,20 +3712,6 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3369f5ac52d5eb6ab48c6b4ffdc8efbcad6b89c765749064ba298f2c68a16a76" -[[package]] -name = "testdir" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a45fc921e7c4ad1aedb3484811514f3e5cd187886e0bbf1302c175f7578ef552" -dependencies = [ - "anyhow", - "backtrace", - "cargo_metadata", - "once_cell", - "sysinfo", - "whoami", -] - [[package]] name = "testdir" version = "0.8.0" diff --git a/Cargo.toml b/Cargo.toml index 0a1d152..0718fdf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,11 +25,12 @@ include = ["src/**/*", "LICENSE.md", "README.md", "CHANGELOG.md"] [workspace] members = [ "crates/accounts", - "crates/oauth_authorization_server", - "crates/oauth_authentication", + "crates/activity_pub", "crates/common", + "crates/database", "crates/media", - "crates/activity_pub", + "crates/oauth_authentication", + "crates/oauth_authorization_server", "crates/plugin_interface" ] @@ -39,17 +40,18 @@ members = [ accounts = { path = "./crates/accounts" } activity_pub = { path = "./crates/activity_pub" } common = { path = "./crates/common" } +database = { path = "./crates/database" } media = { path = "./crates/media" } -oauth_authorization_server = { path = "./crates/oauth_authorization_server" } oauth_authentication = { path = "./crates/oauth_authentication" } +oauth_authorization_server = { path = "./crates/oauth_authorization_server" } # 3rd party dependencies -abi_stable = { version = "0.11.1" } +abi_stable = "0.11.1" activitypub_federation = "0.4.6" -async-trait = { version = "0.1.73" } +async-trait = "0.1.73" axum = { version = "0.6.20", features = ["ws", "headers"] } -axum-test = { version = "12.1.0" } -anyhow = { version = "1.0.72" } +axum-test = "12.1.0" +anyhow = "1.0.72" core_extensions = { version = "1.5.2", default_features = false, features = ["std"] } chrono = { version = "0.4.26", features = ["serde"] } @@ -58,21 +60,33 @@ futures = "0.3.25" futures-channel = "0.3.25" futures-util = "0.3.25" -http = { version = "0.2.9" } -hyper = { version = "0.14", features = ["full"] } -mime = { version = "0.3" } -mockall = { version = "0.11.4" } +http = "0.2.9" +hyper = { version = "0.14.27", features = ["full"] } + +log = "0.4.19" +mime = "0.3" +mockall = "0.11.4" -rstest = { version = "0.18.1" } -rumqttc = { version = "0.22.0" } +openidconnect = { version = "3.2.0", features = ["accept-rfc3339-timestamps", "accept-string-booleans"] } + +pretty_assertions = "1.3.0" + +rand = "0.8.5" +rstest = "0.18.2" +rumqttc = "0.22.0" +rsa = "0.9.2" +reqwest = { version = "0.11", default-features = false, features = ["blocking", "json", "stream", "multipart"] } sea-orm = { version = "0.12.2", features = [ "runtime-tokio-rustls", "debug-print", "mock" ] } -serde = { version = "1.0.183" } +serde = "1.0.183" serde_json = { version = "1.0.104", features = ["raw_value"] } +serde_urlencoded = "0.7.1" smallvec = "1.8.0" -sqlx = { version = "0.7.1" } +sqlx = "0.7.1" -time = { version = "0.3.27" } +testdir = "0.8.0" +thiserror = "1.0.40" +time = "0.3.27" tokio = { version = "1.30.0", features = ["full"] } tokio-stream = { version = "0.1.11", features = ["net"] } tokio-util = { version = "0.7.4", features = ["rt"] } @@ -82,6 +96,7 @@ tracing = "0.1.37" tracing-subscriber = { version = "0.3.17", features = ["registry", "fmt", "std", "json"] } tracing-appender = "0.2.2" +url = { version = "2.4.0", features = ["serde"] } uuid = { version = "1.4.1", features = ["serde", "v4"] } [dependencies.photos_network_plugin] @@ -95,6 +110,7 @@ path = "./crates/plugin_interface" accounts.workspace = true activity_pub.workspace = true common.workspace = true +database.workspace = true media.workspace = true oauth_authentication.workspace = true oauth_authorization_server.workspace = true @@ -129,6 +145,6 @@ sqlx.workspace = true [dev-dependencies] -pretty_assertions = "1.3.0" -serde_urlencoded = "0.7.1" -reqwest = { version = "0.11", default-features = false, features = ["blocking", "json", "stream", "multipart"] } +pretty_assertions.workspace = true +serde_urlencoded.workspace = true +reqwest.workspace = true diff --git a/README.md b/README.md index de5dadd..d90780d 100644 --- a/README.md +++ b/README.md @@ -66,8 +66,22 @@ To run tests for all crates in this workspace, run: $ cargo test --workspace --all-targets ``` +### Visual Studio Code -### ๐Ÿ“œ Roadmap (MvP) +The fastest start into development can be archived by using [Visual Studio Code](https://code.visualstudio.com/) and [Docker](https://www.docker.com/get-started). + +1. Install [Docker](https://www.docker.com/get-started) +2. Install [Visual Studio Code](https://code.visualstudio.com/) +3. Install [Visual Studio Code Remote - Containers](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-containers) +4. Clone and Open this repository in Visual Studio Code +5. Click the "Reopen in Container" Dialog +6. Launch **Photos.network** from the `RUN` window. + +![VS Code with devcontainers](vscode.gif) + + + +## ๐Ÿ“œ Roadmap (MvP) - Authenticate via openID - Create a new media item @@ -83,21 +97,6 @@ $ cargo test --workspace --all-targets -### Visual Studio Code - -The fastest start into development can be archived by using [Visual Studio Code](https://code.visualstudio.com/) and [Docker](https://www.docker.com/get-started). - -1. Install [Docker](https://www.docker.com/get-started) -2. Install [Visual Studio Code](https://code.visualstudio.com/) -3. Install [Visual Studio Code Remote - Containers](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-containers) -4. Clone and Open this repository in Visual Studio Code -5. Click the "Reopen in Container" Dialog -6. Launch **Photos.network** from the `RUN` window. - -![VS Code with devcontainers](vscode.gif) - - - ## ๐Ÿ›๏ธ License ``` diff --git a/SECURITY.md b/SECURITY.md index 2eb2cef..c57d41a 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -15,6 +15,8 @@ If possible, encrypt your message with our PGP key; - **Public key**: https://keys.openpgp.org/search?q=security@photos.network +It is also possible to report it via the [Github security page](https://github.com/photos-network/core/security) + Please do not make vulnerabilities public without notifying us and giving us at least 4 weeks to respond. If you are going to write about Photos.networkโ€™s security, please get in touch, so we can make sure that all claims are correct. diff --git a/crates/accounts/Cargo.toml b/crates/accounts/Cargo.toml index 7df316d..ae05234 100644 --- a/crates/accounts/Cargo.toml +++ b/crates/accounts/Cargo.toml @@ -25,7 +25,6 @@ tower-http.workspace = true # persistency sea-orm = { workspace = true } -log = "0.4.19" # testing mockall = { workspace = true } diff --git a/crates/database/Cargo.toml b/crates/database/Cargo.toml new file mode 100644 index 0000000..4db2262 --- /dev/null +++ b/crates/database/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "database" +version.workspace = true +authors.workspace = true +description.workspace = true +homepage.workspace = true +documentation.workspace = true +repository.workspace = true +readme.workspace = true +license.workspace = true +edition.workspace = true + +[lib] +name = "database" +path = "src/lib.rs" +doctest = false + +[dependencies] +async-trait = { workspace = true } +sea-orm = { workspace = true, features = ["sqlx-postgres", "runtime-tokio-rustls", "mock"] } diff --git a/crates/database/README.md b/crates/database/README.md new file mode 100644 index 0000000..190137e --- /dev/null +++ b/crates/database/README.md @@ -0,0 +1,15 @@ +# database + +This crate provides an database abstraction used within [Photos.network](https://photos.network). + +## Framework choice + +The decision for [sea-orm](https://www.sea-ql.org/SeaORM/) has 3 main reasons compared to [Diesel](https://diesel.rs/) + +- supports async +- testable +- written in rust + +A big downsite of sea-orm: + +- performance diff --git a/crates/media/src/repository/mod.rs b/crates/database/src/lib.rs similarity index 87% rename from crates/media/src/repository/mod.rs rename to crates/database/src/lib.rs index 8a7da4a..e9d12b8 100644 --- a/crates/media/src/repository/mod.rs +++ b/crates/database/src/lib.rs @@ -15,4 +15,5 @@ * along with this program. If not, see . */ -pub mod repository; +//! This crate offers a database abstraction for [Photos.network](https://photos.network) core application. +//! diff --git a/crates/media/Cargo.toml b/crates/media/Cargo.toml index 053e40a..ed833f7 100644 --- a/crates/media/Cargo.toml +++ b/crates/media/Cargo.toml @@ -30,9 +30,6 @@ mime = { workspace = true } # persistency sea-orm = { workspace = true } -log = "0.4.19" - - uuid = { workspace = true, features = ["serde"] } diff --git a/crates/media/src/api/router.rs b/crates/media/src/api/router.rs index e0c4582..b2bb7c9 100644 --- a/crates/media/src/api/router.rs +++ b/crates/media/src/api/router.rs @@ -20,7 +20,7 @@ use std::sync::Arc; use axum::routing::{delete, get, patch, post}; use axum::Router; -use crate::repository::repository::{MediaRepository, MediaRepositoryState}; +use crate::repository::{MediaRepository, MediaRepositoryState}; use super::routes::delete_media_id::delete_media_id; use super::routes::get_albums::get_albums; diff --git a/crates/media/src/api/routes/get_media.rs b/crates/media/src/api/routes/get_media.rs index 88e7cb3..42551f6 100644 --- a/crates/media/src/api/routes/get_media.rs +++ b/crates/media/src/api/routes/get_media.rs @@ -7,7 +7,7 @@ use serde::{Deserialize, Serialize}; use std::result::Result; use tracing::error; -use crate::repository::repository::MediaRepositoryState; +use crate::repository::MediaRepositoryState; #[derive(Serialize, Deserialize)] pub(crate) struct MediaListQuery { diff --git a/crates/media/src/api/routes/post_media.rs b/crates/media/src/api/routes/post_media.rs index 05d4836..9155265 100644 --- a/crates/media/src/api/routes/post_media.rs +++ b/crates/media/src/api/routes/post_media.rs @@ -2,8 +2,7 @@ //! use axum::http::StatusCode; use common::model::auth::user::User; -use log::debug; -use tracing::error; +use tracing::{debug, error}; pub(crate) async fn post_media(user: User) -> std::result::Result { error!("POST /media user={}", user); diff --git a/crates/media/src/data/mod.rs b/crates/media/src/data/mod.rs index 315adc1..5529475 100644 --- a/crates/media/src/data/mod.rs +++ b/crates/media/src/data/mod.rs @@ -1,27 +1,5 @@ -use std::time::Duration; - -use log::LevelFilter; -use sea_orm::{ConnectOptions, Database, DatabaseConnection, DbErr}; - pub mod error; pub mod exif_info; pub mod file; pub mod location; pub mod media_item; - -pub async fn open_db_conn(db_url: String) -> std::result::Result { - let mut opt = ConnectOptions::new(db_url); - opt.max_connections(100) - .min_connections(5) - .connect_timeout(Duration::from_secs(8)) - .acquire_timeout(Duration::from_secs(8)) - .idle_timeout(Duration::from_secs(8)) - .max_lifetime(Duration::from_secs(8)) - .sqlx_logging(true) - .sqlx_logging_level(LevelFilter::Info); - // .set_schema_search_path("my_schema".into()); // Setting default PostgreSQL schema - - let db = Database::connect(opt).await?; - - Ok(db) -} diff --git a/crates/media/src/repository/repository.rs b/crates/media/src/repository.rs similarity index 93% rename from crates/media/src/repository/repository.rs rename to crates/media/src/repository.rs index 5e8698d..0ec90e6 100644 --- a/crates/media/src/repository/repository.rs +++ b/crates/media/src/repository.rs @@ -25,7 +25,6 @@ use uuid::Uuid; use crate::data::error::DataAccessError; use crate::data::media_item::MediaItem; -use crate::data::open_db_conn; pub struct MediaRepository { #[allow(dead_code)] @@ -72,9 +71,7 @@ impl MediaRepository { #[async_trait] impl MediaRepositoryTrait for MediaRepository { async fn new(db_url: &'static str) -> MediaRepository { - let db = open_db_conn(db_url.to_string()) - .await - .expect("Could not connect do database 'media'!"); + let db = DatabaseConnection::Disconnected; MediaRepository { db, db_url } } diff --git a/crates/oauth_authentication/Cargo.toml b/crates/oauth_authentication/Cargo.toml index 5278d72..909c0fd 100644 --- a/crates/oauth_authentication/Cargo.toml +++ b/crates/oauth_authentication/Cargo.toml @@ -17,20 +17,20 @@ doctest = false [dependencies] # OIDC interfaces -openidconnect = { version = "3.2.0", features = ["accept-rfc3339-timestamps", "accept-string-booleans"] } +openidconnect.workspace = true -anyhow = { workspace = true } +anyhow.workspace = true # error handling -thiserror = "1.0.40" +thiserror.workspace = true # URL parsing -url = { version = "2.4.0", features = ["serde"] } +url.workspace = true # OIDC Router -axum = { workspace = true } +axum.workspace = true tower-http.workspace = true @@ -54,8 +54,8 @@ tracing-subscriber.workspace = true [dev-dependencies] -testdir = { version="0.8.0" } -rand = { version = "0.8.5" } -tokio = { workspace = true } -serde_json = { workspace = true } -axum-test = { workspace = true } +testdir.workspace = true +rand.workspace = true +tokio.workspace = true +serde_json.workspace = true +axum-test.workspace = true diff --git a/crates/oauth_authorization_server/Cargo.toml b/crates/oauth_authorization_server/Cargo.toml index f5bedba..b58b26d 100644 --- a/crates/oauth_authorization_server/Cargo.toml +++ b/crates/oauth_authorization_server/Cargo.toml @@ -17,26 +17,26 @@ doctest = false [dependencies] # OIDC interfaces -openidconnect = { version = "3.2.0", features = ["accept-rfc3339-timestamps", "accept-string-booleans"] } +openidconnect.workspace = true # OIDC Router -axum = { workspace = true } +axum.workspace = true # json payload -serde = { workspace = true, features = ["derive"] } +serde.workspace = true # time related data in models chrono.workspace = true uuid.workspace = true # key signing and cryptographics -rsa = { version = "0.9.2" } +rsa.workspace = true # error handling -thiserror = "1.0.40" +thiserror.workspace = true # URL parsing -url = { version = "2.4.0", features = ["serde"] } +url.workspace = true # Rendering login form # dioxus = "0.3.2" @@ -48,8 +48,8 @@ tracing-subscriber.workspace = true [dev-dependencies] -testdir = { version="0.7.3" } -rand = { version = "0.8.5" } -tokio = { workspace = true } -serde_json = { workspace = true } -axum-test = { workspace = true } +testdir.workspace = true +rand.workspace = true +tokio.workspace = true +serde_json.workspace = true +axum-test.workspace = true diff --git a/crates/oauth_authorization_server/src/realm.rs b/crates/oauth_authorization_server/src/realm.rs index 3540302..cbf619f 100644 --- a/crates/oauth_authorization_server/src/realm.rs +++ b/crates/oauth_authorization_server/src/realm.rs @@ -57,11 +57,13 @@ impl Realm { .join(name) .with_extension("pem"), ) - .expect(&format!( - "key ({}) not found in directory ({})!", - name, - realm_keys_base_path.as_ref().display() - )); + .unwrap_or_else(|_| { + panic!( + "key ({}) not found in directory ({})!", + name, + realm_keys_base_path.as_ref().display() + ) + }); let mut realm_key_str = String::new(); realm_key_file .read_to_string(&mut realm_key_str) diff --git a/src/lib.rs b/src/lib.rs index d8dbc6e..70c822b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -27,7 +27,6 @@ //! See also the following crates //! * [Authentication](../oauth_authentication/index.html) -use core::time::Duration; use std::collections::HashMap; use std::fs; use std::net::SocketAddr; @@ -48,7 +47,6 @@ use oauth_authorization_server::state::ServerState; use oauth_authorization_server::AuthorizationServerManager; use photos_network_plugin::{PluginFactoryRef, PluginId}; use serde::{Deserialize, Serialize}; -use sqlx::postgres::PgPoolOptions; use std::path::Path; use tower_http::cors::CorsLayer; use tower_http::services::ServeDir; @@ -101,13 +99,13 @@ pub async fn start_server() -> Result<()> { let mut app_state = ApplicationState::new(config.clone()); let cfg = ServerConfig { - listen_addr: String::from(format!("{}", config.internal_url)), - domain: String::from(format!("{}", config.external_url)), + listen_addr: config.internal_url.to_owned(), + domain: config.external_url.to_owned(), use_ssl: true, realm_keys_base_path: Path::new("config").to_path_buf(), realms: vec![ConfigRealm { name: String::from("master"), - domain: Some(String::from(format!("{}", config.external_url))), + domain: Some(config.external_url.to_owned()), clients: vec![Client { id: String::from("mobile-app"), secret: None, @@ -117,17 +115,6 @@ pub async fn start_server() -> Result<()> { }; let server = ServerState::new(cfg)?; - // TODO: read from config - let db_connection_str = std::env::var("DATABASE_URL") - .unwrap_or_else(|_| "postgres://postgres:password@localhost".to_string()); - - let pool = PgPoolOptions::new() - .max_connections(5) - .acquire_timeout(Duration::from_secs(3)) - .connect(&db_connection_str) - .await - .expect("can't connect to database"); - let mut router = Router::new() // favicon .nest_service("/assets", ServeDir::new("src/api/static")) @@ -155,7 +142,7 @@ pub async fn start_server() -> Result<()> { .layer(DefaultBodyLimit::disable()) // add database connection pool - .with_state(pool) + //.with_state(pool) // TODO: share app state with routes