diff --git a/.github/workflows/test-all.yaml b/.github/workflows/test-all.yaml index c0bc16281f6d..027c059bca43 100644 --- a/.github/workflows/test-all.yaml +++ b/.github/workflows/test-all.yaml @@ -106,7 +106,9 @@ jobs: uses: Swatinem/rust-cache@v2 with: save-if: ${{ github.ref == 'refs/heads/main' }} - - run: cargo tarpaulin --skip-clean --all-targets --out=Xml + - run: + cargo tarpaulin --skip-clean --all-targets --features=test-dbs + --out=Xml - name: Upload to codecov.io uses: codecov/codecov-action@v3 - name: Upload code coverage results diff --git a/.github/workflows/test-rust.yaml b/.github/workflows/test-rust.yaml index 0152e5e44f7d..4734e8c3e551 100644 --- a/.github/workflows/test-rust.yaml +++ b/.github/workflows/test-rust.yaml @@ -64,8 +64,8 @@ jobs: command: test args: "--no-run --locked --target=${{ inputs.target }} ${{ inputs.target - == 'x86_64-unknown-linux-gnu' && '--features=test-external-dbs' || - '' }}" + == 'x86_64-unknown-linux-gnu' && '--features=test-dbs-external' || + '--features=test-dbs' }}" - name: Run docker compose run: docker compose up -d working-directory: ./prql-compiler/tests/integration @@ -82,9 +82,9 @@ jobs: command: insta # Autoformatting doesn't make this clear to read, but this tertiary # expression states to only check these on ubuntu linux: - # - External DB integration tests — `--features=test-external-dbs`. + # - External DB integration tests — `--features=test-dbs-external`. # - Unreferenced snapshots - `--unreferenced=auto`. args: test --target=${{ inputs.target }} ${{ inputs.target== 'x86_64-unknown-linux-gnu' && '--unreferenced=auto - --features=test-external-dbs' || '' }} + --features=test-dbs-external' || '--features=test-dbs' }} diff --git a/Taskfile.yml b/Taskfile.yml index b92dbedd4ec8..f61dfffdfe75 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -210,11 +210,9 @@ tasks: # - "**/*.snap" cmds: # Only delete unreferenced snapshots on the default target — lots are - # excluded under wasm. Note that this will also over-delete on Windows. - - cargo insta test --accept --unreferenced=auto + # excluded under wasm. + - cargo insta test --accept --features=test-dbs --unreferenced=auto - cargo insta test --accept --target=wasm32-unknown-unknown - # We build the book too, because that acts as a test - - cd web/book && mdbook build test-rust-fast: desc: Test prql-compiler's unit tests. @@ -232,10 +230,6 @@ tasks: RUST_BACKTRACE: 1 sources: # I don't think we can specify this is a single glob, which would be nice. - # Ideally we want to exclude `.pending-snap` & `.snap.new` files so runs - # doesn't cause an infinite loop of other runs (though possibly mitigated - # by `--accept` and task not interrupting existing runs, so all files go - # to `.snap`s which match the generated output?). - "prql-compiler/**/*.rs" - "prql-compiler/**/*.snap" cmds: diff --git a/bacon.toml b/bacon.toml index 8aaf6c71963c..36e2fa2ebe33 100644 --- a/bacon.toml +++ b/bacon.toml @@ -4,10 +4,10 @@ default_job = "clippy" # PRQL additions [jobs.test] -command = ['cargo', 'insta', 'test', "--color=always", "--unreferenced=auto"] +command = ['cargo', 'insta', 'test', "--color=always", "--features=test-dbs", "--unreferenced=auto"] [jobs.test-accept] -command = ['cargo', 'insta', 'test', '--accept', "--color=always", "--unreferenced=auto"] +command = ['cargo', 'insta', 'test', '--accept', "--color=always", "--features=test-dbs", "--unreferenced=auto"] watch = ["*"] [jobs.test-accept-fast] diff --git a/prql-compiler/Cargo.toml b/prql-compiler/Cargo.toml index 5a20a9858c9a..a0f6cfc5031a 100644 --- a/prql-compiler/Cargo.toml +++ b/prql-compiler/Cargo.toml @@ -10,6 +10,13 @@ version.workspace = true metadata.msrv = "1.65.0" +[features] +default = [] +# Technically tokio could be limited to external tests, but its types are in +# signatures which would require lots of conditional compilation. +test-dbs = ["duckdb", "rusqlite", "tokio"] +test-dbs-external = ["chrono", "duckdb", "mysql", "pg_bigdecimal", "postgres", "rusqlite", "tiberius", "tokio", "tokio-util"] + [dependencies] anstream = {version = "0.3.2", features = ["auto"]} anyhow = {version = "1.0.57", features = ["backtrace"]} @@ -34,29 +41,31 @@ strum_macros = "0.25.0" # Chumsky's default features have issues when running in wasm (though we only # see it when compiling on MacOS), so we only include features when running # outside wasm. -[target.'cfg(not(target_family="wasm"))'.dependencies] -chumsky = "0.9.2" [target.'cfg(target_family="wasm")'.dependencies] chumsky = {version = "0.9.2", features = ["ahash", "std"], default-features = false} +[target.'cfg(not(target_family="wasm"))'.dependencies] +# For integration tests. These are not listed as dev-dependencies because +# dev-dependencies can't be optional. +chrono = {version = "0.4", optional = true, features = [], default-features = false} +duckdb = {version = "0.8.0", optional = true, features = ["bundled", "chrono"]} +mysql = {version = "24", optional = true} +pg_bigdecimal = {version = "0.1", optional = true} +postgres = {version = "0.19", optional = true} +rusqlite = {version = "0.29.0", optional = true, features = ["bundled", "csvtab"]} +tiberius = {version = "0.12", optional = true, features = ["sql-browser-tokio", "bigdecimal", "time"]} +tokio = {version = "1", optional = true, features = ["full"]} +tokio-util = {version = "0.7", optional = true, features = ["compat"]} +# Default chumsky features outside wasm. +chumsky = "0.9.2" + [dev-dependencies] cfg-if = "1.0" insta = {version = "1.29", features = ["colors", "glob", "yaml"]} similar-asserts = "1.4.2" -# For integration tests [target.'cfg(not(target_family="wasm"))'.dev-dependencies] -chrono = {version = "0.4", features = [], default-features = false} -criterion = "0.5.1" -csv = "1.2" -duckdb = {version = "0.8.0", features = ["bundled", "chrono"]} -mysql = "24" -pg_bigdecimal = "0.1" -postgres = "0.19" -rusqlite = {version = "0.29.0", features = ["bundled", "csvtab"]} -tiberius = {version = "0.12", features = ["sql-browser-tokio", "bigdecimal", "time"]} -tokio = {version = "1", features = ["full"]} -tokio-util = {version = "0.7", features = ["compat"]} +criterion = {version = "0.5.1"} [[bench]] harness = false @@ -115,6 +124,3 @@ exactly = 1 file = "../.github/actions/build-prqlc/action.yaml" replace = 'prefix-key: {{version}}' search = 'prefix-key: [\d.]+' - -[features] -test-external-dbs = [] diff --git a/prql-compiler/tests/integration/README.md b/prql-compiler/tests/integration/README.md index 7ec4756623e9..15c158de1cc4 100644 --- a/prql-compiler/tests/integration/README.md +++ b/prql-compiler/tests/integration/README.md @@ -45,7 +45,7 @@ cargo build. ### External DBs These will not run as a part of `cargo test`. Use -`cargo test --features=test-external-dbs` instead. Make sure to start docker +`cargo test --features=test-dbs-external` instead. Make sure to start docker compose before (see below). Currently Postgres, MySQL, SQL Server and ClickHouse are tested. @@ -66,7 +66,7 @@ Steps to run the tests: 2. Run the tests: ```sh - cargo test --features=test-external-dbs + cargo test --features=test-dbs-external ``` ## Test organization diff --git a/prql-compiler/tests/integration/connection.rs b/prql-compiler/tests/integration/connection.rs index 0883b948b384..c2c6db756f20 100644 --- a/prql-compiler/tests/integration/connection.rs +++ b/prql-compiler/tests/integration/connection.rs @@ -1,18 +1,6 @@ -use std::time::SystemTime; - -use anyhow::{bail, Result}; -use chrono::{DateTime, Utc}; -use mysql::prelude::Queryable; -use mysql::Value; -use pg_bigdecimal::PgNumeric; -use postgres::types::Type; +use anyhow::Result; use prql_compiler::sql::Dialect; -use tiberius::numeric::BigDecimal; -use tiberius::time::time::PrimitiveDateTime; -use tiberius::ColumnType; -use tokio::net::TcpStream; use tokio::runtime::Runtime; -use tokio_util::compat::Compat; pub type Row = Vec; @@ -99,126 +87,147 @@ impl DbProtocol for rusqlite::Connection { } } -impl DbProtocol for postgres::Client { - fn run_query(&mut self, sql: &str, _runtime: &Runtime) -> Result> { - let rows = self.query(sql, &[])?; - let mut vec = vec![]; - for row in rows { - let mut columns = vec![]; - for i in 0..row.len() { - let col = &(*row.columns())[i]; - let value = match col.type_() { - &Type::BOOL => (row.get::(i)).to_string(), - &Type::INT4 => match row.try_get::(i) { - Ok(v) => v.to_string(), - Err(_) => String::new(), - }, - &Type::INT8 => match row.try_get::(i) { - Ok(v) => v.to_string(), - Err(_) => String::new(), - }, - &Type::TEXT | &Type::VARCHAR | &Type::JSON | &Type::JSONB => { - match row.try_get::(i) { - Ok(v) => v, - // handle null - Err(_) => String::new(), - } - } - &Type::FLOAT4 => (row.get::(i)).to_string(), - &Type::FLOAT8 => (row.get::(i)).to_string(), - &Type::NUMERIC => row - .get::(i) - .n - .map(|d| d.normalized()) - .unwrap() - .to_string(), - &Type::TIMESTAMPTZ | &Type::TIMESTAMP => { - let time = row.get::(i); - let date_time: DateTime = time.into(); - date_time.to_rfc3339() - } - typ => bail!("postgres type {:?}", typ), - }; - columns.push(value); - } - vec.push(columns); - } - Ok(vec) - } -} +#[cfg(feature = "test-dbs-external")] +mod external_dbs { -impl DbProtocol for mysql::Pool { - fn run_query(&mut self, sql: &str, _runtime: &Runtime) -> Result> { - let mut conn = self.get_conn()?; - let rows: Vec = conn.query(sql)?; - let mut vec = vec![]; - for row in rows { - let mut columns = vec![]; - for v in row.unwrap() { - let value = match v { - Value::NULL => String::new(), - Value::Bytes(v) => String::from_utf8(v).unwrap_or_else(|_| "BLOB".to_string()), - Value::Int(v) => v.to_string(), - Value::UInt(v) => v.to_string(), - Value::Float(v) => v.to_string(), - Value::Double(v) => v.to_string(), - typ => bail!("mysql type {:?}", typ), - }; - columns.push(value); - } - vec.push(columns); - } - Ok(vec) - } -} + use anyhow::{bail, Result}; + use chrono::{DateTime, Utc}; + use mysql::prelude::Queryable; + use mysql::Value; + use pg_bigdecimal::PgNumeric; + use postgres::types::Type; + use std::time::SystemTime; + use tiberius::numeric::BigDecimal; + use tiberius::time::time::PrimitiveDateTime; + use tiberius::ColumnType; + use tokio::net::TcpStream; + use tokio_util::compat::Compat; + + use super::*; -impl DbProtocol for tiberius::Client> { - fn run_query(&mut self, sql: &str, runtime: &Runtime) -> Result> { - runtime.block_on(async { - let mut stream = self.query(sql, &[]).await?; + impl DbProtocol for postgres::Client { + fn run_query(&mut self, sql: &str, _runtime: &Runtime) -> Result> { + let rows = self.query(sql, &[])?; let mut vec = vec![]; - let cols_option = stream.columns().await?; - if cols_option.is_none() { - return Ok(vec); - } - let cols = cols_option.unwrap().to_vec(); - for row in stream.into_first_result().await.unwrap() { + for row in rows { let mut columns = vec![]; - for (i, col) in cols.iter().enumerate() { - let value = match col.column_type() { - ColumnType::Null => String::new(), - ColumnType::Bit => String::from(row.get::<&str, usize>(i).unwrap()), - ColumnType::Intn | ColumnType::Int4 => row - .get::(i) - .map_or_else(String::new, |i| i.to_string()), - ColumnType::Floatn => vec![ - row.try_get::(i).map(|o| o.map(f64::from)), - row.try_get::(i), - ] - .into_iter() - .find(|r| r.is_ok()) - .unwrap() - .unwrap() - .map_or_else(String::new, |i| i.to_string()), - ColumnType::Numericn | ColumnType::Decimaln => row - .get::(i) + for i in 0..row.len() { + let col = &(*row.columns())[i]; + let value = match col.type_() { + &Type::BOOL => (row.get::(i)).to_string(), + &Type::INT4 => match row.try_get::(i) { + Ok(v) => v.to_string(), + Err(_) => String::new(), + }, + &Type::INT8 => match row.try_get::(i) { + Ok(v) => v.to_string(), + Err(_) => String::new(), + }, + &Type::TEXT | &Type::VARCHAR | &Type::JSON | &Type::JSONB => { + match row.try_get::(i) { + Ok(v) => v, + // handle null + Err(_) => String::new(), + } + } + &Type::FLOAT4 => (row.get::(i)).to_string(), + &Type::FLOAT8 => (row.get::(i)).to_string(), + &Type::NUMERIC => row + .get::(i) + .n .map(|d| d.normalized()) .unwrap() .to_string(), - ColumnType::BigVarChar | ColumnType::NVarchar => { - String::from(row.get::<&str, usize>(i).unwrap_or("")) - } - ColumnType::Datetimen => { - row.get::(i).unwrap().to_string() + &Type::TIMESTAMPTZ | &Type::TIMESTAMP => { + let time = row.get::(i); + let date_time: DateTime = time.into(); + date_time.to_rfc3339() } - typ => bail!("mssql type {:?}", typ), + typ => bail!("postgres type {:?}", typ), }; columns.push(value); } vec.push(columns); } + Ok(vec) + } + } + impl DbProtocol for mysql::Pool { + fn run_query(&mut self, sql: &str, _runtime: &Runtime) -> Result> { + let mut conn = self.get_conn()?; + let rows: Vec = conn.query(sql)?; + let mut vec = vec![]; + for row in rows { + let mut columns = vec![]; + for v in row.unwrap() { + let value = match v { + Value::NULL => String::new(), + Value::Bytes(v) => { + String::from_utf8(v).unwrap_or_else(|_| "BLOB".to_string()) + } + Value::Int(v) => v.to_string(), + Value::UInt(v) => v.to_string(), + Value::Float(v) => v.to_string(), + Value::Double(v) => v.to_string(), + typ => bail!("mysql type {:?}", typ), + }; + columns.push(value); + } + vec.push(columns); + } Ok(vec) - }) + } + } + + impl DbProtocol for tiberius::Client> { + fn run_query(&mut self, sql: &str, runtime: &Runtime) -> Result> { + runtime.block_on(async { + let mut stream = self.query(sql, &[]).await?; + let mut vec = vec![]; + let cols_option = stream.columns().await?; + if cols_option.is_none() { + return Ok(vec); + } + let cols = cols_option.unwrap().to_vec(); + for row in stream.into_first_result().await.unwrap() { + let mut columns = vec![]; + for (i, col) in cols.iter().enumerate() { + let value = match col.column_type() { + ColumnType::Null => String::new(), + ColumnType::Bit => String::from(row.get::<&str, usize>(i).unwrap()), + ColumnType::Intn | ColumnType::Int4 => row + .get::(i) + .map_or_else(String::new, |i| i.to_string()), + ColumnType::Floatn => vec![ + row.try_get::(i).map(|o| o.map(f64::from)), + row.try_get::(i), + ] + .into_iter() + .find(|r| r.is_ok()) + .unwrap() + .unwrap() + .map_or_else(String::new, |i| i.to_string()), + ColumnType::Numericn | ColumnType::Decimaln => row + .get::(i) + .map(|d| d.normalized()) + .unwrap() + .to_string(), + ColumnType::BigVarChar | ColumnType::NVarchar => { + String::from(row.get::<&str, usize>(i).unwrap_or("")) + } + ColumnType::Datetimen => { + row.get::(i).unwrap().to_string() + } + typ => bail!("mssql type {:?}", typ), + }; + columns.push(value); + } + vec.push(columns); + } + + Ok(vec) + }) + } } } diff --git a/prql-compiler/tests/integration/main.rs b/prql-compiler/tests/integration/main.rs index 59985e9139c6..fda1f013c5e6 100644 --- a/prql-compiler/tests/integration/main.rs +++ b/prql-compiler/tests/integration/main.rs @@ -1,4 +1,5 @@ #![cfg(not(target_family = "wasm"))] +#![cfg(any(feature = "test-dbs", feature = "test-dbs-external"))] use std::{env, fs}; @@ -95,7 +96,7 @@ impl IntegrationTest for Dialect { protocol: Box::new(rusqlite::Connection::open_in_memory().unwrap()), }), - #[cfg(feature = "test-external-dbs")] + #[cfg(feature = "test-dbs-external")] Dialect::Postgres => Some(DbConnection { dialect: Dialect::Postgres, protocol: Box::new( @@ -106,21 +107,21 @@ impl IntegrationTest for Dialect { .unwrap(), ), }), - #[cfg(feature = "test-external-dbs")] + #[cfg(feature = "test-dbs-external")] Dialect::MySql => Some(DbConnection { dialect: Dialect::MySql, protocol: Box::new( mysql::Pool::new("mysql://root:root@localhost:3306/dummy").unwrap(), ), }), - #[cfg(feature = "test-external-dbs")] + #[cfg(feature = "test-dbs-external")] Dialect::ClickHouse => Some(DbConnection { dialect: Dialect::ClickHouse, protocol: Box::new( mysql::Pool::new("mysql://default:@localhost:9004/dummy").unwrap(), ), }), - #[cfg(feature = "test-external-dbs")] + #[cfg(feature = "test-dbs-external")] Dialect::MsSql => { use tiberius::{AuthMethod, Client, Config}; use tokio::net::TcpStream; diff --git a/web/book/src/contributing/development.md b/web/book/src/contributing/development.md index 3c11f21db065..aa65bee5525d 100644 --- a/web/book/src/contributing/development.md +++ b/web/book/src/contributing/development.md @@ -298,7 +298,8 @@ inconsistent in watchexec. Let's revert back if it gets solved. - **[Integration tests](https://github.com/PRQL/prql/blob/main/prql-compiler/tests/integration)** — we run tests with example queries against real databases to ensure we're - producing correct SQL. + producing correct SQL. These can be run locally with + `cargo test --features=test-dbs`. - **[GitHub Actions on every commit](https://github.com/PRQL/prql/blob/main/.github/workflows/pull-request.yaml)** — we run the tests described up to this point on every commit to a pull