Oracle bug fix

.github/workflows/R_CMD_check_Hades.yaml

@@ -48,9 +48,20 @@ jobs:
       CDM5_REDSHIFT_PASSWORD: ${{ secrets.CDM5_REDSHIFT_PASSWORD }}
       CDM5_REDSHIFT_SERVER: ${{ secrets.CDM5_REDSHIFT_SERVER }}
       CDM5_REDSHIFT_USER: ${{ secrets.CDM5_REDSHIFT_USER }}
+      CDM_SNOWFLAKE_CDM53_SCHEMA: ${{ secrets.CDM_SNOWFLAKE_CDM53_SCHEMA }}
+      CDM_SNOWFLAKE_OHDSI_SCHEMA: ${{ secrets.CDM_SNOWFLAKE_OHDSI_SCHEMA }}
+      CDM_SNOWFLAKE_PASSWORD: ${{ secrets.CDM_SNOWFLAKE_PASSWORD }}
+      CDM_SNOWFLAKE_CONNECTION_STRING: ${{ secrets.CDM_SNOWFLAKE_CONNECTION_STRING }}
+      CDM_SNOWFLAKE_USER: ${{ secrets.CDM_SNOWFLAKE_USER }}
       CDM5_SPARK_USER: ${{ secrets.CDM5_SPARK_USER }}
       CDM5_SPARK_PASSWORD: ${{ secrets.CDM5_SPARK_PASSWORD }}
       CDM5_SPARK_CONNECTION_STRING: ${{ secrets.CDM5_SPARK_CONNECTION_STRING }}
+      CDM5_SPARK_CDM_SCHEMA: ${{ secrets.CDM5_SPARK_CDM_SCHEMA }}
+      CDM5_SPARK_OHDSI_SCHEMA: ${{ secrets.CDM5_SPARK_OHDSI_SCHEMA }}
+      CDM_BIG_QUERY_CONNECTION_STRING: ${{ secrets.CDM_BIG_QUERY_CONNECTION_STRING }}
+      CDM_BIG_QUERY_KEY_FILE: ${{ secrets.CDM_BIG_QUERY_KEY_FILE }}
+      CDM_BIG_QUERY_CDM_SCHEMA: ${{ secrets.CDM_BIG_QUERY_CDM_SCHEMA }}
+      CDM_BIG_QUERY_OHDSI_SCHEMA: ${{ secrets.CDM_BIG_QUERY_OHDSI_SCHEMA }}
 
     steps:
       - uses: actions/checkout@v3

R/Database.R

@@ -60,7 +60,7 @@ insertAndromedaToDatabase <- function(
     databaseSchema,
     tableName,
     andromedaObject,
-    tempEmulationSchema,
+    tempEmulationSchema = getOption("sqlRenderTempEmulationSchema"),
     bulkLoad = T,
     tablePrefix = "c_",
     minCellCount = 0,

R/RunCharacterization.R

@@ -132,7 +132,7 @@ runCharacterizationAnalyses <- function(
     targetTable,
     outcomeDatabaseSchema,
     outcomeTable,
-    tempEmulationSchema = NULL,
+    tempEmulationSchema = getOption("sqlRenderTempEmulationSchema"),
     cdmDatabaseSchema,
     characterizationSettings,
     saveDirectory,

inst/sql/sql_server/createTargetOutcomeCombinations.sql

@@ -1,12 +1,12 @@
 --need to know indication/target/outcome tuples
-drop table if exists #targets_agg_all;
+IF OBJECT_ID('tempdb..#targets_agg_all', 'U') IS NOT NULL DROP TABLE #targets_agg_all;
 select * into #targets_agg_all
 from @target_database_schema.@target_table
 where cohort_definition_id in
 (@target_ids);
 
 -- first T with > minPrioObs
-drop table if exists #targets_agg;
+IF OBJECT_ID('tempdb..#targets_agg', 'U') IS NOT NULL DROP TABLE #targets_agg;
 select * into #targets_agg
 from
 (select *,
@@ -22,24 +22,25 @@ and temp_t.cohort_start_date <= op.observation_period_end_date
 where temp_t.rn = 1
 and datediff(day, op.observation_period_start_date, temp_t.cohort_start_date) >= @min_prior_observation;
 
-drop table if exists #outcomes_agg;
+--drop table if exists #outcomes_agg;
+IF OBJECT_ID('tempdb..#outcomes_agg', 'U') IS NOT NULL DROP TABLE #outcomes_agg;
 select * into #outcomes_agg
 from @outcome_database_schema.@outcome_table
 where cohort_definition_id in
 (@outcome_ids);
 
 -- first outcomes
-drop table if exists #outcomes_agg_first;
+IF OBJECT_ID('tempdb..#outcomes_agg_first', 'U') IS NOT NULL DROP TABLE #outcomes_agg_first;
 select * into #outcomes_agg_first
 from (select *,
 row_number() over(partition by subject_id, cohort_definition_id order by cohort_start_date asc) as rn
 from #outcomes_agg
-) as o
+) o
 where o.rn = 1
 ;
 
 -- create all the cohort details
-drop table if exists #cohort_details;
+IF OBJECT_ID('tempdb..#cohort_details', 'U') IS NOT NULL DROP TABLE #cohort_details;
 
 select *,
 ROW_NUMBER() OVER (ORDER BY cohort_type, target_cohort_id, outcome_cohort_id) as cohort_definition_id
@@ -52,9 +53,9 @@ t.cohort_definition_id as target_cohort_id,
 o.cohort_definition_id as outcome_cohort_id,
 'TnO' as cohort_type
 from
-(select distinct cohort_definition_id from #targets_agg) as t
+(select distinct cohort_definition_id from #targets_agg) t
 CROSS JOIN
-(select distinct cohort_definition_id from #outcomes_agg) as o
+(select distinct cohort_definition_id from #outcomes_agg) o
 
 union
 
@@ -63,9 +64,9 @@ t.cohort_definition_id as target_cohort_id,
 o.cohort_definition_id as outcome_cohort_id,
 'OnT' as cohort_type
 from
-(select distinct cohort_definition_id from #targets_agg) as t
+(select distinct cohort_definition_id from #targets_agg) t
 CROSS JOIN
-(select distinct cohort_definition_id from #outcomes_agg) as o
+(select distinct cohort_definition_id from #outcomes_agg) o
 
 union
 
@@ -74,9 +75,9 @@ union
 --o.cohort_definition_id as outcome_cohort_id,
 --'TnOc' as cohort_type
 --from
---(select distinct cohort_definition_id from #targets_agg) as t
+--(select distinct cohort_definition_id from #targets_agg) t
 --CROSS JOIN
---(select distinct cohort_definition_id from #outcomes_agg) as o
+--(select distinct cohort_definition_id from #outcomes_agg) o
 
 --union
 
@@ -85,9 +86,9 @@ t.cohort_definition_id as target_cohort_id,
 o.cohort_definition_id as outcome_cohort_id,
 'TnfirstO' as cohort_type
 from
-(select distinct cohort_definition_id from #targets_agg) as t
+(select distinct cohort_definition_id from #targets_agg) t
 CROSS JOIN
-(select distinct cohort_definition_id from #outcomes_agg) as o
+(select distinct cohort_definition_id from #outcomes_agg) o
 
 union
 
@@ -96,9 +97,9 @@ t.cohort_definition_id as target_cohort_id,
 o.cohort_definition_id as outcome_cohort_id,
 'firstOnT' as cohort_type
 from
-(select distinct cohort_definition_id from #targets_agg) as t
+(select distinct cohort_definition_id from #targets_agg) t
 CROSS JOIN
-(select distinct cohort_definition_id from #outcomes_agg) as o
+(select distinct cohort_definition_id from #outcomes_agg) o
 
 union
 
@@ -107,48 +108,49 @@ union
 --o.cohort_definition_id as outcome_cohort_id,
 --'TnfirstOc' as cohort_type
 --from
---(select distinct cohort_definition_id from #targets_agg) as t
+--(select distinct cohort_definition_id from #targets_agg) t
 --CROSS JOIN
---(select distinct cohort_definition_id from #outcomes_agg) as o
+--(select distinct cohort_definition_id from #outcomes_agg) o
 
 --union
 
 select distinct
 t.cohort_definition_id as target_cohort_id,
 0 as outcome_cohort_id,
 'T' as cohort_type
-from (select distinct cohort_definition_id from #targets_agg) as t
+from (select distinct cohort_definition_id from #targets_agg) t
 
 union
 
 select distinct
 t.cohort_definition_id as target_cohort_id,
 0 as outcome_cohort_id,
 'allT' as cohort_type
-from (select distinct cohort_definition_id from #targets_agg) as t
+from (select distinct cohort_definition_id from #targets_agg) t
 
 union
 
 select distinct
 0 as target_cohort_id,
 o.cohort_definition_id as outcome_cohort_id,
 'O' as cohort_type
-from (select distinct cohort_definition_id from #outcomes_agg) as o
+from (select distinct cohort_definition_id from #outcomes_agg) o
 
 union
 
 select distinct
 0 as target_cohort_id,
 o.cohort_definition_id as outcome_cohort_id,
 'firstO' as cohort_type
-from (select distinct cohort_definition_id from #outcomes_agg) as o
+from (select distinct cohort_definition_id from #outcomes_agg) o
 
 
 ) temp;
 
 
 -- 1) get all the people with the outcome in TAR
-drop table if exists #target_with_outcome;
+--drop table if exists #target_with_outcome;
+IF OBJECT_ID('tempdb..#target_with_outcome', 'U') IS NOT NULL DROP TABLE #target_with_outcome;
 
 -- TnO
 select
@@ -170,7 +172,8 @@ and
 o.cohort_start_date >= dateadd(day, @tar_start, t.@tar_start_anchor);
 
 -- TnfirstO
-drop table if exists #target_outcome_f;
+--drop table if exists #target_outcome_f;
+IF OBJECT_ID('tempdb..#target_outcome_f', 'U') IS NOT NULL DROP TABLE #target_outcome_f;
 select
 t.subject_id,
 t.cohort_start_date,
@@ -191,7 +194,8 @@ o.cohort_start_date >= dateadd(day, @tar_start, t.@tar_start_anchor);
 
 
 -- 2) get all the people without the outcome in TAR
---drop table if exists #target_nooutcome;
+----drop table if exists #target_nooutcome;
+--IF OBJECT_ID('tempdb..#target_nooutcome', 'U') IS NOT NULL DROP TABLE #target_nooutcome;
 --select
 --t.subject_id,
 --t.cohort_start_date,
@@ -208,7 +212,8 @@ o.cohort_start_date >= dateadd(day, @tar_start, t.@tar_start_anchor);
 --and o.cohort_definition_id = two.outcome_cohort_id
 --where two.subject_id IS NULL;
 
---drop table if exists #target_noout_f;
+----drop table if exists #target_noout_f;
+--IF OBJECT_ID('tempdb..#target_noout_f', 'U') IS NOT NULL DROP TABLE #target_noout_f;
 --select
 --t.subject_id,
 --t.cohort_start_date,
@@ -331,7 +336,7 @@ t.subject_id,
 t.cohort_start_date,
 t.cohort_end_date,
 cd.cohort_definition_id
-from #targets_agg as t
+from #targets_agg t
 INNER JOIN #cohort_details cd
 on cd.target_cohort_id = t.cohort_definition_id
 and cd.cohort_type = 'T'
@@ -343,7 +348,7 @@ t.subject_id,
 t.cohort_start_date,
 t.cohort_end_date,
 cd.cohort_definition_id
-from #targets_agg_all as t
+from #targets_agg_all t
 INNER JOIN #cohort_details cd
 on cd.target_cohort_id = t.cohort_definition_id
 and cd.cohort_type = 'allT'
@@ -355,7 +360,7 @@ o.subject_id,
 o.cohort_start_date,
 o.cohort_end_date,
 cd.cohort_definition_id
-from #outcomes_agg as o
+from #outcomes_agg o
 INNER JOIN #cohort_details cd
 on cd.outcome_cohort_id = o.cohort_definition_id
 and cd.cohort_type = 'O'
@@ -367,7 +372,7 @@ o.subject_id,
 o.cohort_start_date,
 o.cohort_end_date,
 cd.cohort_definition_id
-from #outcomes_agg_first as o
+from #outcomes_agg_first o
 INNER JOIN #cohort_details cd
 on cd.outcome_cohort_id = o.cohort_definition_id
 and cd.cohort_type = 'firstO'

tests/testthat/helper.R

@@ -0,0 +1,131 @@
+getPlatformConnectionDetails <- function(dbmsPlatform) {
+  # Get drivers for test platform
+  if (dir.exists(Sys.getenv("DATABASECONNECTOR_JAR_FOLDER"))) {
+    jdbcDriverFolder <- Sys.getenv("DATABASECONNECTOR_JAR_FOLDER")
+  } else {
+    jdbcDriverFolder <- "~/.jdbcDrivers"
+    dir.create(jdbcDriverFolder, showWarnings = FALSE)
+  }
+
+  options("sqlRenderTempEmulationSchema" = NULL)
+  if (dbmsPlatform == "sqlite") {
+    connectionDetails <- Eunomia::getEunomiaConnectionDetails()
+    cdmDatabaseSchema <- "main"
+    vocabularyDatabaseSchema <- "main"
+    cohortDatabaseSchema <- "main"
+    options("sqlRenderTempEmulationSchema" = NULL)
+    cohortTable <- "cohort"
+  } else {
+    if (dbmsPlatform == "bigquery") {
+      # To avoid rate limit on BigQuery, only test on 1 OS:
+      if (.Platform$OS.type == "windows") {
+        bqKeyFile <- tempfile(fileext = ".json")
+        writeLines(Sys.getenv("CDM_BIG_QUERY_KEY_FILE"), bqKeyFile)
+        if (testthat::is_testing()) {
+          withr::defer(unlink(bqKeyFile, force = TRUE), testthat::teardown_env())
+        }
+        bqConnectionString <- gsub(
+          "<keyfile path>",
+          normalizePath(bqKeyFile, winslash = "/"),
+          Sys.getenv("CDM_BIG_QUERY_CONNECTION_STRING")
+        )
+        connectionDetails <- DatabaseConnector::createConnectionDetails(
+          dbms = dbmsPlatform,
+          user = "",
+          password = "",
+          connectionString = !!bqConnectionString,
+          pathToDriver = jdbcDriverFolder
+        )
+        cdmDatabaseSchema <- Sys.getenv("CDM_BIG_QUERY_CDM_SCHEMA")
+        vocabularyDatabaseSchema <- Sys.getenv("CDM_BIG_QUERY_CDM_SCHEMA")
+        cohortDatabaseSchema <- Sys.getenv("CDM_BIG_QUERY_OHDSI_SCHEMA")
+        options(sqlRenderTempEmulationSchema = Sys.getenv("CDM_BIG_QUERY_OHDSI_SCHEMA"))
+      } else {
+        return(NULL)
+      }
+    } else if (dbmsPlatform == "oracle") {
+      connectionDetails <- DatabaseConnector::createConnectionDetails(
+        dbms = dbmsPlatform,
+        user = Sys.getenv("CDM5_ORACLE_USER"),
+        password = URLdecode(Sys.getenv("CDM5_ORACLE_PASSWORD")),
+        server = Sys.getenv("CDM5_ORACLE_SERVER"),
+        pathToDriver = jdbcDriverFolder
+      )
+      cdmDatabaseSchema <- Sys.getenv("CDM5_ORACLE_CDM_SCHEMA")
+      vocabularyDatabaseSchema <- Sys.getenv("CDM5_ORACLE_CDM_SCHEMA")
+      cohortDatabaseSchema <- Sys.getenv("CDM5_ORACLE_OHDSI_SCHEMA")
+      options(sqlRenderTempEmulationSchema = Sys.getenv("CDM5_ORACLE_OHDSI_SCHEMA"))
+    } else if (dbmsPlatform == "postgresql") {
+      connectionDetails <- DatabaseConnector::createConnectionDetails(
+        dbms = dbmsPlatform,
+        user = Sys.getenv("CDM5_POSTGRESQL_USER"),
+        password = URLdecode(Sys.getenv("CDM5_POSTGRESQL_PASSWORD")),
+        server = Sys.getenv("CDM5_POSTGRESQL_SERVER"),
+        pathToDriver = jdbcDriverFolder
+      )
+      cdmDatabaseSchema <- Sys.getenv("CDM5_POSTGRESQL_CDM_SCHEMA")
+      vocabularyDatabaseSchema <- Sys.getenv("CDM5_POSTGRESQL_CDM_SCHEMA")
+      cohortDatabaseSchema <- Sys.getenv("CDM5_POSTGRESQL_OHDSI_SCHEMA")
+    } else if (dbmsPlatform == "redshift") {
+      connectionDetails <- DatabaseConnector::createConnectionDetails(
+        dbms = dbmsPlatform,
+        user = Sys.getenv("CDM5_REDSHIFT_USER"),
+        password = URLdecode(Sys.getenv("CDM5_REDSHIFT_PASSWORD")),
+        server = Sys.getenv("CDM5_REDSHIFT_SERVER"),
+        pathToDriver = jdbcDriverFolder
+      )
+      cdmDatabaseSchema <- Sys.getenv("CDM5_REDSHIFT_CDM_SCHEMA")
+      vocabularyDatabaseSchema <- Sys.getenv("CDM5_REDSHIFT_CDM_SCHEMA")
+      cohortDatabaseSchema <- Sys.getenv("CDM5_REDSHIFT_OHDSI_SCHEMA")
+    } else if (dbmsPlatform == "snowflake") {
+      connectionDetails <- DatabaseConnector::createConnectionDetails(
+        dbms = dbmsPlatform,
+        user = Sys.getenv("CDM_SNOWFLAKE_USER"),
+        password = URLdecode(Sys.getenv("CDM_SNOWFLAKE_PASSWORD")),
+        connectionString = Sys.getenv("CDM_SNOWFLAKE_CONNECTION_STRING"),
+        pathToDriver = jdbcDriverFolder
+      )
+      cdmDatabaseSchema <- Sys.getenv("CDM_SNOWFLAKE_CDM53_SCHEMA")
+      vocabularyDatabaseSchema <- Sys.getenv("CDM_SNOWFLAKE_CDM53_SCHEMA")
+      cohortDatabaseSchema <- Sys.getenv("CDM_SNOWFLAKE_OHDSI_SCHEMA")
+      options(sqlRenderTempEmulationSchema = Sys.getenv("CDM_SNOWFLAKE_OHDSI_SCHEMA"))
+    } else if (dbmsPlatform == "spark") {
+      connectionDetails <- DatabaseConnector::createConnectionDetails(
+        dbms = dbmsPlatform,
+        user = Sys.getenv("CDM5_SPARK_USER"),
+        password = URLdecode(Sys.getenv("CDM5_SPARK_PASSWORD")),
+        connectionString = Sys.getenv("CDM5_SPARK_CONNECTION_STRING"),
+        pathToDriver = jdbcDriverFolder
+      )
+      cdmDatabaseSchema <- Sys.getenv("CDM5_SPARK_CDM_SCHEMA")
+      vocabularyDatabaseSchema <- Sys.getenv("CDM5_SPARK_CDM_SCHEMA")
+      cohortDatabaseSchema <- Sys.getenv("CDM5_SPARK_OHDSI_SCHEMA")
+      options(sqlRenderTempEmulationSchema = Sys.getenv("CDM5_SPARK_OHDSI_SCHEMA"))
+    } else if (dbmsPlatform == "sql server") {
+      connectionDetails <- createConnectionDetails(
+        dbms = dbmsPlatform,
+        user = Sys.getenv("CDM5_SQL_SERVER_USER"),
+        password = URLdecode(Sys.getenv("CDM5_SQL_SERVER_PASSWORD")),
+        server = Sys.getenv("CDM5_SQL_SERVER_SERVER"),
+        pathToDriver = jdbcDriverFolder
+      )
+      cdmDatabaseSchema <- Sys.getenv("CDM5_SQL_SERVER_CDM_SCHEMA")
+      vocabularyDatabaseSchema <- Sys.getenv("CDM5_SQL_SERVER_CDM_SCHEMA")
+      cohortDatabaseSchema <- Sys.getenv("CDM5_SQL_SERVER_OHDSI_SCHEMA")
+    }
+
+    # Add drivers
+    DatabaseConnector::downloadJdbcDrivers(dbmsPlatform, pathToDriver = jdbcDriverFolder)
+    # Table created to avoid collisions
+    cohortTable <- paste0("ct_", Sys.getpid(), format(Sys.time(), "%s"), sample(1:100, 1))
+  }
+
+  return(list(
+    dbmsPlatform = dbmsPlatform,
+    connectionDetails = connectionDetails,
+    cohortDatabaseSchema = cohortDatabaseSchema,
+    cohortTable = cohortTable,
+    cdmDatabaseSchema = cdmDatabaseSchema,
+    vocabularyDatabaseSchema = vocabularyDatabaseSchema
+  ))
+}