test(pipeline): use containerized Liquibase for unit tests

Refactor the `UnitTest` base class to utilize the `liquibase` container
for database migrations and onboarding, aligning its behavior with
`FunctionalTest`. This change removes the need for manual Liquibase
orchestration within the test code.

- Update `FunctionalTest` wait strategies to include SQL Server healthchecks and Liquibase migration completion logs.
- Modify `DataSourceConfig` to throw a `RuntimeException` instead of logging a warning when a custom compose file is not found.
- Simplify `UnitTest` by removing manual migration and onboarding script execution logic.
- Standardize environment initialization and container management across unit and functional test suites.

Author: ericbuckley

reporting-pipeline-service/src/test/java/gov/cdc/nbs/report/pipeline/integration/functional/FunctionalTest.java

@@ -24,34 +24,34 @@
 @Tag("Functional")
 public abstract class FunctionalTest {
 
-  private static final Logger logger = LoggerFactory.getLogger(FunctionalTest.class);
-  private static final Slf4jLogConsumer consumer = new Slf4jLogConsumer(logger);
-  private static boolean started = false;
-
+  private static final Logger log = LoggerFactory.getLogger(FunctionalTest.class);
+  private static final Slf4jLogConsumer consumer = new Slf4jLogConsumer(log);
   private static final File base = new File("../docker-compose.yaml");
 
+  private static boolean started = false;
   private static ComposeContainer environment;
 
   @Autowired
   @Qualifier("customComposeFile")
-  private static File customComposeFile;
+  private File customComposeFile;
 
   @SuppressWarnings("resource")
-  static void initializeEnvironment() {
-    DockerImageName dockerImage = DockerImageName.parse("docker:25.0.5");
+  void initializeEnvironment() {
     List<File> composeFiles = new ArrayList<>(Arrays.asList(base));
     if (customComposeFile != null) {
-      logger.info(
+      log.info(
           "Using custom compose override. Base: {}, Custom: {}",
           base.getPath(),
           customComposeFile.getPath());
       composeFiles.add(customComposeFile);
     }
     environment =
-        new ComposeContainer(dockerImage, composeFiles)
+        new ComposeContainer(DockerImageName.parse("docker:25.0.5"), composeFiles)
             // List specific services to prevent launching wildfly container
             .withServices("nbs-mssql", "liquibase", "kafka", "debezium", "kafka-connect")
+            .waitingFor("nbs-mssql", Wait.forHealthcheck())
             .waitingFor("debezium", Wait.forHealthcheck())
+            .waitingFor("liquibase", Wait.forLogMessage(".*Migrations complete.*\\n", 1))
             .waitingFor("kafka-connect", Wait.forHealthcheck())
             // Pull logs from the containers for better debugging
             .withLogConsumer("nbs-mssql", consumer)
@@ -64,7 +64,7 @@ static void initializeEnvironment() {
   }
 
   @BeforeAll
-  static void setUp() {
+  void setUp() {
     // Start up necessary containers if they are not already running.
     // ComposeContainer does not allow container reuse natively
     if (!started) {
@@ -77,7 +77,7 @@ static void setUp() {
   }
 
   @AfterAll
-  static void tearDown() {
+  void tearDown() {
     if (started) {
       if (environment == null) {
         initializeEnvironment();

reporting-pipeline-service/src/test/java/gov/cdc/nbs/report/pipeline/integration/support/config/DataSourceConfig.java

@@ -76,7 +76,6 @@ public File customFunctionalCompose(Environment env) {
       return parentDirFile;
     }
 
-    log.warn("Custom compose file '{}' not found in current or parent directory.", filePath);
-    return null;
+    throw new RuntimeException("Custom compose file '%s' could not be found.".formatted(filePath));
   }
 }

reporting-pipeline-service/src/test/java/gov/cdc/nbs/report/pipeline/integration/unit/UnitTest.java

@@ -2,38 +2,24 @@
 
 import gov.cdc.nbs.report.pipeline.integration.support.config.DataSourceConfig;
 import java.io.File;
-import java.nio.file.Files;
-import java.sql.Connection;
 import java.time.Duration;
 import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.Comparator;
 import java.util.List;
-import java.util.Map;
-import java.util.Objects;
 import javax.sql.DataSource;
-import liquibase.Scope;
-import liquibase.command.CommandScope;
-import liquibase.command.core.UpdateCommandStep;
-import liquibase.database.Database;
-import liquibase.database.DatabaseFactory;
-import liquibase.database.jvm.JdbcConnection;
-import liquibase.resource.CompositeResourceAccessor;
-import liquibase.resource.DirectoryResourceAccessor;
-import liquibase.resource.ResourceAccessor;
-import lombok.extern.slf4j.Slf4j;
 import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Tag;
 import org.junit.jupiter.api.TestInstance;
 import org.junit.jupiter.api.TestInstance.Lifecycle;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Qualifier;
 import org.springframework.boot.test.autoconfigure.jdbc.AutoConfigureTestDatabase;
 import org.springframework.boot.test.autoconfigure.jdbc.AutoConfigureTestDatabase.Replace;
 import org.springframework.boot.test.autoconfigure.orm.jpa.DataJpaTest;
 import org.springframework.context.annotation.Import;
-import org.springframework.jdbc.core.JdbcTemplate;
 import org.springframework.test.context.ActiveProfiles;
 import org.testcontainers.containers.ComposeContainer;
 import org.testcontainers.containers.wait.strategy.Wait;
@@ -45,136 +31,61 @@
 @Import(DataSourceConfig.class)
 @AutoConfigureTestDatabase(replace = Replace.NONE)
 @TestInstance(Lifecycle.PER_CLASS)
-@Slf4j
 public abstract class UnitTest {
 
-  private static boolean started = false;
+  private static final Logger log = LoggerFactory.getLogger(UnitTest.class);
   private static final File base = new File("../docker-compose.yaml");
 
-  @SuppressWarnings("resource")
-  private static final ComposeContainer environment =
-      new ComposeContainer(DockerImageName.parse("docker:25.0.5"), base)
-          .withServices("nbs-mssql")
-          .waitingFor("nbs-mssql", Wait.forHealthcheck())
-          // Set the maximum startup timeout all the waits set are bounded to
-          .withStartupTimeout(Duration.ofMinutes(5));
+  private static boolean started = false;
+  private static ComposeContainer environment;
+
+  @Autowired
+  @Qualifier("customComposeFile")
+  private File customComposeFile;
 
   @Autowired
   @Qualifier("adminDataSource")
   private DataSource adminDataSource;
 
-  // The root from which ALL db files (changelogs and sql) can be found
-  private static final String RESOURCE_ROOT = "../liquibase-service/src/main/resources";
-
-  // Relative paths from RESOURCE_ROOT
-  private static final String MIGRATION_DIR = RESOURCE_ROOT + "/db";
-  private static final String ONBOARDING_DIR =
-      RESOURCE_ROOT + "/db/001-master/02_onboarding_script_data_load";
-
-  @AfterAll
-  void tearDown() throws Exception {
-    if (started) {
-      environment.stop();
-      started = false;
+  @SuppressWarnings("resource")
+  void initializeEnvironment() {
+    List<File> composeFiles = new ArrayList<>(Arrays.asList(base));
+    if (customComposeFile != null) {
+      log.warn(
+          "Using custom compose override. Base: {}, Custom: {}",
+          base.getPath(),
+          customComposeFile.getPath());
+      composeFiles.add(customComposeFile);
     }
+    environment =
+        new ComposeContainer(DockerImageName.parse("docker:25.0.5"), composeFiles)
+            .withServices("nbs-mssql")
+            .waitingFor("nbs-mssql", Wait.forHealthcheck())
+            .withServices("liquibase")
+            .waitingFor("liquibase", Wait.forLogMessage(".*Migrations complete.*\\n", 1))
+            // Set the maximum startup timeout all the waits set are bounded to
+            .withStartupTimeout(Duration.ofMinutes(5));
   }
 
   @BeforeAll
   void setUp() throws Exception {
     if (!started) {
+      if (environment == null) {
+        initializeEnvironment();
+      }
       environment.start();
       started = true;
     }
-    // checkDatabaseConnection();
-    log.info("Starting Migration Phase...");
-
-    applyMigration("NBS_ODSE", "db.odse.admin.tasks.changelog-16.1.yaml");
-    applyMigration("NBS_ODSE", "db.odse.changelog-16.1.yaml");
-    applyMigration("NBS_SRTE", "db.srte.admin.tasks.changelog-16.1.yaml");
-    applyMigration("rdb_modern", "db.rdb.changelog-16.1.yaml");
-    applyMigration("rdb_modern", "db.rdb_modern.changelog-16.1.yaml");
-
-    log.info("Starting Onboarding Phase...");
-    applyOnboardingScripts("NBS_ODSE");
   }
 
-  private void applyMigration(String dbName, String changelogFile) throws Exception {
-    try (Connection connection = adminDataSource.getConnection()) {
-      connection.setCatalog(dbName);
-
-      // 1. Setup the Database wrapper
-      Database database =
-          DatabaseFactory.getInstance()
-              .findCorrectDatabaseImplementation(new JdbcConnection(connection));
-
-      // 2. Build the recursive Search Path (Accessors)
-      // This is necessary as all the migrations are organized in nested
-      // subdirectories, but the changelog files only reference the migrations
-      // by their filename, and not their relative path.
-      List<DirectoryResourceAccessor> accessors = new ArrayList<>();
-      accessors.add(new DirectoryResourceAccessor(new File(RESOURCE_ROOT)));
-      try (var stream = Files.walk(new File(MIGRATION_DIR).toPath())) {
-        stream
-            .filter(Files::isDirectory)
-            .map(
-                path -> {
-                  try {
-                    return new DirectoryResourceAccessor(path.toFile());
-                  } catch (Exception e) {
-                    return null;
-                  }
-                })
-            .filter(Objects::nonNull)
-            .forEach(accessors::add);
-      }
-
-      // 3. Execute the Command
-      log.debug("Applying: " + changelogFile + " to " + dbName);
-      try (var compositeAccessor =
-          new CompositeResourceAccessor(accessors.toArray(ResourceAccessor[]::new))) {
-        Scope.child(
-            Map.of(Scope.Attr.resourceAccessor.name(), compositeAccessor),
-            () -> {
-              new CommandScope(UpdateCommandStep.COMMAND_NAME)
-                  .addArgumentValue(
-                      UpdateCommandStep.CHANGELOG_FILE_ARG, "db/changelog/" + changelogFile)
-                  .addArgumentValue("database", database)
-                  .execute();
-            });
-      } finally {
-        if (database != null) {
-          database.close();
-        }
+  @AfterAll
+  void tearDown() throws Exception {
+    if (started) {
+      if (environment == null) {
+        initializeEnvironment();
       }
-    }
-  }
-
-  private void applyOnboardingScripts(String dbName) throws Exception {
-    File onboardingDir = new File(ONBOARDING_DIR);
-
-    if (!onboardingDir.exists()) {
-      throw new RuntimeException(
-          "Onboarding directory not found: " + onboardingDir.getAbsolutePath());
-    }
-
-    File[] sqlFiles = onboardingDir.listFiles((dir, name) -> name.toLowerCase().endsWith(".sql"));
-    if (sqlFiles == null || sqlFiles.length == 0) {
-      log.error("No SQL files found in onboarding directory: {}", ONBOARDING_DIR);
-      return;
-    }
-    Arrays.sort(sqlFiles, Comparator.comparing(File::getName));
-
-    // Create a local JdbcTemplate to gain access to the low-level .execute() method
-    JdbcTemplate jdbcTemplate = new JdbcTemplate(adminDataSource);
-
-    for (File sqlFile : sqlFiles) {
-      log.debug("Executing SQL: " + sqlFile.getName());
-      String content = Files.readString(sqlFile.toPath());
-
-      Arrays.stream(content.split("(?i)\\bGO\\b"))
-          .map(String::trim)
-          .filter(batch -> !batch.isEmpty())
-          .forEach(batch -> jdbcTemplate.execute(batch));
+      environment.stop();
+      started = false;
     }
   }
 }