Merge pull request #1330 from tdrwenski/use-test-containers-2

Use TestContainers

README.md

@@ -73,7 +73,6 @@ THREDDS Catalog documentation (including the specification) is available at
 netCDF-Java is released under the BSD-3 licence, which can be found [here](https://github.com/Unidata/netcdf-java/blob/maint-5.x/LICENSE).
 
 Furthermore, this project includes code from third-party open-source software components:
-* [Gretty](https://github.com/akhikhl/gretty)
 * [ERDDAP](https://coastwatch.pfeg.noaa.gov/erddap/index.html): for details, see `waterml/README.md`
 * [JUnit](https://github.com/junit-team/junit4): for details, see `cdm-test-utils/README.md`
 * Edal (The University of Reading): The CDM calendars are implemented using classes from Jon Blower's `uk.ac.rdg.resc.edal.time` package.

build.gradle

@@ -30,7 +30,6 @@ buildscript {
     classpath buildPlugins.shadow
     classpath buildPlugins.sonarqube
     classpath buildPlugins.spotless
-    classpath buildPlugins.gretty
     classpath buildPlugins.protobuf
     classpath buildPlugins.depcheck
     classpath buildPlugins.nexus
@@ -91,13 +90,6 @@ ext {
   // Will hold the list of projects that apply the java plugin
   // Used by :docs (javadocAll), project wide coverage reports, and sonarqube
   javaProjects = []
-
-  // Will hold the list of projects that apply the maven-publish plugin
-  // A project is registered when the gradle/any/publishing.gradle script plugin
-  // is applied. We track these so that we can be sure to remove any dependencies added
-  // by the gretty plugin (for example, httpservices certainly does not depend on javax.servlet-api,
-  // spring-boot-starter-tomcat, or spring-boot-starter-jetty :-/
-  publishedProjects = []
 }
 
 gradle.projectsEvaluated {

cdm-test-utils/build.gradle

@@ -15,6 +15,7 @@ dependencies {
   implementation 'junit:junit'
   implementation 'org.slf4j:slf4j-api'
   implementation 'com.google.re2j:re2j'
+  implementation 'org.testcontainers:testcontainers'
 
   testRuntimeOnly 'ch.qos.logback:logback-classic'
 }

cdm-test-utils/src/main/java/ucar/unidata/util/test/DapTestContainer.java

@@ -0,0 +1,41 @@
+package ucar.unidata.util.test;
+
+import java.lang.invoke.MethodHandles;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.testcontainers.containers.GenericContainer;
+import org.testcontainers.images.builder.ImageFromDockerfile;
+
+/**
+ * Set up d4ts and dts servers using TestContainers to use for testing opendap, dap4, and httpservices.
+ **/
+public abstract class DapTestContainer {
+  private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+  private static final GenericContainer<?> CONTAINER;
+
+  public static final String NAME = "D4TS and DTS TestContainer";
+  public static final String D4TS_TEST_PATH = "d4ts/testfiles";
+  public static final String DTS_TEST_PATH = "dts";
+
+  public static final String HOST;
+  public static final Integer PORT;
+  public static final String SERVER;
+  public static final String D4TS_PATH;
+  public static final String DTS_PATH;
+
+  static {
+    CONTAINER = new GenericContainer<>(
+        new ImageFromDockerfile().withFileFromClasspath("Dockerfile", "/ucar/unidata/util/test/Dockerfile"))
+            .withExposedPorts(8080);
+    CONTAINER.start();
+
+    HOST = CONTAINER.getHost();
+    PORT = CONTAINER.getFirstMappedPort();
+    SERVER = HOST + ":" + PORT;
+    D4TS_PATH = SERVER + "/" + D4TS_TEST_PATH;
+    DTS_PATH = SERVER + "/" + DTS_TEST_PATH;
+
+    logger.info("Starting d4ts and dts using docker TestContainer at {}", SERVER);
+  }
+}

cdm-test-utils/src/main/java/ucar/unidata/util/test/TestDir.java

@@ -33,10 +33,6 @@
  *        <td>unidata.testdata.path
  *        <td>Property name for the path to the Unidata test data directory,
  *        e.g unidata.testdata.path=/share/testdata
- *        <tr>
- *        <td>remoteTestServerPropName
- *        <td>remotetest
- *        <td>Property name for the hostname of the C-library remote test server.
  *        </table>
  *        <p>
  *        <table>
@@ -58,11 +54,6 @@
  *        <td>NA
  *        <td>../cdm/src/test/data
  *        <td>Level 1 test data directory (distributed with code and MAY be used in Unidata nightly testing).
- *        <tr>
- *        <td>remoteTestServer
- *        <td>remotetestserver
- *        <td>remotetest.unidata.ucar.edu
- *        <td>The hostname of the test server for doing C library remote tests.
  *        </table>
  *
  */
@@ -93,22 +84,6 @@ public class TestDir {
   /** cdm-test data directory (distributed with code but depends on data not in github) */
   public static String cdmTestDataDir = "../cdm-test/src/test/data/";
 
-  //////////////////////////////////////////////////////////////////////
-  // Various Test Server machines
-  //////////////////////////////////////////////////////////////////////
-
-  // Remote Test server(s)
-  private static String remoteTestServerPropName = "remotetestserver";
-  public static String remoteTestServer = "localhost:8081";
-
-  // DAP 2 Test server (for testing)
-  public static String dap2TestServerPropName = "dts";
-  public static String dap2TestServer = "localhost:8080";
-
-  // DAP4 Test server (for testing)
-  public static String dap4TestServerPropName = "d4ts";
-  public static String dap4TestServer = "localhost:8080";
-
   static {
     testdataDir = System.getProperty(testdataDirPropName); // Check the system property.
 
@@ -130,20 +105,6 @@ public class TestDir {
       logger.warn("cdmUnitTest directory does not exist: {}", file.getAbsolutePath());
     }
 
-    // Initialize various server values
-
-    String rts = System.getProperty(remoteTestServerPropName);
-    if (rts != null && rts.length() > 0)
-      remoteTestServer = rts;
-
-    String dts = System.getProperty(dap2TestServerPropName);
-    if (dts != null && dts.length() > 0)
-      dap2TestServer = dts;
-
-    String d4ts = System.getProperty(dap4TestServerPropName);
-    if (d4ts != null && d4ts.length() > 0)
-      dap4TestServer = d4ts;
-
     AliasTranslator.addAlias("${cdmUnitTest}", cdmUnitTestDir);
   }
 

cdm-test-utils/src/main/java/ucar/unidata/util/test/UnitTestCommon.java

@@ -47,15 +47,11 @@ public abstract class UnitTestCommon {
   static final Set<NetcdfDataset.Enhance> ENHANCEMENT = EnumSet.of(NetcdfDataset.Enhance.CoordSystems);
 
   protected static String threddsroot = null;
-  protected static String threddsServer = null;
 
   static {
     // Compute the root path
     threddsroot = locateThreddsRoot();
     assert threddsroot != null : "Cannot locate /thredds parent dir";
-    threddsServer = TestDir.remoteTestServer;
-    if (DEBUG)
-      System.err.println("UnitTestCommon: threddsServer=" + threddsServer);
   }
 
   //////////////////////////////////////////////////

cdm-test-utils/src/main/resources/ucar/unidata/util/test/Dockerfile

@@ -0,0 +1,39 @@
+FROM unidata/tomcat-docker:8.5-jdk11 AS base
+
+MAINTAINER Unidata
+
+USER root
+
+# TODO use release version of d4ts
+#ENV D4TS_WAR_URL https://artifacts.unidata.ucar.edu/repository/unidata-releases/edu/ucar/d4ts/5.4/d4ts-5.4.war
+ENV D4TS_WAR_URL https://artifacts.unidata.ucar.edu/repository/unidata-snapshots/edu/ucar/d4ts/5.5-SNAPSHOT/d4ts-5.5-20240410.110406-179.war
+ENV DTS_WAR_URL https://artifacts.unidata.ucar.edu/repository/unidata-releases/edu/ucar/dtswar/5.4/dtswar-5.4.war
+
+# Install necessary packages
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends curl unzip && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN curl -fSL "${D4TS_WAR_URL}" -o d4ts.war
+RUN curl -fSL "${DTS_WAR_URL}" -o dts.war
+
+RUN unzip d4ts.war -d ${CATALINA_HOME}/webapps/d4ts/ && \
+    rm -f d4ts.war && \
+    unzip dts.war -d ${CATALINA_HOME}/webapps/dts/ && \
+    rm -f dts.war && \
+    chmod 755 ${CATALINA_HOME}/bin/*.sh && \
+    mkdir -p ${CATALINA_HOME}/javaUtilPrefs/.systemPrefs
+
+EXPOSE 8080 8443
+
+WORKDIR ${CATALINA_HOME}
+
+# Inherited from parent container
+ENTRYPOINT ["/entrypoint.sh"]
+
+# Start container
+CMD ["catalina.sh", "run"]
+
+HEALTHCHECK --interval=10s --timeout=3s \
+    CMD (curl --fail 'http://localhost:8080/d4ts/' && curl --fail 'http://localhost:8080/dts/')) || exit 1

cdm/core/src/test/java/ucar/nc2/util/TestUriCreate.java

@@ -9,7 +9,6 @@
 import org.junit.Test;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import ucar.unidata.util.test.TestDir;
 
 /** Examine how java.net.URI brakes up a uri string */
 public class TestUriCreate {
@@ -47,7 +46,7 @@ public void testRelativeFile() throws MalformedURLException, URISyntaxException
 
   @Test
   public void testDods() throws URISyntaxException {
-    String uriString = "http://" + TestDir.dap2TestServer + "/dts/test.53.dods?types[0:1:9]";
+    String uriString = "http://localhost:8080/dts/test.53.dods?types[0:1:9]";
     showUri(uriString);
   }
 

dap4/build.gradle

@@ -2,7 +2,6 @@ ext.title = 'Data Access Protocol (DAP) version 4.0'  // Will be inherited by su
 
 apply from: "$rootDir/gradle/any/dependencies.gradle"
 apply from: "$rootDir/gradle/any/java-library.gradle"
-apply from: "$rootDir/gradle/any/gretty.gradle"
 
 dependencies {
   api enforcedPlatform(project(':netcdf-java-platform'))
@@ -35,7 +34,4 @@ test {
   include 'dap4/test/TestRemote.class'
   include 'dap4/test/TestConstraints.class'
   include 'dap4/test/TestHyrax.class'
-
-  dependsOn('farmBeforeIntegrationTest')
-  finalizedBy('farmAfterIntegrationTest')
 }

dap4/src/test/java/dap4/test/DapTestCommon.java

@@ -8,7 +8,6 @@
 import dap4.dap4lib.HttpDSP;
 import ucar.nc2.dataset.NetcdfDataset;
 import ucar.nc2.write.Ncdump;
-import ucar.unidata.util.test.TestDir;
 import ucar.unidata.util.test.UnitTestCommon;
 
 import java.io.File;
@@ -101,8 +100,6 @@ static String locateDAP4Root(String threddsroot) {
   //////////////////////////////////////////////////
   // Instance variables
 
-  protected String d4tsserver = null;
-
   protected String title = "Dap4 Testing";
 
   public DapTestCommon() {
@@ -112,9 +109,6 @@ public DapTestCommon() {
   public DapTestCommon(String name) {
     super(name);
     setSystemProperties();
-    this.d4tsserver = TestDir.dap4TestServer;
-    if (DEBUG)
-      System.err.println("DapTestCommon: d4tsServer=" + d4tsserver);
   }
 
   /**

dap4/src/test/java/dap4/test/TestConstraints.java

@@ -18,6 +18,7 @@
 import java.nio.file.NoSuchFileException;
 import java.util.ArrayList;
 import java.util.List;
+import ucar.unidata.util.test.DapTestContainer;
 
 /**
  * This Test uses the JUNIT Version 4 parameterized test mechanism.
@@ -32,11 +33,6 @@ public class TestConstraints extends DapTestCommon implements Dap4ManifestIF {
   //////////////////////////////////////////////////
   // Constants
 
-  // Define the server to use
-  protected static final String SERVER = "remotetest.unidata.ucar.edu";
-  protected static final int SERVERPORT = -1;
-  protected static final String SERVERPATH = "d4ts/testfiles";
-
   // Define the input set location(s)
   protected static final String INPUTEXT = ".nc"; // note that the .dap is deliberately left off
   protected static final String INPUTQUERY = "?" + DapConstants.CHECKSUMTAG + "=false&";
@@ -59,8 +55,8 @@ public class TestConstraints extends DapTestCommon implements Dap4ManifestIF {
   public static Dap4Server server;
 
   static {
-    // This test uses remotetest
-    server = new Dap4Server("remotetest", SERVER, SERVERPORT, SERVERPATH);
+    server = new Dap4Server(DapTestContainer.NAME, DapTestContainer.HOST, DapTestContainer.PORT,
+        DapTestContainer.D4TS_TEST_PATH);
     Dap4Server.register(true, server);
     resourceroot = getResourceRoot();
   }

dap4/src/test/java/dap4/test/TestDap4Url.java

@@ -16,6 +16,7 @@
 import java.lang.invoke.MethodHandles;
 import java.util.ArrayList;
 import java.util.List;
+import ucar.unidata.util.test.DapTestContainer;
 
 /**
  * This Test uses the JUNIT Version 4 parameterized test mechanism.
@@ -36,9 +37,9 @@ public class TestDap4Url extends DapTestCommon implements Dap4ManifestIF {
   // Constants
 
   // Legal url formats
-  protected static String[] urls = {"https://remotetest.unidata.ucar.edu/d4ts/testfiles/test_one_var.nc#dap4",
-      "dap4://remotetest.unidata.ucar.edu/d4ts/testfiles/test_one_var.nc",
-      "https://remotetest.unidata.ucar.edu/d4ts/testfiles/test_one_var.nc",};
+  protected static String[] urls = {"http://" + DapTestContainer.D4TS_PATH + "test_one_var.nc#dap4",
+      "dap4://" + DapTestContainer.D4TS_PATH + "test_one_var.nc",
+      "http://" + DapTestContainer.D4TS_PATH + "test_one_var.nc",};
 
   //////////////////////////////////////////////////
   // Static Fields

dap4/src/test/java/dap4/test/TestRemote.java

@@ -8,7 +8,6 @@
 import dap4.core.util.DapConstants;
 import org.junit.Assert;
 import org.junit.Before;
-import org.junit.Ignore;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
@@ -19,6 +18,7 @@
 import java.nio.file.NoSuchFileException;
 import java.util.ArrayList;
 import java.util.List;
+import ucar.unidata.util.test.DapTestContainer;
 
 /**
  * This Test uses the JUNIT Version 4 parameterized test mechanism.
@@ -27,18 +27,12 @@
  * tests by adding fields to the TestCase object.
  */
 
-@Ignore("Disable while RemoteTest server is not working")
 @RunWith(Parameterized.class)
 public class TestRemote extends DapTestCommon implements Dap4ManifestIF {
 
   //////////////////////////////////////////////////
   // Constants
 
-  // Define the server to use
-  protected static final String SERVER = "remotetest.unidata.ucar.edu";
-  protected static final int SERVERPORT = -1;
-  protected static final String SERVERPATH = "d4ts/testfiles";
-
   // Define the input set location(s)
   protected static final String INPUTEXT = ".nc"; // note that the .dap is deliberately left off
   protected static final String INPUTQUERY = "?" + DapConstants.CHECKSUMTAG + "=true";
@@ -61,7 +55,8 @@ public class TestRemote extends DapTestCommon implements Dap4ManifestIF {
   public static Dap4Server server;
 
   static {
-    server = new Dap4Server("remotetest", SERVER, SERVERPORT, SERVERPATH);
+    server = new Dap4Server(DapTestContainer.NAME, DapTestContainer.HOST, DapTestContainer.PORT,
+        DapTestContainer.D4TS_TEST_PATH);
     Dap4Server.register(true, server);
     resourceroot = getResourceRoot();
   }

gradle/README.md

@@ -21,8 +21,6 @@ Most of what we need in the `any/` directory can be found in these two script pl
 
      Currently, two gradle subprojects use this - `dap4:d4tests` and `cdm-test`.
 
-`gretty/`: Contains configuration files related to the `gretty` plugin (logging, cert for testing, etc.)
-
 **TODO:** There are still a few Gradle things left to do, but at least we're fully functional at this point.
 
 1. Address any issues in our plugin scripts identified by gradle in terms of Gradle 7 compatibility.