diff --git a/README.md b/README.md index 03f51ca2ac..e95c83dc02 100644 --- a/README.md +++ b/README.md @@ -73,7 +73,6 @@ THREDDS Catalog documentation (including the specification) is available at netCDF-Java is released under the BSD-3 licence, which can be found [here](https://github.com/Unidata/netcdf-java/blob/maint-5.x/LICENSE). Furthermore, this project includes code from third-party open-source software components: -* [Gretty](https://github.com/akhikhl/gretty) * [ERDDAP](https://coastwatch.pfeg.noaa.gov/erddap/index.html): for details, see `waterml/README.md` * [JUnit](https://github.com/junit-team/junit4): for details, see `cdm-test-utils/README.md` * Edal (The University of Reading): The CDM calendars are implemented using classes from Jon Blower's `uk.ac.rdg.resc.edal.time` package. diff --git a/build.gradle b/build.gradle index 46d367fccd..e59df25ff8 100644 --- a/build.gradle +++ b/build.gradle @@ -30,7 +30,6 @@ buildscript { classpath buildPlugins.shadow classpath buildPlugins.sonarqube classpath buildPlugins.spotless - classpath buildPlugins.gretty classpath buildPlugins.protobuf classpath buildPlugins.depcheck classpath buildPlugins.nexus @@ -91,13 +90,6 @@ ext { // Will hold the list of projects that apply the java plugin // Used by :docs (javadocAll), project wide coverage reports, and sonarqube javaProjects = [] - - // Will hold the list of projects that apply the maven-publish plugin - // A project is registered when the gradle/any/publishing.gradle script plugin - // is applied. We track these so that we can be sure to remove any dependencies added - // by the gretty plugin (for example, httpservices certainly does not depend on javax.servlet-api, - // spring-boot-starter-tomcat, or spring-boot-starter-jetty :-/ - publishedProjects = [] } gradle.projectsEvaluated { diff --git a/cdm-test-utils/build.gradle b/cdm-test-utils/build.gradle index abb418dc83..01deef5025 100644 --- a/cdm-test-utils/build.gradle +++ b/cdm-test-utils/build.gradle @@ -15,6 +15,7 @@ dependencies { implementation 'junit:junit' implementation 'org.slf4j:slf4j-api' implementation 'com.google.re2j:re2j' + implementation 'org.testcontainers:testcontainers' testRuntimeOnly 'ch.qos.logback:logback-classic' } diff --git a/cdm-test-utils/src/main/java/ucar/unidata/util/test/DapTestContainer.java b/cdm-test-utils/src/main/java/ucar/unidata/util/test/DapTestContainer.java new file mode 100644 index 0000000000..040c38c364 --- /dev/null +++ b/cdm-test-utils/src/main/java/ucar/unidata/util/test/DapTestContainer.java @@ -0,0 +1,41 @@ +package ucar.unidata.util.test; + +import java.lang.invoke.MethodHandles; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.images.builder.ImageFromDockerfile; + +/** + * Set up d4ts and dts servers using TestContainers to use for testing opendap, dap4, and httpservices. + **/ +public abstract class DapTestContainer { + private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + private static final GenericContainer CONTAINER; + + public static final String NAME = "D4TS and DTS TestContainer"; + public static final String D4TS_TEST_PATH = "d4ts/testfiles"; + public static final String DTS_TEST_PATH = "dts"; + + public static final String HOST; + public static final Integer PORT; + public static final String SERVER; + public static final String D4TS_PATH; + public static final String DTS_PATH; + + static { + CONTAINER = new GenericContainer<>( + new ImageFromDockerfile().withFileFromClasspath("Dockerfile", "/ucar/unidata/util/test/Dockerfile")) + .withExposedPorts(8080); + CONTAINER.start(); + + HOST = CONTAINER.getHost(); + PORT = CONTAINER.getFirstMappedPort(); + SERVER = HOST + ":" + PORT; + D4TS_PATH = SERVER + "/" + D4TS_TEST_PATH; + DTS_PATH = SERVER + "/" + DTS_TEST_PATH; + + logger.info("Starting d4ts and dts using docker TestContainer at {}", SERVER); + } +} diff --git a/cdm-test-utils/src/main/java/ucar/unidata/util/test/TestDir.java b/cdm-test-utils/src/main/java/ucar/unidata/util/test/TestDir.java index b28e8a4be3..ab858c3dec 100644 --- a/cdm-test-utils/src/main/java/ucar/unidata/util/test/TestDir.java +++ b/cdm-test-utils/src/main/java/ucar/unidata/util/test/TestDir.java @@ -33,10 +33,6 @@ * unidata.testdata.path * Property name for the path to the Unidata test data directory, * e.g unidata.testdata.path=/share/testdata - * - * remoteTestServerPropName - * remotetest - * Property name for the hostname of the C-library remote test server. * *

* @@ -58,11 +54,6 @@ * - *
NA * ../cdm/src/test/data * Level 1 test data directory (distributed with code and MAY be used in Unidata nightly testing). - *
remoteTestServer - * remotetestserver - * remotetest.unidata.ucar.edu - * The hostname of the test server for doing C library remote tests. *
* */ @@ -93,22 +84,6 @@ public class TestDir { /** cdm-test data directory (distributed with code but depends on data not in github) */ public static String cdmTestDataDir = "../cdm-test/src/test/data/"; - ////////////////////////////////////////////////////////////////////// - // Various Test Server machines - ////////////////////////////////////////////////////////////////////// - - // Remote Test server(s) - private static String remoteTestServerPropName = "remotetestserver"; - public static String remoteTestServer = "localhost:8081"; - - // DAP 2 Test server (for testing) - public static String dap2TestServerPropName = "dts"; - public static String dap2TestServer = "localhost:8080"; - - // DAP4 Test server (for testing) - public static String dap4TestServerPropName = "d4ts"; - public static String dap4TestServer = "localhost:8080"; - static { testdataDir = System.getProperty(testdataDirPropName); // Check the system property. @@ -130,20 +105,6 @@ public class TestDir { logger.warn("cdmUnitTest directory does not exist: {}", file.getAbsolutePath()); } - // Initialize various server values - - String rts = System.getProperty(remoteTestServerPropName); - if (rts != null && rts.length() > 0) - remoteTestServer = rts; - - String dts = System.getProperty(dap2TestServerPropName); - if (dts != null && dts.length() > 0) - dap2TestServer = dts; - - String d4ts = System.getProperty(dap4TestServerPropName); - if (d4ts != null && d4ts.length() > 0) - dap4TestServer = d4ts; - AliasTranslator.addAlias("${cdmUnitTest}", cdmUnitTestDir); } diff --git a/cdm-test-utils/src/main/java/ucar/unidata/util/test/UnitTestCommon.java b/cdm-test-utils/src/main/java/ucar/unidata/util/test/UnitTestCommon.java index 04607014ac..56a459dce9 100644 --- a/cdm-test-utils/src/main/java/ucar/unidata/util/test/UnitTestCommon.java +++ b/cdm-test-utils/src/main/java/ucar/unidata/util/test/UnitTestCommon.java @@ -47,15 +47,11 @@ public abstract class UnitTestCommon { static final Set ENHANCEMENT = EnumSet.of(NetcdfDataset.Enhance.CoordSystems); protected static String threddsroot = null; - protected static String threddsServer = null; static { // Compute the root path threddsroot = locateThreddsRoot(); assert threddsroot != null : "Cannot locate /thredds parent dir"; - threddsServer = TestDir.remoteTestServer; - if (DEBUG) - System.err.println("UnitTestCommon: threddsServer=" + threddsServer); } ////////////////////////////////////////////////// diff --git a/cdm-test-utils/src/main/resources/ucar/unidata/util/test/Dockerfile b/cdm-test-utils/src/main/resources/ucar/unidata/util/test/Dockerfile new file mode 100644 index 0000000000..0bd5bdec05 --- /dev/null +++ b/cdm-test-utils/src/main/resources/ucar/unidata/util/test/Dockerfile @@ -0,0 +1,39 @@ +FROM unidata/tomcat-docker:8.5-jdk11 AS base + +MAINTAINER Unidata + +USER root + +# TODO use release version of d4ts +#ENV D4TS_WAR_URL https://artifacts.unidata.ucar.edu/repository/unidata-releases/edu/ucar/d4ts/5.4/d4ts-5.4.war +ENV D4TS_WAR_URL https://artifacts.unidata.ucar.edu/repository/unidata-snapshots/edu/ucar/d4ts/5.5-SNAPSHOT/d4ts-5.5-20240410.110406-179.war +ENV DTS_WAR_URL https://artifacts.unidata.ucar.edu/repository/unidata-releases/edu/ucar/dtswar/5.4/dtswar-5.4.war + +# Install necessary packages +RUN apt-get update && \ + apt-get install -y --no-install-recommends curl unzip && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +RUN curl -fSL "${D4TS_WAR_URL}" -o d4ts.war +RUN curl -fSL "${DTS_WAR_URL}" -o dts.war + +RUN unzip d4ts.war -d ${CATALINA_HOME}/webapps/d4ts/ && \ + rm -f d4ts.war && \ + unzip dts.war -d ${CATALINA_HOME}/webapps/dts/ && \ + rm -f dts.war && \ + chmod 755 ${CATALINA_HOME}/bin/*.sh && \ + mkdir -p ${CATALINA_HOME}/javaUtilPrefs/.systemPrefs + +EXPOSE 8080 8443 + +WORKDIR ${CATALINA_HOME} + +# Inherited from parent container +ENTRYPOINT ["/entrypoint.sh"] + +# Start container +CMD ["catalina.sh", "run"] + +HEALTHCHECK --interval=10s --timeout=3s \ + CMD (curl --fail 'http://localhost:8080/d4ts/' && curl --fail 'http://localhost:8080/dts/')) || exit 1 diff --git a/cdm/core/src/test/java/ucar/nc2/util/TestUriCreate.java b/cdm/core/src/test/java/ucar/nc2/util/TestUriCreate.java index 37c275f172..843049d943 100644 --- a/cdm/core/src/test/java/ucar/nc2/util/TestUriCreate.java +++ b/cdm/core/src/test/java/ucar/nc2/util/TestUriCreate.java @@ -9,7 +9,6 @@ import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import ucar.unidata.util.test.TestDir; /** Examine how java.net.URI brakes up a uri string */ public class TestUriCreate { @@ -47,7 +46,7 @@ public void testRelativeFile() throws MalformedURLException, URISyntaxException @Test public void testDods() throws URISyntaxException { - String uriString = "http://" + TestDir.dap2TestServer + "/dts/test.53.dods?types[0:1:9]"; + String uriString = "http://localhost:8080/dts/test.53.dods?types[0:1:9]"; showUri(uriString); } diff --git a/dap4/build.gradle b/dap4/build.gradle index 5956343144..4704936440 100644 --- a/dap4/build.gradle +++ b/dap4/build.gradle @@ -2,7 +2,6 @@ ext.title = 'Data Access Protocol (DAP) version 4.0' // Will be inherited by su apply from: "$rootDir/gradle/any/dependencies.gradle" apply from: "$rootDir/gradle/any/java-library.gradle" -apply from: "$rootDir/gradle/any/gretty.gradle" dependencies { api enforcedPlatform(project(':netcdf-java-platform')) @@ -35,7 +34,4 @@ test { include 'dap4/test/TestRemote.class' include 'dap4/test/TestConstraints.class' include 'dap4/test/TestHyrax.class' - - dependsOn('farmBeforeIntegrationTest') - finalizedBy('farmAfterIntegrationTest') } diff --git a/dap4/src/test/java/dap4/test/DapTestCommon.java b/dap4/src/test/java/dap4/test/DapTestCommon.java index a98c55e941..87e3e20a72 100644 --- a/dap4/src/test/java/dap4/test/DapTestCommon.java +++ b/dap4/src/test/java/dap4/test/DapTestCommon.java @@ -8,7 +8,6 @@ import dap4.dap4lib.HttpDSP; import ucar.nc2.dataset.NetcdfDataset; import ucar.nc2.write.Ncdump; -import ucar.unidata.util.test.TestDir; import ucar.unidata.util.test.UnitTestCommon; import java.io.File; @@ -101,8 +100,6 @@ static String locateDAP4Root(String threddsroot) { ////////////////////////////////////////////////// // Instance variables - protected String d4tsserver = null; - protected String title = "Dap4 Testing"; public DapTestCommon() { @@ -112,9 +109,6 @@ public DapTestCommon() { public DapTestCommon(String name) { super(name); setSystemProperties(); - this.d4tsserver = TestDir.dap4TestServer; - if (DEBUG) - System.err.println("DapTestCommon: d4tsServer=" + d4tsserver); } /** diff --git a/dap4/src/test/java/dap4/test/TestConstraints.java b/dap4/src/test/java/dap4/test/TestConstraints.java index 1477a605fb..32358059b0 100644 --- a/dap4/src/test/java/dap4/test/TestConstraints.java +++ b/dap4/src/test/java/dap4/test/TestConstraints.java @@ -18,6 +18,7 @@ import java.nio.file.NoSuchFileException; import java.util.ArrayList; import java.util.List; +import ucar.unidata.util.test.DapTestContainer; /** * This Test uses the JUNIT Version 4 parameterized test mechanism. @@ -32,11 +33,6 @@ public class TestConstraints extends DapTestCommon implements Dap4ManifestIF { ////////////////////////////////////////////////// // Constants - // Define the server to use - protected static final String SERVER = "remotetest.unidata.ucar.edu"; - protected static final int SERVERPORT = -1; - protected static final String SERVERPATH = "d4ts/testfiles"; - // Define the input set location(s) protected static final String INPUTEXT = ".nc"; // note that the .dap is deliberately left off protected static final String INPUTQUERY = "?" + DapConstants.CHECKSUMTAG + "=false&"; @@ -59,8 +55,8 @@ public class TestConstraints extends DapTestCommon implements Dap4ManifestIF { public static Dap4Server server; static { - // This test uses remotetest - server = new Dap4Server("remotetest", SERVER, SERVERPORT, SERVERPATH); + server = new Dap4Server(DapTestContainer.NAME, DapTestContainer.HOST, DapTestContainer.PORT, + DapTestContainer.D4TS_TEST_PATH); Dap4Server.register(true, server); resourceroot = getResourceRoot(); } diff --git a/dap4/src/test/java/dap4/test/TestDap4Url.java b/dap4/src/test/java/dap4/test/TestDap4Url.java index 47c125b4aa..dfb2347fbf 100644 --- a/dap4/src/test/java/dap4/test/TestDap4Url.java +++ b/dap4/src/test/java/dap4/test/TestDap4Url.java @@ -16,6 +16,7 @@ import java.lang.invoke.MethodHandles; import java.util.ArrayList; import java.util.List; +import ucar.unidata.util.test.DapTestContainer; /** * This Test uses the JUNIT Version 4 parameterized test mechanism. @@ -36,9 +37,9 @@ public class TestDap4Url extends DapTestCommon implements Dap4ManifestIF { // Constants // Legal url formats - protected static String[] urls = {"https://remotetest.unidata.ucar.edu/d4ts/testfiles/test_one_var.nc#dap4", - "dap4://remotetest.unidata.ucar.edu/d4ts/testfiles/test_one_var.nc", - "https://remotetest.unidata.ucar.edu/d4ts/testfiles/test_one_var.nc",}; + protected static String[] urls = {"http://" + DapTestContainer.D4TS_PATH + "test_one_var.nc#dap4", + "dap4://" + DapTestContainer.D4TS_PATH + "test_one_var.nc", + "http://" + DapTestContainer.D4TS_PATH + "test_one_var.nc",}; ////////////////////////////////////////////////// // Static Fields diff --git a/dap4/src/test/java/dap4/test/TestRemote.java b/dap4/src/test/java/dap4/test/TestRemote.java index 8c40522ef0..0442c01621 100644 --- a/dap4/src/test/java/dap4/test/TestRemote.java +++ b/dap4/src/test/java/dap4/test/TestRemote.java @@ -8,7 +8,6 @@ import dap4.core.util.DapConstants; import org.junit.Assert; import org.junit.Before; -import org.junit.Ignore; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; @@ -19,6 +18,7 @@ import java.nio.file.NoSuchFileException; import java.util.ArrayList; import java.util.List; +import ucar.unidata.util.test.DapTestContainer; /** * This Test uses the JUNIT Version 4 parameterized test mechanism. @@ -27,18 +27,12 @@ * tests by adding fields to the TestCase object. */ -@Ignore("Disable while RemoteTest server is not working") @RunWith(Parameterized.class) public class TestRemote extends DapTestCommon implements Dap4ManifestIF { ////////////////////////////////////////////////// // Constants - // Define the server to use - protected static final String SERVER = "remotetest.unidata.ucar.edu"; - protected static final int SERVERPORT = -1; - protected static final String SERVERPATH = "d4ts/testfiles"; - // Define the input set location(s) protected static final String INPUTEXT = ".nc"; // note that the .dap is deliberately left off protected static final String INPUTQUERY = "?" + DapConstants.CHECKSUMTAG + "=true"; @@ -61,7 +55,8 @@ public class TestRemote extends DapTestCommon implements Dap4ManifestIF { public static Dap4Server server; static { - server = new Dap4Server("remotetest", SERVER, SERVERPORT, SERVERPATH); + server = new Dap4Server(DapTestContainer.NAME, DapTestContainer.HOST, DapTestContainer.PORT, + DapTestContainer.D4TS_TEST_PATH); Dap4Server.register(true, server); resourceroot = getResourceRoot(); } diff --git a/gradle/README.md b/gradle/README.md index 4474915601..8b37708b02 100644 --- a/gradle/README.md +++ b/gradle/README.md @@ -21,8 +21,6 @@ Most of what we need in the `any/` directory can be found in these two script pl Currently, two gradle subprojects use this - `dap4:d4tests` and `cdm-test`. -`gretty/`: Contains configuration files related to the `gretty` plugin (logging, cert for testing, etc.) - **TODO:** There are still a few Gradle things left to do, but at least we're fully functional at this point. 1. Address any issues in our plugin scripts identified by gradle in terms of Gradle 7 compatibility. diff --git a/gradle/any/gretty.gradle b/gradle/any/gretty.gradle deleted file mode 100644 index f14c381668..0000000000 --- a/gradle/any/gretty.gradle +++ /dev/null @@ -1,119 +0,0 @@ -// In Gradle, dependencies specified in a parent's buildscript {} block are visible to all children. -// However, that behavior doesn't seem to hold for script plugins (this file) applied from the parent script. -// So we have to repeat ourselves. See the root project's build script for more notes on buildscript {} block weirdness. -buildscript { - apply from: "$rootDir/gradle/any/shared-mvn-coords.gradle" - - repositories { - mavenCentral() - gradlePluginPortal() - } - dependencies { - classpath buildPlugins.gretty // We want to import FarmBeforeIntegrationTestTask, etc. - } -} - -apply from: "$rootDir/gradle/any/properties.gradle" -apply plugin: 'org.gretty' -apply plugin: 'jacoco' - -// when applying the gretty buildScript to a project, make sure that project has access the gretty tomcat runners -repositories { - exclusiveContent { - forRepository { - mavenCentral() - gradlePluginPortal() - } - // only look for unidata gretty related artifacts from the above repos - filter { - includeGroup 'org.gretty' - } - } -} - -import org.akhikhl.gretty.GradleUtils -import org.akhikhl.gretty.FarmStartTask - -gretty { - servletContainer = 'tomcat85' // All webapps run on Tomcat 8.5.x - - // Pass filtered system properties to the embedded Tomcat that Gretty will launch. - systemProperties addFilteredSysProps([:]) - - // We want to modify Gretty tasks but they won't be added to the project until after evaluation: - // http://goo.gl/qlFXTD. So, our modifications must also occur after evaluation. - afterEvaluate { - // Change the log directory of all tasks that start a server. - // Due to a bug, the normal instanceof operator won't work here. Nor will tasks.withType(). - tasks.matching { - GradleUtils.instanceOf(it, FarmStartTask.name) - }.each { - // LauncherConfig is only meant to be retrieved during the execution phase, hence the doFirst{}. - // If we try to grab it during evaluation, we get: - // Error opening zip file or JAR manifest missing : - // build/tmp/expandedArchives/org.jacoco.agent-0.7.7.201606060606.jar_aj7uwwfluzmrbfb6clshgf6du/jacocoagent.jar - it.doFirst { - // tomcatBaseDir is the working directory for embedded tomcat. We want to write our logs there as well. - File tomcatBaseDir = it.launcherConfig.baseDir - - // Will be something like: "$projectDir/build/serverBaseDir_tomcat8/logs". - it.logDir = file("$tomcatBaseDir/logs").absolutePath - } - } - - appBeforeIntegrationTest { - // we do not need to capture coverage on these webapps, as they only exist - // for client testing - jacoco.enabled = false - } - } -} - -farm { - jvmArgs = ["-Dlog4j.configurationFile=$rootDir/gradle/gretty/log4j2Config.xml"] - // used by :dap4 test - webapp 'edu.ucar:d4ts:5.0.0-beta8@war', contextPath: '/d4ts' - - // :opendap and :httpservices test. - webapp 'edu.ucar:dtswar:5.0.0-beta8@war', contextPath: '/dts' - integrationTestTask = 'test' - - // Enable TLS - httpsEnabled = true - sslKeyStorePath = "$rootDir/gradle/gretty/tomcat/keystore" - sslKeyStorePassword = 'secret666' -} - -if (project in publishedProjects) { - apply plugin: 'maven-publish' - // the gradle plugin adds a dependency on javax.servlet, as well as exclusions for - // some spring-boot jars, which end up in the poms on dependencies like slf4j-api. - // ain't nobody got time for that - publishing { - publications { - mavenSources(MavenPublication) { - pom.withXml { - Node pomNode = asNode() - pomNode.dependencies.'*'.findAll() { - it.artifactId.text() == 'javax.servlet-api' - }.each() { - it.parent().remove(it) - } - pomNode = asNode() - pomNode.dependencies.dependency.exclusions.'*'.findAll() { - it.artifactId.text() == 'spring-boot-starter-tomcat' | - it.artifactId.text() == 'spring-boot-starter-jetty' - }.each() { - it.parent().remove(it) - } - pomNode = asNode() - pomNode.dependencies.dependency.exclusions.findAll() { - it.children().isEmpty() - }.each() { - it.parent().remove(it) - } - } - } - } - } -} diff --git a/gradle/any/properties.gradle b/gradle/any/properties.gradle index 106606af1a..1932be4932 100644 --- a/gradle/any/properties.gradle +++ b/gradle/any/properties.gradle @@ -6,7 +6,7 @@ ext { // Used to publish static analysis report to https://sonarcloud.io/dashboard?id=edu.ucar.unidata:ncj SONARQUBE_USER_TOKEN_KEY = 'sonarqube.user.token.ncj' - // Do not propagate these system properties to the Gretty web-apps or Gradle test executors. + // Do not propagate these system properties to the Gradle test executors. systemPropertiesExclude = [ // Passing this causes "java.lang.ClassNotFoundException: org.jacoco.agent.rt.internal_932a715.PreMain" // See https://discuss.gradle.org/t/jacoco-related-failure-in-multiproject-build/6216 diff --git a/gradle/any/publishing.gradle b/gradle/any/publishing.gradle index c4939f93aa..66addd7f6f 100644 --- a/gradle/any/publishing.gradle +++ b/gradle/any/publishing.gradle @@ -1,9 +1,6 @@ // safe to apply to root project. apply plugin: 'maven-publish' -// keeps track of any project with published components -publishedProjects.add(project) - tasks.withType(GenerateModuleMetadata) { enabled = false } diff --git a/gradle/any/shared-mvn-coords.gradle b/gradle/any/shared-mvn-coords.gradle index 71c2c4f1a4..1e5807d647 100644 --- a/gradle/any/shared-mvn-coords.gradle +++ b/gradle/any/shared-mvn-coords.gradle @@ -7,7 +7,6 @@ ext { // plugin version management buildPlugins = [:] - buildPlugins.gretty = 'org.gretty:gretty:3.0.9' buildPlugins.shadow = 'com.github.jengelman.gradle.plugins:shadow:5.2.0' buildPlugins.sonarqube = 'org.sonarsource.scanner.gradle:sonarqube-gradle-plugin:3.0' buildPlugins.spotless = 'com.diffplug.spotless:spotless-plugin-gradle:4.5.1' diff --git a/gradle/any/testing.gradle b/gradle/any/testing.gradle index d9ba397717..50ec02963f 100644 --- a/gradle/any/testing.gradle +++ b/gradle/any/testing.gradle @@ -46,6 +46,17 @@ tasks.withType(Test).all { if (!isUcarNetworkAvailable) { excludeCategories 'ucar.unidata.util.test.category.NeedsUcarNetwork' } + + if (skipDockerTests) { + // dap4 + exclude 'dap4/test/**' + // httpservices + exclude 'ucar/httpservices/**' + exclude 'ucar/nc2/util/net/**' + // opendap + exclude 'opendap/test/**' + exclude 'ucar/nc2/dods/**' + } } } } diff --git a/gradle/root/testing.gradle b/gradle/root/testing.gradle index 1ad812c0cb..a60a11144c 100644 --- a/gradle/root/testing.gradle +++ b/gradle/root/testing.gradle @@ -28,6 +28,7 @@ ext { runSlowTests = System.properties['runSlowTests'] as boolean isRdaDataAvailable = System.properties['rdaDataAvailable'] as boolean isUcarNetworkAvailable = System.properties['ucarNetworkAvailable'] as boolean + skipDockerTests = System.properties['skipDockerTests'] as boolean // Option to run all tests regardless of environment or resource availability runAllTests = System.properties['runAllTestExceptIgnored'] as boolean @@ -65,6 +66,10 @@ ext { if (!isUcarNetworkAvailable) { logger.warn 'Skipping all tests that require access to the UCAR Network.' } + + if (skipDockerTests) { + logger.warn 'Skipping all tests that require docker.' + } } } diff --git a/httpservices/build.gradle b/httpservices/build.gradle index 5d7b633543..4a2240b84b 100755 --- a/httpservices/build.gradle +++ b/httpservices/build.gradle @@ -2,7 +2,6 @@ ext.title = 'HttpClient Wrappers' apply from: "$rootDir/gradle/any/dependencies.gradle" apply from: "$rootDir/gradle/any/java-library.gradle" -apply from: "$rootDir/gradle/any/gretty.gradle" dependencies { api enforcedPlatform(project(':netcdf-java-platform')) @@ -22,8 +21,3 @@ dependencies { testRuntimeOnly 'ch.qos.logback:logback-classic' } - -test { - dependsOn('farmBeforeIntegrationTest') - finalizedBy('farmAfterIntegrationTest') -} diff --git a/httpservices/src/test/java/ucar/httpservices/TestThreading.java b/httpservices/src/test/java/ucar/httpservices/TestThreading.java index b997843235..483e498acb 100644 --- a/httpservices/src/test/java/ucar/httpservices/TestThreading.java +++ b/httpservices/src/test/java/ucar/httpservices/TestThreading.java @@ -39,7 +39,7 @@ import org.junit.experimental.categories.Category; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import ucar.unidata.util.test.TestDir; +import ucar.unidata.util.test.DapTestContainer; import ucar.unidata.util.test.UnitTestCommon; import java.lang.invoke.MethodHandles; import ucar.unidata.util.test.category.NotPullRequest; @@ -62,9 +62,9 @@ public class TestThreading extends UnitTestCommon { protected static final int DFALTTHREADS = 100; protected static final int DFALTMAXCONNS = (DFALTTHREADS / 2); - protected static final String DFALTSERVER = "http://" + TestDir.dap2TestServer; + protected static final String DFALTSERVER = "http://" + DapTestContainer.DTS_PATH; - protected static final String DFALTURLFMT = DFALTSERVER + "/dts/test.%02d"; + protected static final String DFALTURLFMT = DFALTSERVER + "/test.%02d"; static { HTTPSession.TESTING = true; diff --git a/httpservices/src/test/java/ucar/nc2/util/net/TestHTTPMethod.java b/httpservices/src/test/java/ucar/nc2/util/net/TestHTTPMethod.java index 34f1dea037..3cb67eb355 100644 --- a/httpservices/src/test/java/ucar/nc2/util/net/TestHTTPMethod.java +++ b/httpservices/src/test/java/ucar/nc2/util/net/TestHTTPMethod.java @@ -39,7 +39,7 @@ import ucar.httpservices.HTTPFactory; import ucar.httpservices.HTTPMethod; import ucar.httpservices.HTTPSession; -import ucar.unidata.util.test.TestDir; +import ucar.unidata.util.test.DapTestContainer; import ucar.unidata.util.test.UnitTestCommon; import java.io.InputStream; import java.lang.invoke.MethodHandles; @@ -50,7 +50,7 @@ public class TestHTTPMethod extends UnitTestCommon { ////////////////////////////////////////////////// // Constants - protected final String baseurl = "http://" + TestDir.dap2TestServer + "/dts"; + protected final String baseurl = "http://" + DapTestContainer.DTS_PATH; static final String testcase = "test.01.dds"; diff --git a/httpservices/src/test/java/ucar/nc2/util/net/TestHang.java b/httpservices/src/test/java/ucar/nc2/util/net/TestHang.java index 95aeb53965..ea3df2fe9c 100644 --- a/httpservices/src/test/java/ucar/nc2/util/net/TestHang.java +++ b/httpservices/src/test/java/ucar/nc2/util/net/TestHang.java @@ -22,7 +22,7 @@ import ucar.httpservices.HTTPFactory; import ucar.httpservices.HTTPMethod; import ucar.httpservices.HTTPSession; -import ucar.unidata.util.test.TestDir; +import ucar.unidata.util.test.DapTestContainer; import java.lang.invoke.MethodHandles; import java.util.ArrayList; import java.util.List; @@ -42,9 +42,9 @@ public class TestHang { static private HTTPSession session; - protected static final String server = "http://" + TestDir.dap2TestServer; + protected static final String server = "http://" + DapTestContainer.DTS_PATH; - protected static final String url = server + "/dts/test.%02d.dds"; + protected static final String url = server + "/test.%02d.dds"; static boolean isxfail(int x) { for (Integer i : XFAIL) { diff --git a/httpservices/src/test/java/ucar/nc2/util/net/TestState.java b/httpservices/src/test/java/ucar/nc2/util/net/TestState.java index c320a16ed0..6e9d2f3f04 100644 --- a/httpservices/src/test/java/ucar/nc2/util/net/TestState.java +++ b/httpservices/src/test/java/ucar/nc2/util/net/TestState.java @@ -40,7 +40,7 @@ import ucar.httpservices.HTTPMethod; import ucar.httpservices.HTTPMethodStream; import ucar.httpservices.HTTPSession; -import ucar.unidata.util.test.TestDir; +import ucar.unidata.util.test.DapTestContainer; import ucar.unidata.util.test.UnitTestCommon; import java.lang.invoke.MethodHandles; import java.nio.charset.Charset; @@ -61,7 +61,7 @@ public class TestState extends UnitTestCommon { ////////////////////////////////////////////////// // Constants - protected final String SESSIONURL = "http://" + TestDir.dap2TestServer + "/dts"; + protected final String SESSIONURL = "http://" + DapTestContainer.DTS_PATH; protected final String TESTSOURCE1 = SESSIONURL + "/test.01.dds"; public TestState() { diff --git a/httpservices/src/test/java/ucar/nc2/util/net/TestURIParse.java b/httpservices/src/test/java/ucar/nc2/util/net/TestURIParse.java index 575bb9c9e9..5f161d1abd 100644 --- a/httpservices/src/test/java/ucar/nc2/util/net/TestURIParse.java +++ b/httpservices/src/test/java/ucar/nc2/util/net/TestURIParse.java @@ -37,7 +37,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import ucar.httpservices.HTTPUtil; -import ucar.unidata.util.test.TestDir; import ucar.unidata.util.test.UnitTestCommon; import java.lang.invoke.MethodHandles; import java.net.URI; @@ -52,8 +51,8 @@ public class TestURIParse extends UnitTestCommon { public static boolean DEBUG = false; - static final String CARON = "http://" + TestDir.remoteTestServer - + "/thredds/cdmremote/scanCdmUnitTests/formats/hdf5/grid_1_3d_xyz_aug.h5?req=data&var=HDFEOS_INFORMATION/StructMetadata\\.0"; + static final String CARON = + "http://localhost:8081/thredds/cdmremote/scanCdmUnitTests/formats/hdf5/grid_1_3d_xyz_aug.h5?req=data&var=HDFEOS_INFORMATION/StructMetadata\\.0"; static final String[] httptests = {"http://ucar.edu:8081/dts/test\\/fake\\.01", CARON,}; diff --git a/netcdf-java-testing-platform/build.gradle b/netcdf-java-testing-platform/build.gradle index cf0db0f07b..3163b2a19c 100644 --- a/netcdf-java-testing-platform/build.gradle +++ b/netcdf-java-testing-platform/build.gradle @@ -31,6 +31,9 @@ dependencies { // cdm-test (GRIB related testing) api 'org.apache.commons:commons-compress:1.12' + // opendap, dap4, and httpservices + api 'org.testcontainers:testcontainers:1.19.7' + // In Spock, allows mocking of classes (in addition to interfaces). // todo: remove with legacy in 6 runtime 'cglib:cglib-nodep:3.2.4' diff --git a/opendap/build.gradle b/opendap/build.gradle index 4d1d2c683d..6f6d0355e3 100644 --- a/opendap/build.gradle +++ b/opendap/build.gradle @@ -9,7 +9,6 @@ ext.url = 'http://opendap.org/' apply from: "$rootDir/gradle/any/dependencies.gradle" apply from: "$rootDir/gradle/any/java-library.gradle" -apply from: "$rootDir/gradle/any/gretty.gradle" dependencies { api enforcedPlatform(project(':netcdf-java-platform')) @@ -30,8 +29,3 @@ dependencies { testRuntimeOnly 'ch.qos.logback:logback-classic' } - -test { - dependsOn('farmBeforeIntegrationTest') - finalizedBy('farmAfterIntegrationTest') -} diff --git a/opendap/src/test/java/opendap/test/TestDConnect2.java b/opendap/src/test/java/opendap/test/TestDConnect2.java index 8a08b6167b..bb46426ba7 100644 --- a/opendap/src/test/java/opendap/test/TestDConnect2.java +++ b/opendap/src/test/java/opendap/test/TestDConnect2.java @@ -187,7 +187,6 @@ void testpart(TestPart part, String ce) { public void testDConnect2() throws Exception { System.out.printf("*** Testing %s\n", TITLE); - System.out.println(" Note: The remote tests may be slow or even fail if the server is overloaded"); for (TestSetEnum e : whichtests) { currentTestSet = TestSets.get(e); diff --git a/opendap/src/test/java/opendap/test/TestDuplicates.java b/opendap/src/test/java/opendap/test/TestDuplicates.java index 04c015a682..132c9b6c0a 100644 --- a/opendap/src/test/java/opendap/test/TestDuplicates.java +++ b/opendap/src/test/java/opendap/test/TestDuplicates.java @@ -39,7 +39,7 @@ import ucar.nc2.dods.DODSNetcdfFile; import ucar.nc2.write.CDLWriter; import ucar.unidata.util.test.Diff; -import ucar.unidata.util.test.TestDir; +import ucar.unidata.util.test.DapTestContainer; import ucar.unidata.util.test.UnitTestCommon; import java.io.IOException; import java.io.PrintWriter; @@ -72,7 +72,7 @@ public Result(String title, String url, String cdl) { @Test public void testDuplicates() throws Exception { // Check if we are running against remote or localhost, or what. - String testserver = TestDir.dap2TestServer; + String testserver = DapTestContainer.SERVER; List results = new ArrayList(); results.add(new Result("Top and field vars have same names", "http://" + testserver + "/dts/structdupname", diff --git a/opendap/src/test/java/opendap/test/TestGroups.java b/opendap/src/test/java/opendap/test/TestGroups.java index 4c12ce7ac6..3170ef3278 100644 --- a/opendap/src/test/java/opendap/test/TestGroups.java +++ b/opendap/src/test/java/opendap/test/TestGroups.java @@ -40,7 +40,7 @@ import ucar.nc2.util.rc.RC; import ucar.nc2.write.CDLWriter; import ucar.unidata.util.test.Diff; -import ucar.unidata.util.test.TestDir; +import ucar.unidata.util.test.DapTestContainer; import ucar.unidata.util.test.UnitTestCommon; import java.io.*; import java.lang.invoke.MethodHandles; @@ -70,7 +70,7 @@ public Testcase(String title, String url, String cdl) { public TestGroups() throws Exception { super("DAP Group tests"); // Check if user specified server. - testserver = TestDir.dap2TestServer; + testserver = DapTestContainer.SERVER; definetestcases(); } diff --git a/opendap/src/test/java/opendap/test/TestMisc.java b/opendap/src/test/java/opendap/test/TestMisc.java index 4442b76200..40d9c40678 100644 --- a/opendap/src/test/java/opendap/test/TestMisc.java +++ b/opendap/src/test/java/opendap/test/TestMisc.java @@ -39,7 +39,7 @@ import ucar.nc2.dods.DODSNetcdfFile; import ucar.nc2.write.CDLWriter; import ucar.unidata.util.test.Diff; -import ucar.unidata.util.test.TestDir; +import ucar.unidata.util.test.DapTestContainer; import ucar.unidata.util.test.UnitTestCommon; import java.io.*; import java.lang.invoke.MethodHandles; @@ -68,7 +68,7 @@ public Testcase(String title, String url, String cdl) { public TestMisc() { setTitle("DAP Misc tests"); // Check if we are running against remote or localhost, or what. - testserver = TestDir.dap2TestServer; + testserver = DapTestContainer.SERVER; definetestcases(); } @@ -116,6 +116,9 @@ boolean process1(Testcase testcase) throws Exception { } boolean diff(Testcase testcase, String captured) throws Exception { + // don't match on host and port + captured = captured.replaceAll("dods://.*:\\d+/", "dods://localhost:8080/"); + // See if the cdl is in a file or a string. if (System.getProperty("nodiff") != null) return true; diff --git a/opendap/src/test/java/opendap/test/TestSources.java b/opendap/src/test/java/opendap/test/TestSources.java index d71a378cbb..95721f1e8f 100644 --- a/opendap/src/test/java/opendap/test/TestSources.java +++ b/opendap/src/test/java/opendap/test/TestSources.java @@ -2,7 +2,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import ucar.unidata.util.test.TestDir; +import ucar.unidata.util.test.DapTestContainer; import java.io.File; import java.io.FileReader; import java.lang.invoke.MethodHandles; @@ -15,7 +15,7 @@ public class TestSources extends TestFiles { ////////////////////////////////////////////////// // Remote test info - public static final String URL = "http://" + TestDir.dap2TestServer + "/dts"; + public static final String URL = "http://" + DapTestContainer.DTS_PATH; // These shorter tests are always run static final String[] TESTS_S1 = diff --git a/opendap/src/test/java/ucar/nc2/dods/TestSequence.java b/opendap/src/test/java/ucar/nc2/dods/TestSequence.java index 4e09496eeb..5ce95fb652 100644 --- a/opendap/src/test/java/ucar/nc2/dods/TestSequence.java +++ b/opendap/src/test/java/ucar/nc2/dods/TestSequence.java @@ -12,6 +12,7 @@ import ucar.nc2.dataset.NetcdfDataset; import java.lang.invoke.MethodHandles; import ucar.nc2.dataset.NetcdfDatasets; +import ucar.unidata.util.test.DapTestContainer; public class TestSequence { private static final Logger logger = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); @@ -21,7 +22,7 @@ public void testSequence() { // The old url: "http://tsds.net/tsds/test/Scalar" is no longer valid. // So replaced with an equivalent. // Also had to replace the struct "TimeSeries" and the field "time" - String url = "https://remotetest.unidata.ucar.edu/dts/whoi"; + String url = "http://" + DapTestContainer.DTS_PATH + "/whoi"; try { NetcdfDataset ds = NetcdfDatasets.openDataset(url); assertThat(ds).isNotNull();