From 4eef662f464854be04a165456d36bcb1b4adb433 Mon Sep 17 00:00:00 2001 From: Andrey Pleskach Date: Mon, 3 Jul 2023 18:52:06 +0200 Subject: [PATCH] Clean up and bump Apache libs (#2925) Signed-off-by: Andrey Pleskach --- build.gradle | 25 ++++----- .../auth/http/jwt/keybyoidc/JwtVerifier.java | 2 +- .../security/auth/RolesInjector.java | 2 +- .../transport/SecurityInterceptor.java | 2 +- .../org/opensearch/security/ConfigTests.java | 7 ++- .../security/HttpIntegrationTests.java | 11 ++-- .../test/helper/cluster/ClusterHelper.java | 55 +++++++++++++------ .../security/test/helper/file/FileHelper.java | 16 ++++-- 8 files changed, 73 insertions(+), 47 deletions(-) diff --git a/build.gradle b/build.gradle index c1302656c4..9233b01815 100644 --- a/build.gradle +++ b/build.gradle @@ -26,6 +26,7 @@ buildscript { common_utils_version = System.getProperty("common_utils.version", '3.0.0.0-SNAPSHOT') kafka_version = '3.5.0' + apache_cxf_version = '4.0.2' if (buildVersionQualifier) { opensearch_build += "-${buildVersionQualifier}" @@ -409,10 +410,9 @@ bundlePlugin { configurations { all { resolutionStrategy { - force 'commons-codec:commons-codec:1.14' + force 'commons-codec:commons-codec:1.16.0' force 'org.slf4j:slf4j-api:1.7.30' force 'org.scala-lang:scala-library:2.13.9' - force 'commons-io:commons-io:2.11.0' force "com.fasterxml.jackson:jackson-bom:${versions.jackson}" force "com.fasterxml.jackson.core:jackson-core:${versions.jackson}" force "com.fasterxml.jackson.datatype:jackson-datatype-jdk8:${versions.jackson}" @@ -478,13 +478,11 @@ dependencies { implementation "org.apache.httpcomponents:httpasyncclient:${versions.httpasyncclient}" implementation 'com.google.guava:guava:32.0.1-jre' implementation 'org.greenrobot:eventbus:3.2.0' - implementation 'commons-cli:commons-cli:1.3.1' + implementation 'commons-cli:commons-cli:1.5.0' implementation "org.bouncycastle:bcprov-jdk15to18:${versions.bouncycastle}" implementation 'org.ldaptive:ldaptive:1.2.3' implementation 'io.jsonwebtoken:jjwt-api:0.10.8' - implementation('org.apache.cxf:cxf-rt-rs-security-jose:3.5.5') { - exclude(group: 'jakarta.activation', module: 'jakarta.activation-api') - } + implementation 'com.github.wnameless:json-flattener:0.5.0' implementation 'com.flipkart.zjsonpatch:zjsonpatch:0.4.4' implementation "org.apache.kafka:kafka-clients:${kafka_version}" @@ -493,13 +491,16 @@ dependencies { runtimeOnly 'net.minidev:accessors-smart:2.4.7' - runtimeOnly 'org.apache.cxf:cxf-core:3.5.5' - implementation 'org.apache.cxf:cxf-rt-rs-json-basic:3.5.5' - runtimeOnly 'org.apache.cxf:cxf-rt-security:3.5.5' + implementation("org.apache.cxf:cxf-rt-rs-security-jose:${apache_cxf_version}") { + exclude(group: 'jakarta.activation', module: 'jakarta.activation-api') + } + runtimeOnly "org.apache.cxf:cxf-core:${apache_cxf_version}" + implementation "org.apache.cxf:cxf-rt-rs-json-basic:${apache_cxf_version}" + runtimeOnly "org.apache.cxf:cxf-rt-security:${apache_cxf_version}" runtimeOnly 'com.sun.activation:jakarta.activation:1.2.2' runtimeOnly 'com.eclipsesource.minimal-json:minimal-json:0.9.5' - runtimeOnly 'commons-codec:commons-codec:1.14' + runtimeOnly 'commons-codec:commons-codec:1.16.0' runtimeOnly 'org.cryptacular:cryptacular:1.2.4' runtimeOnly 'com.google.errorprone:error_prone_annotations:2.3.4' runtimeOnly 'com.sun.istack:istack-commons-runtime:3.0.12' @@ -525,7 +526,6 @@ dependencies { runtimeOnly 'org.opensaml:opensaml-soap-api:3.4.5' runtimeOnly 'org.opensaml:opensaml-soap-impl:3.4.5' implementation 'org.opensaml:opensaml-storage-api:3.4.5' - implementation 'commons-lang:commons-lang:2.4' implementation 'commons-collections:commons-collections:3.2.2' implementation 'com.jayway.jsonpath:json-path:2.4.0' implementation 'net.minidev:json-smart:2.4.10' @@ -553,7 +553,7 @@ dependencies { runtimeOnly 'org.scala-lang.modules:scala-java8-compat_3:1.0.2' - implementation 'org.apache.commons:commons-lang3:3.4' + implementation 'org.apache.commons:commons-lang3:3.12.0' testImplementation "org.opensearch:common-utils:${common_utils_version}" testImplementation "org.opensearch.plugin:reindex-client:${opensearch_version}" testImplementation "org.opensearch:opensearch-ssl-config:${opensearch_version}" @@ -562,7 +562,6 @@ dependencies { testImplementation "org.opensearch.plugin:parent-join-client:${opensearch_version}" testImplementation "org.opensearch.plugin:aggs-matrix-stats-client:${opensearch_version}" testImplementation 'org.apache.logging.log4j:log4j-core:2.17.1' - testImplementation 'commons-io:commons-io:2.11.0' testImplementation 'javax.servlet:servlet-api:2.5' testImplementation 'com.unboundid:unboundid-ldapsdk:4.0.9' testImplementation 'com.github.stephenc.jcip:jcip-annotations:1.0-1' diff --git a/src/main/java/com/amazon/dlic/auth/http/jwt/keybyoidc/JwtVerifier.java b/src/main/java/com/amazon/dlic/auth/http/jwt/keybyoidc/JwtVerifier.java index 5893d623a7..8cac2a23d1 100644 --- a/src/main/java/com/amazon/dlic/auth/http/jwt/keybyoidc/JwtVerifier.java +++ b/src/main/java/com/amazon/dlic/auth/http/jwt/keybyoidc/JwtVerifier.java @@ -12,7 +12,7 @@ package com.amazon.dlic.auth.http.jwt.keybyoidc; import com.google.common.base.Strings; -import org.apache.commons.lang.StringEscapeUtils; +import org.apache.commons.lang3.StringEscapeUtils; import org.apache.cxf.rs.security.jose.jwa.SignatureAlgorithm; import org.apache.cxf.rs.security.jose.jwk.JsonWebKey; import org.apache.cxf.rs.security.jose.jwk.KeyType; diff --git a/src/main/java/org/opensearch/security/auth/RolesInjector.java b/src/main/java/org/opensearch/security/auth/RolesInjector.java index 4ddfa53101..6c0eaa0732 100644 --- a/src/main/java/org/opensearch/security/auth/RolesInjector.java +++ b/src/main/java/org/opensearch/security/auth/RolesInjector.java @@ -18,7 +18,7 @@ import java.util.Set; import com.google.common.collect.ImmutableSet; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; diff --git a/src/main/java/org/opensearch/security/transport/SecurityInterceptor.java b/src/main/java/org/opensearch/security/transport/SecurityInterceptor.java index 8755d49134..7bd5024d2c 100644 --- a/src/main/java/org/opensearch/security/transport/SecurityInterceptor.java +++ b/src/main/java/org/opensearch/security/transport/SecurityInterceptor.java @@ -34,7 +34,7 @@ import java.util.stream.Collectors; import com.google.common.collect.Maps; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; diff --git a/src/test/java/org/opensearch/security/ConfigTests.java b/src/test/java/org/opensearch/security/ConfigTests.java index 519faa612d..c64ca55318 100644 --- a/src/test/java/org/opensearch/security/ConfigTests.java +++ b/src/test/java/org/opensearch/security/ConfigTests.java @@ -18,11 +18,12 @@ package org.opensearch.security; import java.io.File; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; -import org.apache.commons.io.FileUtils; import org.junit.Assert; import org.junit.Test; @@ -107,7 +108,7 @@ public void testParseSg67Config() throws Exception { private void check(String file, CType cType) throws Exception { final String adjustedFilePath = SingleClusterTest.TEST_RESOURCE_RELATIVE_PATH + file; - JsonNode jsonNode = YAML.readTree(FileUtils.readFileToString(new File(adjustedFilePath), "UTF-8")); + JsonNode jsonNode = YAML.readTree(Files.readString(new File(adjustedFilePath).toPath(), StandardCharsets.UTF_8)); int configVersion = 1; System.out.println("%%%%%%%% THIS IS A LINE OF INTEREST %%%%%%%"); if (jsonNode.get("_meta") != null) { @@ -128,7 +129,7 @@ private void check(String file, CType cType) throws Exception { private SecurityDynamicConfiguration load(String file, CType cType) throws Exception { final String adjustedFilePath = SingleClusterTest.TEST_RESOURCE_RELATIVE_PATH + file; - JsonNode jsonNode = YAML.readTree(FileUtils.readFileToString(new File(adjustedFilePath), "UTF-8")); + JsonNode jsonNode = YAML.readTree(Files.readString(new File(adjustedFilePath).toPath(), StandardCharsets.UTF_8)); int configVersion = 1; System.out.println("%%%%%%%% THIS IS A LINE OF INTEREST LOAD: CONFIG VERSION: %%%%%%%"); diff --git a/src/test/java/org/opensearch/security/HttpIntegrationTests.java b/src/test/java/org/opensearch/security/HttpIntegrationTests.java index a4011b05f6..be3feafaaa 100644 --- a/src/test/java/org/opensearch/security/HttpIntegrationTests.java +++ b/src/test/java/org/opensearch/security/HttpIntegrationTests.java @@ -26,18 +26,13 @@ package org.opensearch.security; -import java.io.File; -import java.nio.charset.StandardCharsets; - import com.fasterxml.jackson.databind.JsonNode; -import org.apache.commons.io.FileUtils; import org.apache.hc.core5.http.HttpStatus; import org.apache.hc.core5.http.NoHttpResponseException; import org.apache.hc.core5.http.message.BasicHeader; import org.junit.Assert; import org.junit.Ignore; import org.junit.Test; - import org.opensearch.action.admin.indices.alias.IndicesAliasesRequest; import org.opensearch.action.admin.indices.alias.IndicesAliasesRequest.AliasActions; import org.opensearch.action.admin.indices.create.CreateIndexRequest; @@ -57,6 +52,10 @@ import org.opensearch.security.test.helper.rest.RestHelper; import org.opensearch.security.test.helper.rest.RestHelper.HttpResponse; +import java.io.File; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; + import static org.opensearch.security.DefaultObjectMapper.readTree; public class HttpIntegrationTests extends SingleClusterTest { @@ -574,7 +573,7 @@ public void testHTTPPlaintextErrMsg() throws Exception { rh.executeGetRequest("", encodeBasicHeader("worf", "worf")); Assert.fail("NoHttpResponseException expected"); } catch (NoHttpResponseException e) { - String log = FileUtils.readFileToString(new File("unittest.log"), StandardCharsets.UTF_8); + String log = Files.readString(new File("unittest.log").toPath(), StandardCharsets.UTF_8); Assert.assertTrue(log, log.contains("speaks http plaintext instead of ssl, will close the channel")); } catch (Exception e) { Assert.fail("NoHttpResponseException expected but was " + e.getClass() + "#" + e.getMessage()); diff --git a/src/test/java/org/opensearch/security/test/helper/cluster/ClusterHelper.java b/src/test/java/org/opensearch/security/test/helper/cluster/ClusterHelper.java index e60fab8e48..b038949782 100644 --- a/src/test/java/org/opensearch/security/test/helper/cluster/ClusterHelper.java +++ b/src/test/java/org/opensearch/security/test/helper/cluster/ClusterHelper.java @@ -29,6 +29,11 @@ // CS-SUPPRESS-SINGLE: RegexpSingleline https://github.com/opensearch-project/OpenSearch/issues/3663 import java.io.File; import java.io.IOException; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; import java.util.Comparator; import java.util.Iterator; import java.util.LinkedList; @@ -40,7 +45,6 @@ import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; -import org.apache.commons.io.FileUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -129,7 +133,7 @@ public final synchronized ClusterInfo startCluster( switch (clusterState) { case UNINITIALIZED: - FileUtils.deleteDirectory(new File("./target/data/" + clustername)); + deleteTestsDataDirectory(); break; case STARTED: closeAllNodes(); @@ -251,19 +255,15 @@ public void run() { PluginAwareNode node = new PluginAwareNode(setting.clusterManagerNode, settingsForNode, setting.getPlugins()); System.out.println(node.settings()); - new Thread(new Runnable() { - - @Override - public void run() { - try { - node.start(); - latch.countDown(); - } catch (Exception e) { - e.printStackTrace(); - log.error("Unable to start node: ", e); - err.set(e); - latch.countDown(); - } + new Thread(() -> { + try { + node.start(); + latch.countDown(); + } catch (Exception e) { + e.printStackTrace(); + log.error("Unable to start node: ", e); + err.set(e); + latch.countDown(); } }).start(); opensearchNodes.add(node); @@ -308,9 +308,30 @@ public void run() { return cInfo; } - public final void stopCluster() throws Exception { + public void stopCluster() throws Exception { closeAllNodes(); - FileUtils.deleteDirectory(new File("./target/data/" + clustername)); + deleteTestsDataDirectory(); + } + + private void deleteTestsDataDirectory() throws IOException { + final File testsDataDir = new File("target/data/" + clustername); + if (testsDataDir.exists()) { + Files.walkFileTree(testsDataDir.toPath(), new SimpleFileVisitor<>() { + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + System.out.println("Deleting file " + file.getFileName()); + Files.delete(file); + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException { + System.out.println("Deleting directory " + dir.getFileName()); + Files.delete(dir); + return FileVisitResult.CONTINUE; + } + }); + } } private void closeAllNodes() throws Exception { diff --git a/src/test/java/org/opensearch/security/test/helper/file/FileHelper.java b/src/test/java/org/opensearch/security/test/helper/file/FileHelper.java index acb4a7d217..9d554ebd06 100644 --- a/src/test/java/org/opensearch/security/test/helper/file/FileHelper.java +++ b/src/test/java/org/opensearch/security/test/helper/file/FileHelper.java @@ -32,6 +32,8 @@ import java.io.FileReader; import java.io.FileWriter; import java.io.IOException; +import java.io.InputStreamReader; +import java.io.Reader; import java.io.StringReader; import java.io.StringWriter; import java.io.UnsupportedEncodingException; @@ -42,11 +44,11 @@ import java.nio.file.Paths; import java.security.KeyStore; -import org.apache.commons.io.IOUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.common.bytes.BytesReference; +import org.opensearch.common.io.Streams; import org.opensearch.common.xcontent.XContentFactory; import org.opensearch.common.xcontent.XContentType; import org.opensearch.core.xcontent.NamedXContentRegistry; @@ -94,10 +96,14 @@ public static Path getAbsoluteFilePathFromClassPath(final String fileNameFromCla return null; } - public static final String loadFile(final String file) throws IOException { - final StringWriter sw = new StringWriter(); - IOUtils.copy(FileHelper.class.getResourceAsStream("/" + file), sw, StandardCharsets.UTF_8); - return sw.toString(); + public static String loadFile(final String file) throws IOException { + try ( + final StringWriter sw = new StringWriter(); + final Reader reader = new InputStreamReader(FileHelper.class.getResourceAsStream("/" + file), StandardCharsets.UTF_8) + ) { + Streams.copy(reader, sw); + return sw.toString(); + } } public static BytesReference readYamlContent(final String file) {