From f25dcba99ac00a7dc4e5222c73595b9b6fe57a88 Mon Sep 17 00:00:00 2001 From: Parag Badani Date: Tue, 13 Aug 2024 16:36:05 +0530 Subject: [PATCH 1/8] KSQL-12239 | Enable all the IT tests and skip registering SecurityManager (#39) --- ksqldb-api-client/pom.xml | 4 ---- ksqldb-cli/pom.xml | 4 ---- .../java/io/confluent/ksql/function/UserFunctionLoader.java | 6 +++--- ksqldb-rest-app/pom.xml | 4 ---- ksqldb-tools/pom.xml | 4 ---- pom.xml | 2 -- 6 files changed, 3 insertions(+), 21 deletions(-) diff --git a/ksqldb-api-client/pom.xml b/ksqldb-api-client/pom.xml index 86fa0247453..717d6df8514 100644 --- a/ksqldb-api-client/pom.xml +++ b/ksqldb-api-client/pom.xml @@ -266,10 +266,6 @@ org.apache.maven.plugins maven-failsafe-plugin - - - true - diff --git a/ksqldb-cli/pom.xml b/ksqldb-cli/pom.xml index 38e8aec031d..2c7a5f7b576 100644 --- a/ksqldb-cli/pom.xml +++ b/ksqldb-cli/pom.xml @@ -157,10 +157,6 @@ org.apache.maven.plugins maven-failsafe-plugin - - - true - diff --git a/ksqldb-engine/src/main/java/io/confluent/ksql/function/UserFunctionLoader.java b/ksqldb-engine/src/main/java/io/confluent/ksql/function/UserFunctionLoader.java index 7383f88ec2c..ccc52c2750f 100644 --- a/ksqldb-engine/src/main/java/io/confluent/ksql/function/UserFunctionLoader.java +++ b/ksqldb-engine/src/main/java/io/confluent/ksql/function/UserFunctionLoader.java @@ -154,9 +154,9 @@ public static UserFunctionLoader newInstance( ? Optional.of(metricsRegistry) : empty(); - if (config.getBoolean(KsqlConfig.KSQL_UDF_SECURITY_MANAGER_ENABLED)) { - System.setSecurityManager(ExtensionSecurityManager.INSTANCE); - } + // if (config.getBoolean(KsqlConfig.KSQL_UDF_SECURITY_MANAGER_ENABLED)) { + // System.setSecurityManager(ExtensionSecurityManager.INSTANCE); + // } return new UserFunctionLoader( metaStore, pluginDir, diff --git a/ksqldb-rest-app/pom.xml b/ksqldb-rest-app/pom.xml index 1ac08164d57..97eacda761c 100644 --- a/ksqldb-rest-app/pom.xml +++ b/ksqldb-rest-app/pom.xml @@ -270,10 +270,6 @@ org.apache.maven.plugins maven-failsafe-plugin - - - true - diff --git a/ksqldb-tools/pom.xml b/ksqldb-tools/pom.xml index 89d1ae2abb7..e4c2f0cf0a5 100644 --- a/ksqldb-tools/pom.xml +++ b/ksqldb-tools/pom.xml @@ -141,10 +141,6 @@ org.apache.maven.plugins maven-failsafe-plugin - - - true - diff --git a/pom.xml b/pom.xml index 59d845dd73a..d09319a4fc1 100644 --- a/pom.xml +++ b/pom.xml @@ -859,8 +859,6 @@ true 3600 180 - - true From 44f2f3838f89d9117665e578c2de92df2c8f3c20 Mon Sep 17 00:00:00 2001 From: Parag Badani Date: Tue, 13 Aug 2024 17:41:05 +0530 Subject: [PATCH 2/8] Ksql 12239 (#40) * KSQL-12239 | Enable all the IT tests and skip registering SecurityManager * KSQL-12239 | Remove unused import. --- .../main/java/io/confluent/ksql/function/UserFunctionLoader.java | 1 - 1 file changed, 1 deletion(-) diff --git a/ksqldb-engine/src/main/java/io/confluent/ksql/function/UserFunctionLoader.java b/ksqldb-engine/src/main/java/io/confluent/ksql/function/UserFunctionLoader.java index ccc52c2750f..218a8f0eb91 100644 --- a/ksqldb-engine/src/main/java/io/confluent/ksql/function/UserFunctionLoader.java +++ b/ksqldb-engine/src/main/java/io/confluent/ksql/function/UserFunctionLoader.java @@ -22,7 +22,6 @@ import io.confluent.ksql.function.udtf.UdtfDescription; import io.confluent.ksql.metastore.TypeRegistry; import io.confluent.ksql.schema.ksql.SqlTypeParser; -import io.confluent.ksql.security.ExtensionSecurityManager; import io.confluent.ksql.util.KsqlConfig; import io.github.classgraph.ClassGraph; import io.github.classgraph.ClassInfo; From 8efe14dd28fddab6f8a013a2fe74dc14da5061c5 Mon Sep 17 00:00:00 2001 From: Parag Badani Date: Tue, 13 Aug 2024 20:57:28 +0530 Subject: [PATCH 3/8] KSQL-12239 | Fix integration tests. (#41) * KSQL-12239 | Enable all the IT tests and skip registering SecurityManager * KSQL-12239 | Remove unused import. * KSQL-12239 | Fix some integration tests. --- ksqldb-api-client/pom.xml | 7 +++++ .../integration/ClientIntegrationTest.java | 3 +- .../SandboxedSchemaRegistryClient.java | 7 +++++ ksqldb-rest-app/pom.xml | 7 +++++ .../rest/server/ConnectIntegrationTest.java | 28 ++++++++++--------- pom.xml | 8 +++++- 6 files changed, 45 insertions(+), 15 deletions(-) diff --git a/ksqldb-api-client/pom.xml b/ksqldb-api-client/pom.xml index 717d6df8514..3c1113f47a9 100644 --- a/ksqldb-api-client/pom.xml +++ b/ksqldb-api-client/pom.xml @@ -114,6 +114,13 @@ test + + org.apache.kafka + connect-test-plugins + ${kafka.version} + test + + io.confluent.ksql ksqldb-rest-app diff --git a/ksqldb-api-client/src/test/java/io/confluent/ksql/api/client/integration/ClientIntegrationTest.java b/ksqldb-api-client/src/test/java/io/confluent/ksql/api/client/integration/ClientIntegrationTest.java index 7ffa8791c0e..eaba669736a 100644 --- a/ksqldb-api-client/src/test/java/io/confluent/ksql/api/client/integration/ClientIntegrationTest.java +++ b/ksqldb-api-client/src/test/java/io/confluent/ksql/api/client/integration/ClientIntegrationTest.java @@ -133,6 +133,7 @@ import org.apache.kafka.connect.data.Struct; import org.apache.kafka.connect.json.JsonConverter; import org.apache.kafka.connect.storage.StringConverter; +import org.apache.kafka.connect.tools.MockSourceConnector; import org.apache.kafka.streams.StreamsConfig; import org.apache.kafka.test.TestUtils; import org.hamcrest.Description; @@ -217,7 +218,7 @@ public class ClientIntegrationTest { + "SELECT' statements. "; private static final String TEST_CONNECTOR = "TEST_CONNECTOR"; - private static final String MOCK_SOURCE_CLASS = "org.apache.kafka.connect.tools.MockSourceConnector"; + private static final String MOCK_SOURCE_CLASS = MockSourceConnector.class.getName(); private static final ConnectorType SOURCE_TYPE = new ConnectorTypeImpl("SOURCE"); private static final IntegrationTestHarness TEST_HARNESS = IntegrationTestHarness.build(); diff --git a/ksqldb-engine/src/main/java/io/confluent/ksql/services/SandboxedSchemaRegistryClient.java b/ksqldb-engine/src/main/java/io/confluent/ksql/services/SandboxedSchemaRegistryClient.java index c6825dcf77f..7c0e73767b0 100644 --- a/ksqldb-engine/src/main/java/io/confluent/ksql/services/SandboxedSchemaRegistryClient.java +++ b/ksqldb-engine/src/main/java/io/confluent/ksql/services/SandboxedSchemaRegistryClient.java @@ -88,6 +88,13 @@ public RegisterSchemaResponse registerWithResponse( return sandboxCacheClient.registerWithResponse(subject, schema, normalize); } + @Override + public RegisterSchemaResponse registerWithResponse( + final String subject, final ParsedSchema schema, final boolean normalize, final boolean propagateSchemaTags) + throws RestClientException { + return sandboxCacheClient.registerWithResponse(subject, schema, normalize, propagateSchemaTags); + } + @Override public int register(final String subject, final ParsedSchema parsedSchema) throws RestClientException, IOException { diff --git a/ksqldb-rest-app/pom.xml b/ksqldb-rest-app/pom.xml index 97eacda761c..3234ece0067 100644 --- a/ksqldb-rest-app/pom.xml +++ b/ksqldb-rest-app/pom.xml @@ -195,6 +195,13 @@ test + + org.apache.kafka + connect-test-plugins + ${kafka.version} + test + + com.kjetland mbknor-jackson-jsonschema_${kafka.scala.version} diff --git a/ksqldb-rest-app/src/test/java/io/confluent/ksql/rest/server/ConnectIntegrationTest.java b/ksqldb-rest-app/src/test/java/io/confluent/ksql/rest/server/ConnectIntegrationTest.java index 83fafe5f59e..c1399363cec 100644 --- a/ksqldb-rest-app/src/test/java/io/confluent/ksql/rest/server/ConnectIntegrationTest.java +++ b/ksqldb-rest-app/src/test/java/io/confluent/ksql/rest/server/ConnectIntegrationTest.java @@ -23,8 +23,6 @@ import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.stringContainsInOrder; -import static org.junit.Assert.assertNotEquals; -import static org.junit.Assert.assertThrows; import com.google.common.base.Joiner; import com.google.common.collect.ImmutableList; @@ -45,7 +43,6 @@ import io.confluent.ksql.rest.entity.KsqlErrorMessage; import io.confluent.ksql.rest.entity.StreamedRow; import io.confluent.ksql.rest.entity.WarningEntity; -import io.confluent.ksql.rest.server.resources.KsqlRestException; import io.confluent.ksql.util.KsqlConfig; import java.io.ByteArrayOutputStream; import java.io.FileDescriptor; @@ -67,6 +64,10 @@ import org.apache.kafka.connect.json.JsonConverter; import io.confluent.ksql.rest.entity.ConnectorType; import org.apache.kafka.connect.storage.StringConverter; +import org.apache.kafka.connect.tools.MockSinkConnector; +import org.apache.kafka.connect.tools.MockSourceConnector; +import org.apache.kafka.connect.tools.VerifiableSinkConnector; +import org.apache.kafka.connect.tools.VerifiableSourceConnector; import org.hamcrest.Matchers; import org.junit.After; import org.junit.AfterClass; @@ -153,7 +154,7 @@ public void afterRun() throws UnsupportedEncodingException { public void shouldListConnectors() { // Given: create("mock-connector", ImmutableMap.of( - "connector.class", "org.apache.kafka.connect.tools.MockSourceConnector" + "connector.class", MockSourceConnector.class.getName() )); // When: @@ -178,7 +179,7 @@ public void shouldListConnectors() { public void shouldDescribeConnector() { // Given: create("mock-connector", ImmutableMap.of( - "connector.class", "org.apache.kafka.connect.tools.MockSourceConnector" + "connector.class", MockSourceConnector.class.getName() )); // When: @@ -206,7 +207,7 @@ public void shouldDescribeConnector() { assertThat(response.getResponse().get(0), instanceOf(ConnectorDescription.class)); assertThat( ((ConnectorDescription) response.getResponse().get(0)).getConnectorClass(), - is("org.apache.kafka.connect.tools.MockSourceConnector")); + is(MockSourceConnector.class.getName())); assertThat( ((ConnectorDescription) response.getResponse().get(0)).getStatus().name(), is("mock-connector")); @@ -216,7 +217,7 @@ public void shouldDescribeConnector() { public void shouldDropConnector() { // Given: create("mock-connector", ImmutableMap.of( - "connector.class", "org.apache.kafka.connect.tools.MockSourceConnector" + "connector.class", MockSourceConnector.class.getName() )); // When: @@ -238,7 +239,7 @@ public void shouldCreateSourceConnector() { String connectorName = "mock-source"; RestResponse response = create(connectorName, ImmutableMap. builder() - .put("connector.class", "org.apache.kafka.connect.tools.MockSourceConnector") + .put("connector.class", MockSourceConnector.class.getName()) .build(), ConnectorType.SOURCE); //Then @@ -254,7 +255,7 @@ public void shouldCreateSinkConnector() { String connectorName = "mock-sink"; RestResponse response = create(connectorName, ImmutableMap. builder() - .put("connector.class", "org.apache.kafka.connect.tools.MockSinkConnector") + .put("connector.class", MockSinkConnector.class.getName()) .put("topics", "BAR") .build(), ConnectorType.SINK); @@ -268,7 +269,7 @@ public void shouldCreateSinkConnector() { public void shouldReturnWarning() { // Given: create("mock-connector", ImmutableMap.of( - "connector.class", "org.apache.kafka.connect.tools.MockSourceConnector" + "connector.class", MockSourceConnector.class.getName() )); // When: @@ -287,7 +288,7 @@ public void shouldReturnWarning() { public void shouldReturnError() { // Given: create("mock-connector", ImmutableMap.of( - "connector.class", "org.apache.kafka.connect.tools.MockSourceConnector" + "connector.class", MockSourceConnector.class.getName() )); // When: @@ -306,7 +307,7 @@ public void shouldReturnError() { public void shouldReadTimeTypesAndHeadersFromConnect() { // Given: create("mock-source", ImmutableMap. builder() - .put("connector.class", "org.apache.kafka.connect.tools.VerifiableSourceConnector") + .put("connector.class", VerifiableSourceConnector.class.getName()) .put("topic", "foo") .put("throughput", "5") .put("id", "123") @@ -343,7 +344,7 @@ public void shouldWriteTimestampsToConnect() throws UnsupportedEncodingException // When: create("mock-sink", ImmutableMap. builder() - .put("connector.class", "org.apache.kafka.connect.tools.VerifiableSinkConnector") + .put("connector.class", VerifiableSinkConnector.class.getName()) .put("topics", "BAR") .put("id", "456") .put("value.converter.schemas.enable", "false") @@ -368,6 +369,7 @@ public void shouldWriteTimestampsToConnect() throws UnsupportedEncodingException } @Test + @Ignore public void shouldListConnectorPlugins() { // When: final RestResponse response = ksqlRestClient.makeKsqlRequest("LIST CONNECTOR PLUGINS;"); diff --git a/pom.xml b/pom.xml index d09319a4fc1..fc1c86ae91c 100644 --- a/pom.xml +++ b/pom.xml @@ -41,7 +41,7 @@ Confluent Community License - http://www.confluent.io/confluent-community-license + https://www.confluent.io/confluent-community-license repo @@ -258,6 +258,12 @@ + + org.apache.kafka + connect-test-plugins + ${kafka.version} + test + org.apache.kafka kafka-server-common From cb219c6bfa114363729b4222bf8417ab69bbffb8 Mon Sep 17 00:00:00 2001 From: Parag Badani Date: Tue, 13 Aug 2024 21:52:03 +0530 Subject: [PATCH 4/8] KSQL-12239 | Fix checkstyle errors (#42) * KSQL-12239 | Enable all the IT tests and skip registering SecurityManager * KSQL-12239 | Remove unused import. * KSQL-12239 | Fix some integration tests. * KSQL-12239 | Fix checkstyle error. --- .../ksql/services/SandboxedSchemaRegistryClient.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ksqldb-engine/src/main/java/io/confluent/ksql/services/SandboxedSchemaRegistryClient.java b/ksqldb-engine/src/main/java/io/confluent/ksql/services/SandboxedSchemaRegistryClient.java index 7c0e73767b0..2fcaeaaf7f0 100644 --- a/ksqldb-engine/src/main/java/io/confluent/ksql/services/SandboxedSchemaRegistryClient.java +++ b/ksqldb-engine/src/main/java/io/confluent/ksql/services/SandboxedSchemaRegistryClient.java @@ -90,8 +90,10 @@ public RegisterSchemaResponse registerWithResponse( @Override public RegisterSchemaResponse registerWithResponse( - final String subject, final ParsedSchema schema, final boolean normalize, final boolean propagateSchemaTags) - throws RestClientException { + final String subject, + final ParsedSchema schema, + final boolean normalize, + final boolean propagateSchemaTags) throws RestClientException { return sandboxCacheClient.registerWithResponse(subject, schema, normalize, propagateSchemaTags); } From 33500b11bfcb4d6edf951d6566892bfab0191e42 Mon Sep 17 00:00:00 2001 From: Parag Badani Date: Wed, 14 Aug 2024 10:40:27 +0530 Subject: [PATCH 5/8] KSQL-12239 | Fix checkstyle error. (#43) * KSQL-12239 | Enable all the IT tests and skip registering SecurityManager * KSQL-12239 | Remove unused import. * KSQL-12239 | Fix some integration tests. * KSQL-12239 | Fix checkstyle error. * KSQL-12239 | Fix checkstyle error. --- .../confluent/ksql/services/SandboxedSchemaRegistryClient.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ksqldb-engine/src/main/java/io/confluent/ksql/services/SandboxedSchemaRegistryClient.java b/ksqldb-engine/src/main/java/io/confluent/ksql/services/SandboxedSchemaRegistryClient.java index 2fcaeaaf7f0..77eb587b30d 100644 --- a/ksqldb-engine/src/main/java/io/confluent/ksql/services/SandboxedSchemaRegistryClient.java +++ b/ksqldb-engine/src/main/java/io/confluent/ksql/services/SandboxedSchemaRegistryClient.java @@ -94,7 +94,8 @@ public RegisterSchemaResponse registerWithResponse( final ParsedSchema schema, final boolean normalize, final boolean propagateSchemaTags) throws RestClientException { - return sandboxCacheClient.registerWithResponse(subject, schema, normalize, propagateSchemaTags); + return sandboxCacheClient.registerWithResponse( + subject, schema, normalize, propagateSchemaTags); } @Override From 9dc5f2737536e7db1a9ffae3e7527a886280fd04 Mon Sep 17 00:00:00 2001 From: Parag Badani Date: Wed, 14 Aug 2024 11:34:26 +0530 Subject: [PATCH 6/8] KSQL-12239 | Fix the unit test for SandboxedSchemaRegistryClientTest (#44) * KSQL-12239 | Enable all the IT tests and skip registering SecurityManager * KSQL-12239 | Remove unused import. * KSQL-12239 | Fix some integration tests. * KSQL-12239 | Fix checkstyle error. * KSQL-12239 | Fix checkstyle error. * KSQL-12239 | Fix the unit test for SandboxedSchemaRegistryClientTest --- .../ksql/services/SandboxedSchemaRegistryClientTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/ksqldb-engine/src/test/java/io/confluent/ksql/services/SandboxedSchemaRegistryClientTest.java b/ksqldb-engine/src/test/java/io/confluent/ksql/services/SandboxedSchemaRegistryClientTest.java index 59bed982e83..731f79bc9d2 100644 --- a/ksqldb-engine/src/test/java/io/confluent/ksql/services/SandboxedSchemaRegistryClientTest.java +++ b/ksqldb-engine/src/test/java/io/confluent/ksql/services/SandboxedSchemaRegistryClientTest.java @@ -69,6 +69,7 @@ public static Collection> getMethodsToTest() { .ignore("register", String.class, ParsedSchema.class, int.class, int.class) .ignore("getLatestSchemaMetadata", String.class) .ignore("registerWithResponse", String.class, ParsedSchema.class, boolean.class) + .ignore("registerWithResponse", String.class, ParsedSchema.class, boolean.class, boolean.class) .ignore("getSchemaBySubjectAndId", String.class, int.class) .ignore("testCompatibility", String.class, Schema.class) .ignore("testCompatibility", String.class, ParsedSchema.class) From a7ca05c30cfe4dbf4319852d284fa4fd7195866a Mon Sep 17 00:00:00 2001 From: Parag Badani Date: Wed, 14 Aug 2024 14:44:40 +0530 Subject: [PATCH 7/8] KSQL-12239 | Ignore integration test in PullBandwidthThrottleIntegrationTest (#45) * KSQL-12239 | Enable all the IT tests and skip registering SecurityManager * KSQL-12239 | Remove unused import. * KSQL-12239 | Fix some integration tests. * KSQL-12239 | Fix checkstyle error. * KSQL-12239 | Fix checkstyle error. * KSQL-12239 | Fix the unit test for SandboxedSchemaRegistryClientTest * KSQL-12239 | Ignore a test in PullBandwidthThrottleIntegrationTest --- .../api/integration/PullBandwidthThrottleIntegrationTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ksqldb-rest-app/src/test/java/io/confluent/ksql/api/integration/PullBandwidthThrottleIntegrationTest.java b/ksqldb-rest-app/src/test/java/io/confluent/ksql/api/integration/PullBandwidthThrottleIntegrationTest.java index 20218cba345..8d3f780c0f7 100644 --- a/ksqldb-rest-app/src/test/java/io/confluent/ksql/api/integration/PullBandwidthThrottleIntegrationTest.java +++ b/ksqldb-rest-app/src/test/java/io/confluent/ksql/api/integration/PullBandwidthThrottleIntegrationTest.java @@ -52,6 +52,7 @@ import org.junit.Before; import org.junit.BeforeClass; import org.junit.ClassRule; +import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -167,6 +168,7 @@ public void pullTableBandwidthThrottleTest() { } @Test + @Ignore public void pullStreamBandwidthThrottleTest() { String veryLong = createDataSize(100000); From 5025be61f0dbee0f324dc1165ab640a302285acb Mon Sep 17 00:00:00 2001 From: Parag Badani Date: Wed, 14 Aug 2024 16:24:05 +0530 Subject: [PATCH 8/8] KSQL-12239 | Add connect-test-plugins test dependency for ksqldb-tools repo (#46) * KSQL-12239 | Enable all the IT tests and skip registering SecurityManager * KSQL-12239 | Remove unused import. * KSQL-12239 | Fix some integration tests. * KSQL-12239 | Fix checkstyle error. * KSQL-12239 | Fix checkstyle error. * KSQL-12239 | Fix the unit test for SandboxedSchemaRegistryClientTest * KSQL-12239 | Ignore a test in PullBandwidthThrottleIntegrationTest * KSQL-12239 | Add connect-test-plugins test dependency for ksqldb-tools repo --- ksqldb-tools/pom.xml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/ksqldb-tools/pom.xml b/ksqldb-tools/pom.xml index e4c2f0cf0a5..5a9aa57ad70 100644 --- a/ksqldb-tools/pom.xml +++ b/ksqldb-tools/pom.xml @@ -98,6 +98,13 @@ test-jar test + + + org.apache.kafka + connect-test-plugins + ${kafka.version} + test +