From ca2d0911c5966ce63b26fde23defdc437eb2b993 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=90=B4=E5=A4=A9=E5=AE=87?= <1599646055@qq.com> Date: Sun, 20 Jul 2025 18:57:12 +0800 Subject: [PATCH 01/39] feat:Add metalake configuration option in seatunnel-env.sh and seatunnel-env.cmd files; modify EnvCommonOptions class to include metalake configuration option --- config/seatunnel-env.cmd | 11 ++++++++++- config/seatunnel-env.sh | 6 ++++++ .../api/options/EnvCommonOptions.java | 18 ++++++++++++++++++ .../seatunnel/api/options/EnvOptionRule.java | 5 ++++- 4 files changed, 38 insertions(+), 2 deletions(-) diff --git a/config/seatunnel-env.cmd b/config/seatunnel-env.cmd index 79c2d3c117c..0a70c0e1c63 100644 --- a/config/seatunnel-env.cmd +++ b/config/seatunnel-env.cmd @@ -18,4 +18,13 @@ REM Home directory of spark distribution. if "%SPARK_HOME%" == "" set "SPARK_HOME=C:\Program Files\spark" REM Home directory of flink distribution. -if "%FLINK_HOME%" == "" set "FLINK_HOME=C:\Program Files\flink" \ No newline at end of file +if "%FLINK_HOME%" == "" set "FLINK_HOME=C:\Program Files\flink" + +REM Whether to enable metalake (true/false). +if "%METALAKE_ENABLED%" == "" set "META_LAKE_ENABLED=false" + +REM Type of metalake implementation. +if "%METALAKE_TYPE%" == "" set "METALAKE_TYPE=gravitino" + +REM Metalake service URL, format: http://host:port/api/metalakes/{metalake_name}/catalogs/{catalog_name} +if "%METALAKE_URL%" == "" set "METALAKE_URL=http://localhost:8090/api/metalakes/default_metalake_name/catalogs/default_catalog_name" \ No newline at end of file diff --git a/config/seatunnel-env.sh b/config/seatunnel-env.sh index 1bae8c76254..a0c814a9894 100644 --- a/config/seatunnel-env.sh +++ b/config/seatunnel-env.sh @@ -20,3 +20,9 @@ SPARK_HOME=${SPARK_HOME:-/opt/spark} # Home directory of flink distribution. FLINK_HOME=${FLINK_HOME:-/opt/flink} +# Whether to enable metalake (true/false). +METALAKE_ENABLED=${METALAKE_ENABLED:-false} +# Type of metalake implementation. +METALAKE_TYPE=${METALAKE_TYPE:-gravitino} +# Metalake service URL, format: http://host:port/api/metalakes/{metalake_name}/catalogs/{catalog_name}. +METALAKE_URL=${METALAKE_URL:-http://localhost:8090/api/metalakes/default_metalake_name/catalogs/default_catalog_name} diff --git a/seatunnel-api/src/main/java/org/apache/seatunnel/api/options/EnvCommonOptions.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/options/EnvCommonOptions.java index 663727caf82..a2292020afc 100644 --- a/seatunnel-api/src/main/java/org/apache/seatunnel/api/options/EnvCommonOptions.java +++ b/seatunnel-api/src/main/java/org/apache/seatunnel/api/options/EnvCommonOptions.java @@ -107,4 +107,22 @@ public class EnvCommonOptions { .mapType() .noDefaultValue() .withDescription("Define the worker where the job runs by tag"); + + public static Option METALAKE_ENABLED = + Options.key("metalake_enabled") + .boolType() + .defaultValue(false) + .withDescription("Whether to enable metalake"); + + public static Option METALAKE_TYPE = + Options.key("metalake_type") + .stringType() + .noDefaultValue() + .withDescription("Type of metalake implementation"); + + public static Option METALAKE_URL = + Options.key("metalake_url") + .stringType() + .noDefaultValue() + .withDescription("Metalake service URL, format: http://host:port/api/metalakes/{metalake_name}/catalogs/{catalog_name}"); } diff --git a/seatunnel-api/src/main/java/org/apache/seatunnel/api/options/EnvOptionRule.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/options/EnvOptionRule.java index 0c13cba4dfe..cb5273ca778 100644 --- a/seatunnel-api/src/main/java/org/apache/seatunnel/api/options/EnvOptionRule.java +++ b/seatunnel-api/src/main/java/org/apache/seatunnel/api/options/EnvOptionRule.java @@ -46,7 +46,10 @@ public OptionRule optionRule() { EnvCommonOptions.READ_LIMIT_BYTES_PER_SECOND, EnvCommonOptions.SAVEMODE_EXECUTE_LOCATION, EnvCommonOptions.CUSTOM_PARAMETERS, - EnvCommonOptions.NODE_TAG_FILTER) + EnvCommonOptions.NODE_TAG_FILTER, + EnvCommonOptions.METALAKE_ENABLED, + EnvCommonOptions.METALAKE_TYPE, + EnvCommonOptions.METALAKE_URL) .build(); } } From 8ceffc9c19ef0ee30844a6d4c3ff58878c6a1524 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=90=B4=E5=A4=A9=E5=AE=87?= <1599646055@qq.com> Date: Tue, 22 Jul 2025 11:26:06 +0800 Subject: [PATCH 02/39] Feat:Fetch metalake config from environment and merge into task's env --- .../seatunnel/api/options/EnvCommonOptions.java | 14 +++++++------- .../parse/MultipleTableJobConfigParser.java | 17 ++++++++++++++++- 2 files changed, 23 insertions(+), 8 deletions(-) diff --git a/seatunnel-api/src/main/java/org/apache/seatunnel/api/options/EnvCommonOptions.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/options/EnvCommonOptions.java index a2292020afc..d8cd4b364c2 100644 --- a/seatunnel-api/src/main/java/org/apache/seatunnel/api/options/EnvCommonOptions.java +++ b/seatunnel-api/src/main/java/org/apache/seatunnel/api/options/EnvCommonOptions.java @@ -108,21 +108,21 @@ public class EnvCommonOptions { .noDefaultValue() .withDescription("Define the worker where the job runs by tag"); - public static Option METALAKE_ENABLED = - Options.key("metalake_enabled") - .boolType() + public static Option METALAKE_ENABLED = + Options.key("metalake.enabled") + .booleanType() .defaultValue(false) .withDescription("Whether to enable metalake"); public static Option METALAKE_TYPE = - Options.key("metalake_type") + Options.key("metalake.type") .stringType() .noDefaultValue() .withDescription("Type of metalake implementation"); - public static Option METALAKE_URL = - Options.key("metalake_url") + public static Option METALAKE_URL = + Options.key("metalake.url") .stringType() .noDefaultValue() - .withDescription("Metalake service URL, format: http://host:port/api/metalakes/{metalake_name}/catalogs/{catalog_name}"); + .withDescription("Metalake service URL"); } diff --git a/seatunnel-engine/seatunnel-engine-core/src/main/java/org/apache/seatunnel/engine/core/parse/MultipleTableJobConfigParser.java b/seatunnel-engine/seatunnel-engine-core/src/main/java/org/apache/seatunnel/engine/core/parse/MultipleTableJobConfigParser.java index 377040ff953..a0379679fa8 100644 --- a/seatunnel-engine/seatunnel-engine-core/src/main/java/org/apache/seatunnel/engine/core/parse/MultipleTableJobConfigParser.java +++ b/seatunnel-engine/seatunnel-engine-core/src/main/java/org/apache/seatunnel/engine/core/parse/MultipleTableJobConfigParser.java @@ -159,7 +159,22 @@ public MultipleTableJobConfigParser( this.jobConfig = jobConfig; this.commonPluginJars = commonPluginJars; this.isStartWithSavePoint = isStartWithSavePoint; - this.seaTunnelJobConfig = ConfigBuilder.of(Paths.get(jobDefineFilePath), variables); + //this.seaTunnelJobConfig = ConfigBuilder.of(Paths.get(jobDefineFilePath), variables); + boolean metalakeEnabled = Boolean.parseBoolean(System.getenv().getOrDefault("METALAKE_ENABLED", "false")); + Map metalakeConfigMap = new LinkedHashMap<>(); + if (metalakeEnabled) { + String metalakeType = System.getenv("METALAKE_TYPE"); + String metalakeUrl = System.getenv("METALAKE_URL"); + + metalakeConfigMap.put("metalake.enabled", true); + metalakeConfigMap.put("metalake.type", metalakeType != null ? metalakeType : ""); + metalakeConfigMap.put("metalake.url", metalakeUrl != null ? metalakeUrl : ""); + } else { + metalakeConfigMap.put("metalake.enabled", false); + } + Config metalakeConfig = ConfigBuilder.of(metalakeConfigMap); + this.seaTunnelJobConfig = metalakeConfig.withFallback(ConfigBuilder.of(Paths.get(jobDefineFilePath), variables)); + this.envOptions = ReadonlyConfig.fromConfig(seaTunnelJobConfig.getConfig("env")); this.pipelineCheckpoints = pipelineCheckpoints; ConfigValidator.of(this.envOptions).validate(new EnvOptionRule().optionRule()); From b5b3cc0236b823ad41c05021aca155f1cb7e6b5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=90=B4=E5=A4=A9=E5=AE=87?= <1599646055@qq.com> Date: Mon, 28 Jul 2025 22:53:11 +0800 Subject: [PATCH 03/39] feat:Fetch metalake config by sourceId and replace placeholders when metalake_enable=true --- config/seatunnel-env.cmd | 4 +- config/seatunnel-env.sh | 4 +- .../parse/MultipleTableJobConfigParser.java | 72 ++++++++++++++++--- 3 files changed, 67 insertions(+), 13 deletions(-) diff --git a/config/seatunnel-env.cmd b/config/seatunnel-env.cmd index 0a70c0e1c63..c5c4a504656 100644 --- a/config/seatunnel-env.cmd +++ b/config/seatunnel-env.cmd @@ -26,5 +26,5 @@ if "%METALAKE_ENABLED%" == "" set "META_LAKE_ENABLED=false" REM Type of metalake implementation. if "%METALAKE_TYPE%" == "" set "METALAKE_TYPE=gravitino" -REM Metalake service URL, format: http://host:port/api/metalakes/{metalake_name}/catalogs/{catalog_name} -if "%METALAKE_URL%" == "" set "METALAKE_URL=http://localhost:8090/api/metalakes/default_metalake_name/catalogs/default_catalog_name" \ No newline at end of file +REM Metalake service URL, format: http://host:port/api/metalakes/{metalake_name}/catalogs/ +if "%METALAKE_URL%" == "" set "METALAKE_URL=http://localhost:8090/api/metalakes/default_metalake_name/catalogs/" \ No newline at end of file diff --git a/config/seatunnel-env.sh b/config/seatunnel-env.sh index a0c814a9894..c655fe4f314 100644 --- a/config/seatunnel-env.sh +++ b/config/seatunnel-env.sh @@ -24,5 +24,5 @@ FLINK_HOME=${FLINK_HOME:-/opt/flink} METALAKE_ENABLED=${METALAKE_ENABLED:-false} # Type of metalake implementation. METALAKE_TYPE=${METALAKE_TYPE:-gravitino} -# Metalake service URL, format: http://host:port/api/metalakes/{metalake_name}/catalogs/{catalog_name}. -METALAKE_URL=${METALAKE_URL:-http://localhost:8090/api/metalakes/default_metalake_name/catalogs/default_catalog_name} +# Metalake service URL, format: http://host:port/api/metalakes/{metalake_name}/catalogs/. +METALAKE_URL=${METALAKE_URL:-http://localhost:8090/api/metalakes/default_metalake_name/catalogs/} diff --git a/seatunnel-engine/seatunnel-engine-core/src/main/java/org/apache/seatunnel/engine/core/parse/MultipleTableJobConfigParser.java b/seatunnel-engine/seatunnel-engine-core/src/main/java/org/apache/seatunnel/engine/core/parse/MultipleTableJobConfigParser.java index a0379679fa8..61d50a327b0 100644 --- a/seatunnel-engine/seatunnel-engine-core/src/main/java/org/apache/seatunnel/engine/core/parse/MultipleTableJobConfigParser.java +++ b/seatunnel-engine/seatunnel-engine-core/src/main/java/org/apache/seatunnel/engine/core/parse/MultipleTableJobConfigParser.java @@ -20,6 +20,10 @@ import org.apache.seatunnel.shade.com.google.common.base.Preconditions; import org.apache.seatunnel.shade.com.google.common.collect.Lists; import org.apache.seatunnel.shade.com.typesafe.config.Config; +import org.apache.seatunnel.shade.com.typesafe.config.ConfigValue; +import org.apache.seatunnel.shade.com.typesafe.config.ConfigValueType; +import org.apache.seatunnel.shade.com.typesafe.config.ConfigObject; +import org.apache.seatunnel.shade.com.fasterxml.jackson.databind.JsonNode; import org.apache.seatunnel.api.common.PluginIdentifier; import org.apache.seatunnel.api.configuration.ReadonlyConfig; @@ -159,22 +163,72 @@ public MultipleTableJobConfigParser( this.jobConfig = jobConfig; this.commonPluginJars = commonPluginJars; this.isStartWithSavePoint = isStartWithSavePoint; - //this.seaTunnelJobConfig = ConfigBuilder.of(Paths.get(jobDefineFilePath), variables); boolean metalakeEnabled = Boolean.parseBoolean(System.getenv().getOrDefault("METALAKE_ENABLED", "false")); - Map metalakeConfigMap = new LinkedHashMap<>(); if (metalakeEnabled) { + Config jobConfigTmp = ConfigBuilder.of(Paths.get(jobDefineFilePath), variables); + Map metalakeConfigMap = new LinkedHashMap<>(); + String metalakeType = System.getenv("METALAKE_TYPE"); String metalakeUrl = System.getenv("METALAKE_URL"); - metalakeConfigMap.put("metalake.enabled", true); - metalakeConfigMap.put("metalake.type", metalakeType != null ? metalakeType : ""); - metalakeConfigMap.put("metalake.url", metalakeUrl != null ? metalakeUrl : ""); + ConfigObject sourceObj = jobConfigTmp.getObject("source"); + Set sourceKeys = sourceObj.keySet(); + for (String key : sourceKeys) { + if (jobConfigTmp.hasPath("source."+key+".sourceId")){ + String sourceId = jobConfigTmp.getString("source."+key+".sourceId"); + JsonNode metalakeJson = ... + ConfigObject subConfig = jobConfigTmp.getObject("source." + key); + for (Map.Entry entry : subConfig.entrySet()) { + String subKey = entry.getKey(); + ConfigValue value = entry.getValue(); + + if (value.valueType() == ConfigValueType.STRING) { + String strValue = (String) value.unwrapped(); + if (strValue.startsWith("${") && strValue.endsWith("}")) { + String placeholder = strValue.substring(2, strValue.length() - 1); + + if (metalakeJson.has(placeholder)) { + String replaced = metalakeJson.get(placeholder).asText(); + String finalPath = "source." + key + "." + subKey; + metalakeConfigMap.put(finalPath, replaced); + } + } + } + } + } + } + + ConfigObject sinkObj = jobConfigTmp.getObject("sink"); + Set sinkKeys = sinkObj.keySet(); + for (String key : sinkKeys) { + if (jobConfigTmp.hasPath("sink."+key+".sourceId")){ + String sourceId = jobConfigTmp.getString("source."+key+".sourceId"); + JsonNode metalakeJson = ... + ConfigObject subConfig = jobConfigTmp.getObject("sink." + key); + for (Map.Entry entry : subConfig.entrySet()) { + String subKey = entry.getKey(); + ConfigValue value = entry.getValue(); + + if (value.valueType() == ConfigValueType.STRING) { + String strValue = (String) value.unwrapped(); + if (strValue.startsWith("${") && strValue.endsWith("}")) { + String placeholder = strValue.substring(2, strValue.length() - 1); + + if (metalakeJson.has(placeholder)) { + String replaced = metalakeJson.get(placeholder).asText(); + String finalPath = "sink." + key + "." + subKey; + metalakeConfigMap.put(finalPath, replaced); + } + } + } + } + } + } + Config metalakeConfig = ConfigBuilder.of(metalakeConfigMap); + this.seaTunnelJobConfig = metalakeConfig.withFallback(ConfigBuilder.of(Paths.get(jobDefineFilePath), variables)); } else { - metalakeConfigMap.put("metalake.enabled", false); + this.seaTunnelJobConfig = ConfigBuilder.of(Paths.get(jobDefineFilePath), variables); } - Config metalakeConfig = ConfigBuilder.of(metalakeConfigMap); - this.seaTunnelJobConfig = metalakeConfig.withFallback(ConfigBuilder.of(Paths.get(jobDefineFilePath), variables)); - this.envOptions = ReadonlyConfig.fromConfig(seaTunnelJobConfig.getConfig("env")); this.pipelineCheckpoints = pipelineCheckpoints; ConfigValidator.of(this.envOptions).validate(new EnvOptionRule().optionRule()); From d4ec880deb55b79887fa53b3c14b93a8039fde2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=90=B4=E5=A4=A9=E5=AE=87?= <1599646055@qq.com> Date: Sun, 10 Aug 2025 14:57:16 +0800 Subject: [PATCH 04/39] feat:Add MetalakeClient interface & GravitinoClient implementation; Introduce factory for type-based MetalakeClient creation; Fetch Metalake data by sourceId and replace Config placeholders --- seatunnel-api/pom.xml | 5 + .../api/metalake/GravitinoClient.java | 57 ++++++++ .../api/metalake/MetalakeClient.java | 28 ++++ .../api/metalake/MetalakeClientFactory.java | 46 ++++++ .../parse/MultipleTableJobConfigParser.java | 137 ++++++++++-------- 5 files changed, 213 insertions(+), 60 deletions(-) create mode 100644 seatunnel-api/src/main/java/org/apache/seatunnel/api/metalake/GravitinoClient.java create mode 100644 seatunnel-api/src/main/java/org/apache/seatunnel/api/metalake/MetalakeClient.java create mode 100644 seatunnel-api/src/main/java/org/apache/seatunnel/api/metalake/MetalakeClientFactory.java diff --git a/seatunnel-api/pom.xml b/seatunnel-api/pom.xml index 2ca8eea31b7..d68928899df 100644 --- a/seatunnel-api/pom.xml +++ b/seatunnel-api/pom.xml @@ -41,5 +41,10 @@ ${project.version} optional + + com.squareup.okhttp3 + okhttp + 4.12.0 + diff --git a/seatunnel-api/src/main/java/org/apache/seatunnel/api/metalake/GravitinoClient.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/metalake/GravitinoClient.java new file mode 100644 index 00000000000..c0842e55b96 --- /dev/null +++ b/seatunnel-api/src/main/java/org/apache/seatunnel/api/metalake/GravitinoClient.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.api.metalake; + +import okhttp3.*; +import org.apache.seatunnel.shade.com.fasterxml.jackson.databind.JsonNode; +import org.apache.seatunnel.shade.com.fasterxml.jackson.databind.ObjectMapper; +import java.io.IOException; +import java.net.URI; + + +public class GravitinoClient implements MetalakeClient{ + private final String metalakeUrl; + + public GravitinoClient(String metalakeUrl) { + this.metalakeUrl = metalakeUrl; + } + + @Override + public String getType() { + return "gravitino"; + } + + @Override + public JsonNode getMetaInfo(String sourceId) throws IOException { + OkHttpClient client = new OkHttpClient().newBuilder() + .build(); + MediaType mediaType = MediaType.parse("text/plain"); + RequestBody body = RequestBody.create(mediaType, ""); + Request request = new Request.Builder() + .url(this.metalakeUrl+sourceId) + .method("GET", body) + .addHeader("Accept", "application/vnd.gravitino.v1+json") + //.addHeader("Authorization", "Bearer ") + .build(); + Response response = client.newCall(request).execute(); + ObjectMapper mapper = new ObjectMapper(); + JsonNode rootNode = mapper.readTree(response.body().byteStream()); + JsonNode propertiesNode = rootNode.get("properties"); + return propertiesNode; + } +} diff --git a/seatunnel-api/src/main/java/org/apache/seatunnel/api/metalake/MetalakeClient.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/metalake/MetalakeClient.java new file mode 100644 index 00000000000..362e98cf8ba --- /dev/null +++ b/seatunnel-api/src/main/java/org/apache/seatunnel/api/metalake/MetalakeClient.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.api.metalake; + +import org.apache.seatunnel.shade.com.fasterxml.jackson.databind.JsonNode; + +import java.io.IOException; + +public interface MetalakeClient { + String getType(); + + JsonNode getMetaInfo(String sourceId) throws IOException; +} diff --git a/seatunnel-api/src/main/java/org/apache/seatunnel/api/metalake/MetalakeClientFactory.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/metalake/MetalakeClientFactory.java new file mode 100644 index 00000000000..4fc252fd8b7 --- /dev/null +++ b/seatunnel-api/src/main/java/org/apache/seatunnel/api/metalake/MetalakeClientFactory.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.api.metalake; + +import java.util.HashMap; +import java.util.Map; +import java.util.function.Function; + +public class MetalakeClientFactory { + private static final Map> REGISTRY = new HashMap<>(); + + static { + register("gravitino", GravitinoClient::new); + } + + private MetalakeClientFactory() {} + + public static void register(String type, Function constructor) { + REGISTRY.put(type.toLowerCase(), constructor); + } + + public static MetalakeClient create(String type, String metalakeUrl) { + Function constructor = + REGISTRY.get(type.toLowerCase()); + if (constructor == null) { + throw new IllegalArgumentException( + "Unknown MetalakeClient type: " + type); + } + return constructor.apply(metalakeUrl); + } +} diff --git a/seatunnel-engine/seatunnel-engine-core/src/main/java/org/apache/seatunnel/engine/core/parse/MultipleTableJobConfigParser.java b/seatunnel-engine/seatunnel-engine-core/src/main/java/org/apache/seatunnel/engine/core/parse/MultipleTableJobConfigParser.java index 61d50a327b0..9778797f3d9 100644 --- a/seatunnel-engine/seatunnel-engine-core/src/main/java/org/apache/seatunnel/engine/core/parse/MultipleTableJobConfigParser.java +++ b/seatunnel-engine/seatunnel-engine-core/src/main/java/org/apache/seatunnel/engine/core/parse/MultipleTableJobConfigParser.java @@ -17,6 +17,8 @@ package org.apache.seatunnel.engine.core.parse; +import org.apache.seatunnel.api.metalake.MetalakeClient; +import org.apache.seatunnel.api.metalake.MetalakeClientFactory; import org.apache.seatunnel.shade.com.google.common.base.Preconditions; import org.apache.seatunnel.shade.com.google.common.collect.Lists; import org.apache.seatunnel.shade.com.typesafe.config.Config; @@ -77,6 +79,7 @@ import lombok.extern.slf4j.Slf4j; import scala.Tuple2; +import java.io.IOException; import java.io.Serializable; import java.net.MalformedURLException; import java.net.URL; @@ -166,66 +169,8 @@ public MultipleTableJobConfigParser( boolean metalakeEnabled = Boolean.parseBoolean(System.getenv().getOrDefault("METALAKE_ENABLED", "false")); if (metalakeEnabled) { Config jobConfigTmp = ConfigBuilder.of(Paths.get(jobDefineFilePath), variables); - Map metalakeConfigMap = new LinkedHashMap<>(); - - String metalakeType = System.getenv("METALAKE_TYPE"); - String metalakeUrl = System.getenv("METALAKE_URL"); - - ConfigObject sourceObj = jobConfigTmp.getObject("source"); - Set sourceKeys = sourceObj.keySet(); - for (String key : sourceKeys) { - if (jobConfigTmp.hasPath("source."+key+".sourceId")){ - String sourceId = jobConfigTmp.getString("source."+key+".sourceId"); - JsonNode metalakeJson = ... - ConfigObject subConfig = jobConfigTmp.getObject("source." + key); - for (Map.Entry entry : subConfig.entrySet()) { - String subKey = entry.getKey(); - ConfigValue value = entry.getValue(); - - if (value.valueType() == ConfigValueType.STRING) { - String strValue = (String) value.unwrapped(); - if (strValue.startsWith("${") && strValue.endsWith("}")) { - String placeholder = strValue.substring(2, strValue.length() - 1); - - if (metalakeJson.has(placeholder)) { - String replaced = metalakeJson.get(placeholder).asText(); - String finalPath = "source." + key + "." + subKey; - metalakeConfigMap.put(finalPath, replaced); - } - } - } - } - } - } - - ConfigObject sinkObj = jobConfigTmp.getObject("sink"); - Set sinkKeys = sinkObj.keySet(); - for (String key : sinkKeys) { - if (jobConfigTmp.hasPath("sink."+key+".sourceId")){ - String sourceId = jobConfigTmp.getString("source."+key+".sourceId"); - JsonNode metalakeJson = ... - ConfigObject subConfig = jobConfigTmp.getObject("sink." + key); - for (Map.Entry entry : subConfig.entrySet()) { - String subKey = entry.getKey(); - ConfigValue value = entry.getValue(); - - if (value.valueType() == ConfigValueType.STRING) { - String strValue = (String) value.unwrapped(); - if (strValue.startsWith("${") && strValue.endsWith("}")) { - String placeholder = strValue.substring(2, strValue.length() - 1); - - if (metalakeJson.has(placeholder)) { - String replaced = metalakeJson.get(placeholder).asText(); - String finalPath = "sink." + key + "." + subKey; - metalakeConfigMap.put(finalPath, replaced); - } - } - } - } - } - } - Config metalakeConfig = ConfigBuilder.of(metalakeConfigMap); - this.seaTunnelJobConfig = metalakeConfig.withFallback(ConfigBuilder.of(Paths.get(jobDefineFilePath), variables)); + Config metalakeConfig = getMetalakeConfig(jobConfigTmp); + this.seaTunnelJobConfig = metalakeConfig.withFallback(jobConfigTmp); } else { this.seaTunnelJobConfig = ConfigBuilder.of(Paths.get(jobDefineFilePath), variables); } @@ -884,4 +829,76 @@ private ChangeStreamTableSourceCheckpoint getSourceCheckpoint( .collect(Collectors.toList()); return new ChangeStreamTableSourceCheckpoint(coordinatorState, subtaskState); } + + private Config getMetalakeConfig(Config jobConfigTmp) { + Map metalakeConfigMap = new LinkedHashMap<>(); + String metalakeType = System.getenv("METALAKE_TYPE"); + String metalakeUrl = System.getenv("METALAKE_URL"); + + MetalakeClient metalakeClient = MetalakeClientFactory.create(metalakeType, metalakeUrl); + + try { + ConfigObject sourceObj = jobConfigTmp.getObject("source"); + Set sourceKeys = sourceObj.keySet(); + + for (String key : sourceKeys) { + if (jobConfigTmp.hasPath("source." + key + ".sourceId")) { + String sourceId = jobConfigTmp.getString("source." + key + ".sourceId"); + JsonNode metalakeJson = metalakeClient.getMetaInfo(sourceId); + ConfigObject subConfig = jobConfigTmp.getObject("source." + key); + for (Map.Entry entry : subConfig.entrySet()) { + String subKey = entry.getKey(); + ConfigValue value = entry.getValue(); + + if (value.valueType() == ConfigValueType.STRING) { + String strValue = (String) value.unwrapped(); + if (strValue.startsWith("${") && strValue.endsWith("}")) { + String placeholder = strValue.substring(2, strValue.length() - 1); + + if (metalakeJson.has(placeholder)) { + String replaced = metalakeJson.get(placeholder).asText(); + String finalPath = "source." + key + "." + subKey; + metalakeConfigMap.put(finalPath, replaced); + } + } + } + } + } + } + } catch (IOException e){ + log.error("Fail to get MetaInfo, metalakeUrl: {}", metalakeUrl, e); + } + + try { + ConfigObject sinkObj = jobConfigTmp.getObject("sink"); + Set sinkKeys = sinkObj.keySet(); + for (String key : sinkKeys) { + if (jobConfigTmp.hasPath("sink." + key + ".sourceId")) { + String sourceId = jobConfigTmp.getString("source." + key + ".sourceId"); + JsonNode metalakeJson = metalakeClient.getMetaInfo(sourceId); + ConfigObject subConfig = jobConfigTmp.getObject("sink." + key); + for (Map.Entry entry : subConfig.entrySet()) { + String subKey = entry.getKey(); + ConfigValue value = entry.getValue(); + + if (value.valueType() == ConfigValueType.STRING) { + String strValue = (String) value.unwrapped(); + if (strValue.startsWith("${") && strValue.endsWith("}")) { + String placeholder = strValue.substring(2, strValue.length() - 1); + + if (metalakeJson.has(placeholder)) { + String replaced = metalakeJson.get(placeholder).asText(); + String finalPath = "sink." + key + "." + subKey; + metalakeConfigMap.put(finalPath, replaced); + } + } + } + } + } + } + } catch (IOException e){ + log.error("Fail to get MetaInfo, metalakeUrl: {}", metalakeUrl, e); + } + return ConfigBuilder.of(metalakeConfigMap); + } } From d2a84c70c73fce92f550d1cf395b4927c7c342ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=90=B4=E5=A4=A9=E5=AE=87?= <1599646055@qq.com> Date: Thu, 21 Aug 2025 19:30:30 +0800 Subject: [PATCH 05/39] feat:apply spotless code style --- .../api/metalake/GravitinoClient.java | 22 +++++++++---------- .../api/metalake/MetalakeClientFactory.java | 6 ++--- .../parse/MultipleTableJobConfigParser.java | 15 +++++++------ 3 files changed, 20 insertions(+), 23 deletions(-) diff --git a/seatunnel-api/src/main/java/org/apache/seatunnel/api/metalake/GravitinoClient.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/metalake/GravitinoClient.java index c0842e55b96..3c46ff6c50f 100644 --- a/seatunnel-api/src/main/java/org/apache/seatunnel/api/metalake/GravitinoClient.java +++ b/seatunnel-api/src/main/java/org/apache/seatunnel/api/metalake/GravitinoClient.java @@ -17,14 +17,12 @@ package org.apache.seatunnel.api.metalake; -import okhttp3.*; import org.apache.seatunnel.shade.com.fasterxml.jackson.databind.JsonNode; import org.apache.seatunnel.shade.com.fasterxml.jackson.databind.ObjectMapper; -import java.io.IOException; -import java.net.URI; +import java.io.IOException; -public class GravitinoClient implements MetalakeClient{ +public class GravitinoClient implements MetalakeClient { private final String metalakeUrl; public GravitinoClient(String metalakeUrl) { @@ -38,16 +36,16 @@ public String getType() { @Override public JsonNode getMetaInfo(String sourceId) throws IOException { - OkHttpClient client = new OkHttpClient().newBuilder() - .build(); + OkHttpClient client = new OkHttpClient().newBuilder().build(); MediaType mediaType = MediaType.parse("text/plain"); RequestBody body = RequestBody.create(mediaType, ""); - Request request = new Request.Builder() - .url(this.metalakeUrl+sourceId) - .method("GET", body) - .addHeader("Accept", "application/vnd.gravitino.v1+json") - //.addHeader("Authorization", "Bearer ") - .build(); + Request request = + new Request.Builder() + .url(this.metalakeUrl + sourceId) + .method("GET", body) + .addHeader("Accept", "application/vnd.gravitino.v1+json") + // .addHeader("Authorization", "Bearer ") + .build(); Response response = client.newCall(request).execute(); ObjectMapper mapper = new ObjectMapper(); JsonNode rootNode = mapper.readTree(response.body().byteStream()); diff --git a/seatunnel-api/src/main/java/org/apache/seatunnel/api/metalake/MetalakeClientFactory.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/metalake/MetalakeClientFactory.java index 4fc252fd8b7..e1d2edfd11f 100644 --- a/seatunnel-api/src/main/java/org/apache/seatunnel/api/metalake/MetalakeClientFactory.java +++ b/seatunnel-api/src/main/java/org/apache/seatunnel/api/metalake/MetalakeClientFactory.java @@ -35,11 +35,9 @@ public static void register(String type, Function constr } public static MetalakeClient create(String type, String metalakeUrl) { - Function constructor = - REGISTRY.get(type.toLowerCase()); + Function constructor = REGISTRY.get(type.toLowerCase()); if (constructor == null) { - throw new IllegalArgumentException( - "Unknown MetalakeClient type: " + type); + throw new IllegalArgumentException("Unknown MetalakeClient type: " + type); } return constructor.apply(metalakeUrl); } diff --git a/seatunnel-engine/seatunnel-engine-core/src/main/java/org/apache/seatunnel/engine/core/parse/MultipleTableJobConfigParser.java b/seatunnel-engine/seatunnel-engine-core/src/main/java/org/apache/seatunnel/engine/core/parse/MultipleTableJobConfigParser.java index 9778797f3d9..87b5b33a925 100644 --- a/seatunnel-engine/seatunnel-engine-core/src/main/java/org/apache/seatunnel/engine/core/parse/MultipleTableJobConfigParser.java +++ b/seatunnel-engine/seatunnel-engine-core/src/main/java/org/apache/seatunnel/engine/core/parse/MultipleTableJobConfigParser.java @@ -17,19 +17,19 @@ package org.apache.seatunnel.engine.core.parse; -import org.apache.seatunnel.api.metalake.MetalakeClient; -import org.apache.seatunnel.api.metalake.MetalakeClientFactory; +import org.apache.seatunnel.shade.com.fasterxml.jackson.databind.JsonNode; import org.apache.seatunnel.shade.com.google.common.base.Preconditions; import org.apache.seatunnel.shade.com.google.common.collect.Lists; import org.apache.seatunnel.shade.com.typesafe.config.Config; +import org.apache.seatunnel.shade.com.typesafe.config.ConfigObject; import org.apache.seatunnel.shade.com.typesafe.config.ConfigValue; import org.apache.seatunnel.shade.com.typesafe.config.ConfigValueType; -import org.apache.seatunnel.shade.com.typesafe.config.ConfigObject; -import org.apache.seatunnel.shade.com.fasterxml.jackson.databind.JsonNode; import org.apache.seatunnel.api.common.PluginIdentifier; import org.apache.seatunnel.api.configuration.ReadonlyConfig; import org.apache.seatunnel.api.configuration.util.ConfigValidator; +import org.apache.seatunnel.api.metalake.MetalakeClient; +import org.apache.seatunnel.api.metalake.MetalakeClientFactory; import org.apache.seatunnel.api.options.ConnectorCommonOptions; import org.apache.seatunnel.api.options.EnvCommonOptions; import org.apache.seatunnel.api.options.EnvOptionRule; @@ -166,7 +166,8 @@ public MultipleTableJobConfigParser( this.jobConfig = jobConfig; this.commonPluginJars = commonPluginJars; this.isStartWithSavePoint = isStartWithSavePoint; - boolean metalakeEnabled = Boolean.parseBoolean(System.getenv().getOrDefault("METALAKE_ENABLED", "false")); + boolean metalakeEnabled = + Boolean.parseBoolean(System.getenv().getOrDefault("METALAKE_ENABLED", "false")); if (metalakeEnabled) { Config jobConfigTmp = ConfigBuilder.of(Paths.get(jobDefineFilePath), variables); Config metalakeConfig = getMetalakeConfig(jobConfigTmp); @@ -865,7 +866,7 @@ private Config getMetalakeConfig(Config jobConfigTmp) { } } } - } catch (IOException e){ + } catch (IOException e) { log.error("Fail to get MetaInfo, metalakeUrl: {}", metalakeUrl, e); } @@ -896,7 +897,7 @@ private Config getMetalakeConfig(Config jobConfigTmp) { } } } - } catch (IOException e){ + } catch (IOException e) { log.error("Fail to get MetaInfo, metalakeUrl: {}", metalakeUrl, e); } return ConfigBuilder.of(metalakeConfigMap); From f32ac80f4309a0a4019730c95c922601e6de8cb0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=90=B4=E5=A4=A9=E5=AE=87?= <1599646055@qq.com> Date: Thu, 21 Aug 2025 19:37:07 +0800 Subject: [PATCH 06/39] feat: replace wildcard imports with explicit imports for okhttp3 --- .../org/apache/seatunnel/api/metalake/GravitinoClient.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/seatunnel-api/src/main/java/org/apache/seatunnel/api/metalake/GravitinoClient.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/metalake/GravitinoClient.java index 3c46ff6c50f..cde48f95ef7 100644 --- a/seatunnel-api/src/main/java/org/apache/seatunnel/api/metalake/GravitinoClient.java +++ b/seatunnel-api/src/main/java/org/apache/seatunnel/api/metalake/GravitinoClient.java @@ -20,6 +20,12 @@ import org.apache.seatunnel.shade.com.fasterxml.jackson.databind.JsonNode; import org.apache.seatunnel.shade.com.fasterxml.jackson.databind.ObjectMapper; +import okhttp3.MediaType; +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.RequestBody; +import okhttp3.Response; + import java.io.IOException; public class GravitinoClient implements MetalakeClient { From d5450474af23e99ae424ff113c717ae91abab9b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=90=B4=E5=A4=A9=E5=AE=87?= <1599646055@qq.com> Date: Fri, 22 Aug 2025 20:13:58 +0800 Subject: [PATCH 07/39] fix: delete useless EnvOption --- .../api/options/EnvCommonOptions.java | 18 ------------------ .../seatunnel/api/options/EnvOptionRule.java | 5 +---- 2 files changed, 1 insertion(+), 22 deletions(-) diff --git a/seatunnel-api/src/main/java/org/apache/seatunnel/api/options/EnvCommonOptions.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/options/EnvCommonOptions.java index d8cd4b364c2..663727caf82 100644 --- a/seatunnel-api/src/main/java/org/apache/seatunnel/api/options/EnvCommonOptions.java +++ b/seatunnel-api/src/main/java/org/apache/seatunnel/api/options/EnvCommonOptions.java @@ -107,22 +107,4 @@ public class EnvCommonOptions { .mapType() .noDefaultValue() .withDescription("Define the worker where the job runs by tag"); - - public static Option METALAKE_ENABLED = - Options.key("metalake.enabled") - .booleanType() - .defaultValue(false) - .withDescription("Whether to enable metalake"); - - public static Option METALAKE_TYPE = - Options.key("metalake.type") - .stringType() - .noDefaultValue() - .withDescription("Type of metalake implementation"); - - public static Option METALAKE_URL = - Options.key("metalake.url") - .stringType() - .noDefaultValue() - .withDescription("Metalake service URL"); } diff --git a/seatunnel-api/src/main/java/org/apache/seatunnel/api/options/EnvOptionRule.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/options/EnvOptionRule.java index cb5273ca778..0c13cba4dfe 100644 --- a/seatunnel-api/src/main/java/org/apache/seatunnel/api/options/EnvOptionRule.java +++ b/seatunnel-api/src/main/java/org/apache/seatunnel/api/options/EnvOptionRule.java @@ -46,10 +46,7 @@ public OptionRule optionRule() { EnvCommonOptions.READ_LIMIT_BYTES_PER_SECOND, EnvCommonOptions.SAVEMODE_EXECUTE_LOCATION, EnvCommonOptions.CUSTOM_PARAMETERS, - EnvCommonOptions.NODE_TAG_FILTER, - EnvCommonOptions.METALAKE_ENABLED, - EnvCommonOptions.METALAKE_TYPE, - EnvCommonOptions.METALAKE_URL) + EnvCommonOptions.NODE_TAG_FILTER) .build(); } } From 84491daf149c33785f4aa846e06182984e2a721a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=90=B4=E5=A4=A9=E5=AE=87?= <1599646055@qq.com> Date: Mon, 25 Aug 2025 23:52:35 +0800 Subject: [PATCH 08/39] feat: Integration Test for Metalake --- .../connector-seatunnel-e2e-base/pom.xml | 6 + .../engine/e2e/metalake/JdbcCase.java | 66 +++ .../engine/e2e/metalake/MetalakeIT.java | 467 ++++++++++++++++++ .../mysql_to_console_with_metalake.conf | 28 ++ 4 files changed, 567 insertions(+) create mode 100644 seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/metalake/JdbcCase.java create mode 100644 seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/metalake/MetalakeIT.java create mode 100644 seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/mysql_to_console_with_metalake.conf diff --git a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/pom.xml b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/pom.xml index d8f18347ec0..08929ca74be 100644 --- a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/pom.xml +++ b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/pom.xml @@ -151,6 +151,12 @@ ${project.version} test + + org.testcontainers + mysql + 1.17.6 + test + diff --git a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/metalake/JdbcCase.java b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/metalake/JdbcCase.java new file mode 100644 index 00000000000..ea5c4a0bda4 --- /dev/null +++ b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/metalake/JdbcCase.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.engine.e2e.metalake; + +import org.apache.seatunnel.api.table.type.SeaTunnelRow; + +import org.apache.commons.lang3.tuple.Pair; + +import lombok.Builder; +import lombok.Getter; +import lombok.Setter; + +import java.util.List; +import java.util.Map; + +@Builder +@Setter +@Getter +public class JdbcCase { + private String dockerImage; + private String networkAliases; + private String driverClass; + private String host; + private String userName; + private String password; + private int port; + private int localPort; + private String database; + private String schema; + private String sourceTable; + private String sinkTable; + private String jdbcTemplate; + private String jdbcUrl; + private String createSql; + private String sinkCreateSql; + private String additionalSqlOnSource; + private String additionalSqlOnSink; + private String insertSql; + private List configFile; + private Pair> testData; + private Map containerEnv; + private boolean useSaveModeCreateTable; + + private String catalogDatabase; + private String catalogSchema; + private String catalogTable; + + // The full path of the table created when initializing data + // According to whether jdbc api supports setting + private String tablePathFullName; +} \ No newline at end of file diff --git a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/metalake/MetalakeIT.java b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/metalake/MetalakeIT.java new file mode 100644 index 00000000000..f7647a36658 --- /dev/null +++ b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/metalake/MetalakeIT.java @@ -0,0 +1,467 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.engine.e2e.metalake; + +import com.github.dockerjava.api.DockerClient; +import com.github.dockerjava.api.model.Image; +import io.restassured.RestAssured; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.tuple.Pair; +import org.apache.seatunnel.api.table.catalog.Catalog; +import org.apache.seatunnel.api.table.type.SeaTunnelRow; +import org.apache.seatunnel.common.exception.SeaTunnelRuntimeException; +import org.apache.seatunnel.common.utils.ExceptionUtils; +import org.apache.seatunnel.engine.e2e.SeaTunnelEngineContainer; +import org.apache.seatunnel.shade.com.google.common.collect.Lists; +import org.junit.jupiter.api.*; +import org.testcontainers.DockerClientFactory; +import org.testcontainers.containers.Container; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.Network; +import org.testcontainers.containers.output.Slf4jLogConsumer; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.images.PullPolicy; +import org.testcontainers.lifecycle.Startables; +import org.testcontainers.utility.DockerImageName; +import org.testcontainers.utility.DockerLoggerFactory; +import org.testcontainers.containers.MySQLContainer; + +import java.io.IOException; +import java.math.BigDecimal; +import java.sql.*; +import java.sql.Date; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.LocalTime; +import java.util.*; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.awaitility.Awaitility.given; + +public class MetalakeIT extends SeaTunnelEngineContainer { + + protected GenericContainer dbServer; + + protected GenericContainer gravitinoServer; + + protected JdbcCase jdbcCase; + + protected Connection connection; + + protected Catalog catalog; + + protected DockerClient dockerClient = DockerClientFactory.lazyClient(); + + protected static final String HOST = "HOST"; + + private static final String MYSQL_IMAGE = "mysql:8.0"; + private static final String MYSQL_CONTAINER_HOST = "mysql-e2e"; + private static final String MYSQL_DATABASE = "seatunnel"; + private static final String MYSQL_SOURCE = "source"; + private static final String MYSQL_SINK = "sink"; + private static final String CATALOG_DATABASE = "catalog_database"; + + private static final String MYSQL_USERNAME = "root"; + private static final String MYSQL_PASSWORD = "Abc!@#135_seatunnel"; + private static final int MYSQL_PORT = 3306; + private static final String MYSQL_URL = "jdbc:mysql://" + HOST + ":%s/%s?useSSL=false"; + private static final String URL = "jdbc:mysql://" + HOST + ":3306/seatunnel"; + + private static final String SQL = "select * from seatunnel.source"; + + private static final String DRIVER_CLASS = "com.mysql.cj.jdbc.Driver"; + + Network NETWORK = + Network.builder() + .createNetworkCmdModifier(cmd -> cmd.withName("SEATUNNEL-" + UUID.randomUUID())) + .enableIpv6(false) + .build(); + + private static final List CONFIG_FILE = + Lists.newArrayList( + "/mysql_to_mysql_with_metalake.conf"); + private static final String CREATE_SQL = + "CREATE TABLE IF NOT EXISTS %s\n" + + "(\n" + + " `c-bit_1` bit(1) DEFAULT NULL,\n" + + " `c_bit_8` bit(8) DEFAULT NULL,\n" + + " `c_bit_16` bit(16) DEFAULT NULL,\n" + + " `c_bit_32` bit(32) DEFAULT NULL,\n" + + " `c_bit_64` bit(64) DEFAULT NULL,\n" + + " `c_bigint_30` BIGINT(40) unsigned DEFAULT NULL,\n" + + " UNIQUE (c_bigint_30)\n" + + ");"; + + @BeforeAll + @Override + public void startUp() throws Exception{ + super.startUp(); + server.execInContainer( + "bash", + "-c", + "export METALAKE_ENABLED=${METALAKE_ENABLED:-true} && " + + "export METALAKE_TYPE=${METALAKE_TYPE:-gravitino} && " + + "export METALAKE_URL=${METALAKE_URL:-http://localhost:8090/api/metalakes/test_metalake/catalogs/} && " + + "mkdir -p /tmp/seatunnel/plugins/Jdbc/lib && cd /tmp/seatunnel/plugins/Jdbc/lib && wget " + + driverUrl() + + " --no-check-certificate"); + dbServer = initContainer().withImagePullPolicy(PullPolicy.alwaysPull()); + + gravitinoServer = new GenericContainer<>("apache/gravitino:latest") + .withExposedPorts(8090) + .withImagePullPolicy(PullPolicy.alwaysPull()) + .withNetwork(NETWORK) + .withNetworkAliases("gravitino") + .withLogConsumer(new Slf4jLogConsumer(DockerLoggerFactory.getLogger("apache/gravitino:latest"))); + + Startables.deepStart(Stream.of(dbServer, gravitinoServer)).join(); + + jdbcCase = getJdbcCase(); + + given().ignoreExceptions() + .await() + .atMost(360, TimeUnit.SECONDS) + .untilAsserted(() -> this.initializeJdbcConnection(jdbcCase.getJdbcUrl())); + + given().ignoreExceptions() + .await() + .atMost(60, TimeUnit.SECONDS) + .untilAsserted(() -> { + String url = "http://" + gravitinoServer.getHost() + ":" + gravitinoServer.getMappedPort(8090); + int statusCode = RestAssured.get(url + "/health").statusCode(); + Assertions.assertEquals(200, statusCode); + }); + + String gravitinoUrl = "http://" + gravitinoServer.getHost() + ":" + gravitinoServer.getMappedPort(8090); + + String payload = "{" + + "\"name\": \"test_catalog\"," + + "\"type\": \"relational\"," + + "\"provider\": \"mysql\"," + + "\"comment\": \"Catalog for integration test\"," + + "\"properties\": {" + + " \"user\": \"root\"," + + " \"password\": \"Abc!@#135_seatunnel\"" + + " }" + + "}"; + + RestAssured.given() + .contentType("application/json") + .body(payload) + .when() + .post(gravitinoUrl + "/api/metalakes/test_metalake/catalogs/test_catalog") + .then() + .statusCode(200); + + createNeededTables(); + insertTestData(); + } + + @AfterAll + @Override + public void tearDown() throws Exception { + if (catalog != null) { + catalog.close(); + } + + if (connection != null) { + connection.close(); + } + + if (dbServer != null) { + dbServer.close(); + try { + dockerClient.removeImageCmd(dbServer.getDockerImageName()).exec(); + } catch (Exception ignored) { + //log.warn("Failed to delete the image. Another container may be in use", ignored); + ignored.printStackTrace(); + } + } + + if (gravitinoServer != null) { + gravitinoServer.close(); + try { + dockerClient.removeImageCmd(gravitinoServer.getDockerImageName()).exec(); + } catch (Exception ignored) { + //log.warn("Failed to delete the image. Another container may be in use", ignored); + ignored.printStackTrace(); + } + } + super.tearDown(); + } + + @Test + public void TestMetalake() throws IOException, InterruptedException { + Container.ExecResult execResult = executeSeaTunnelJob("/mysql_to_mysql_with_metalake.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + } + + String driverUrl() { + return "https://repo1.maven.org/maven2/com/mysql/mysql-connector-j/8.0.32/mysql-connector-j-8.0.32.jar"; + } + + protected GenericContainer initContainer() { + DockerImageName imageName = DockerImageName.parse(MYSQL_IMAGE); + + GenericContainer container = + new MySQLContainer<>(imageName) + .withUsername(MYSQL_USERNAME) + .withPassword(MYSQL_PASSWORD) + .withDatabaseName(MYSQL_DATABASE) + .withNetwork(NETWORK) + .withNetworkAliases(MYSQL_CONTAINER_HOST) + .withExposedPorts(MYSQL_PORT) + .waitingFor(Wait.forHealthcheck()) + .withLogConsumer( + new Slf4jLogConsumer(DockerLoggerFactory.getLogger(MYSQL_IMAGE))); + + container.setPortBindings( + Lists.newArrayList(String.format("%s:%s", MYSQL_PORT, MYSQL_PORT))); + + return container; + } + + JdbcCase getJdbcCase() { + Map containerEnv = new HashMap<>(); + String jdbcUrl = String.format(MYSQL_URL, MYSQL_PORT, MYSQL_DATABASE); + Pair> testDataSet = initTestData(); + String[] fieldNames = testDataSet.getKey(); + + String insertSql = insertTable(MYSQL_DATABASE, MYSQL_SOURCE, fieldNames); + + return JdbcCase.builder() + .dockerImage(MYSQL_IMAGE) + .networkAliases(MYSQL_CONTAINER_HOST) + .containerEnv(containerEnv) + .driverClass(DRIVER_CLASS) + .host(HOST) + .port(MYSQL_PORT) + .localPort(MYSQL_PORT) + .jdbcTemplate(MYSQL_URL) + .jdbcUrl(jdbcUrl) + .userName(MYSQL_USERNAME) + .password(MYSQL_PASSWORD) + .database(MYSQL_DATABASE) + .sourceTable(MYSQL_SOURCE) + .sinkTable(MYSQL_SINK) + .createSql(CREATE_SQL) + .configFile(CONFIG_FILE) + .insertSql(insertSql) + .testData(testDataSet) + .catalogDatabase(CATALOG_DATABASE) + .catalogTable(MYSQL_SINK) + .tablePathFullName(MYSQL_DATABASE + "." + MYSQL_SOURCE) + .build(); + } + + protected void initializeJdbcConnection(String jdbcUrl) + throws SQLException, InstantiationException, IllegalAccessException { + Driver driver = (Driver) loadDriverClass().newInstance(); + Properties props = new Properties(); + + if (StringUtils.isNotBlank(jdbcCase.getUserName())) { + props.put("user", jdbcCase.getUserName()); + } + + if (StringUtils.isNotBlank(jdbcCase.getPassword())) { + props.put("password", jdbcCase.getPassword()); + } + + if (dbServer != null) { + jdbcUrl = jdbcUrl.replace(HOST, dbServer.getHost()); + } + + this.connection = driver.connect(jdbcUrl, props); + connection.setAutoCommit(false); + } + + protected void createNeededTables() { + try (Statement statement = connection.createStatement()) { + String createTemplate = jdbcCase.getCreateSql(); + + String createSource = + String.format( + createTemplate, + buildTableInfoWithSchema( + jdbcCase.getDatabase(), + jdbcCase.getSchema(), + jdbcCase.getSourceTable())); + statement.execute(createSource); + + if (jdbcCase.getAdditionalSqlOnSource() != null) { + String additionalSql = + String.format( + jdbcCase.getAdditionalSqlOnSource(), + buildTableInfoWithSchema( + jdbcCase.getDatabase(), + jdbcCase.getSchema(), + jdbcCase.getSourceTable())); + statement.execute(additionalSql); + } + + if (!jdbcCase.isUseSaveModeCreateTable()) { + if (jdbcCase.getSinkCreateSql() != null) { + createTemplate = jdbcCase.getSinkCreateSql(); + } + String createSink = + String.format( + createTemplate, + buildTableInfoWithSchema( + jdbcCase.getDatabase(), + jdbcCase.getSchema(), + jdbcCase.getSinkTable())); + statement.execute(createSink); + } + + if (jdbcCase.getAdditionalSqlOnSink() != null) { + String additionalSql = + String.format( + jdbcCase.getAdditionalSqlOnSink(), + buildTableInfoWithSchema( + jdbcCase.getDatabase(), + jdbcCase.getSchema(), + jdbcCase.getSinkTable())); + statement.execute(additionalSql); + } + + connection.commit(); + } catch (Exception exception) { + //log.error(ExceptionUtils.getMessage(exception)); + //throw new SeaTunnelRuntimeException(JdbcITErrorCode.CREATE_TABLE_FAILED, exception); + exception.printStackTrace(); + } + } + + protected void insertTestData() { + try (PreparedStatement preparedStatement = + connection.prepareStatement(jdbcCase.getInsertSql())) { + + List rows = jdbcCase.getTestData().getValue(); + + for (SeaTunnelRow row : rows) { + for (int index = 0; index < row.getArity(); index++) { + preparedStatement.setObject(index + 1, row.getField(index)); + } + preparedStatement.addBatch(); + } + + preparedStatement.executeBatch(); + + connection.commit(); + } catch (Exception exception) { + //log.error(ExceptionUtils.getMessage(exception)); + //throw new SeaTunnelRuntimeException(JdbcITErrorCode.INSERT_DATA_FAILED, exception); + exception.printStackTrace(); + } + } + + Pair> initTestData() { + String[] fieldNames = + new String[] { + "c-bit_1", + "c_bit_8", + "c_bit_16", + "c_bit_32", + "c_bit_64", + "c_bigint_30", + }; + + List rows = new ArrayList<>(); + BigDecimal bigintValue = new BigDecimal("2844674407371055000"); + BigDecimal decimalValue = new BigDecimal("999999999999999999999999999899"); + for (int i = 0; i < 100; i++) { + byte byteArr = Integer.valueOf(i).byteValue(); + SeaTunnelRow row; + if (i == 99) { + row = + new SeaTunnelRow( + new Object[] { + null, + null, + null, + null, + null, + // https://github.com/apache/seatunnel/issues/5559 this value + // cannot set null, this null + // value column's row will be lost in + // jdbc_mysql_source_and_sink_parallel.conf,jdbc_mysql_source_and_sink_parallel_upper_lower.conf. + bigintValue.add(BigDecimal.valueOf(i)), + }); + } else { + row = + new SeaTunnelRow( + new Object[] { + i % 2 == 0 ? (byte) 1 : (byte) 0, + new byte[] {byteArr}, + new byte[] {byteArr, byteArr}, + new byte[] {byteArr, byteArr, byteArr, byteArr}, + new byte[] { + byteArr, byteArr, byteArr, byteArr, byteArr, byteArr, + byteArr, byteArr + }, + bigintValue.add(BigDecimal.valueOf(i)), + }); + } + rows.add(row); + } + + return Pair.of(fieldNames, rows); + } + + public String insertTable(String schema, String table, String... fields) { + String columns = + Arrays.stream(fields).map(this::quoteIdentifier).collect(Collectors.joining(", ")); + String placeholders = Arrays.stream(fields).map(f -> "?").collect(Collectors.joining(", ")); + + return "INSERT INTO " + + buildTableInfoWithSchema(schema, table) + + " (" + + columns + + " )" + + " VALUES (" + + placeholders + + ")"; + } + + protected Class loadDriverClass() { + try { + return Class.forName(jdbcCase.getDriverClass()); + } catch (Exception e) { + throw new RuntimeException( + "Failed to load driver class: " + jdbcCase.getDriverClass(), e); + } + } + + protected String buildTableInfoWithSchema(String database, String schema, String table) { + return buildTableInfoWithSchema(database, table); + } + + public String buildTableInfoWithSchema(String schema, String table) { + if (StringUtils.isNotBlank(schema)) { + return quoteIdentifier(schema) + "." + quoteIdentifier(table); + } else { + return quoteIdentifier(table); + } + } + + public String quoteIdentifier(String field) { + return "`" + field + "`"; + } +} diff --git a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/mysql_to_console_with_metalake.conf b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/mysql_to_console_with_metalake.conf new file mode 100644 index 00000000000..241e33722eb --- /dev/null +++ b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/mysql_to_console_with_metalake.conf @@ -0,0 +1,28 @@ +env { + execution.parallelism = 1 + job.mode = "BATCH" +} + +source { + Jdbc { + url = "jdbc:mysql://mysql-e2e:3306/seatunnel" + driver = "com.mysql.cj.jdbc.Driver" + connection_check_timeout_sec = 100 + sourceId = "test_catalog" + user = "${user}" + password = "${password}" + query = "select * from source" + properties { + useSSL=false + rewriteBatchedStatements=true + } + } +} + +transform { +} + +sink { + Console { + } +} \ No newline at end of file From 768815b37068d7c5e49cf2c854d928d96a039e9a Mon Sep 17 00:00:00 2001 From: wtybxqm <1599646055@qq.com> Date: Wed, 27 Aug 2025 19:08:13 +0800 Subject: [PATCH 09/39] feat: Integration Test --- .../api/metalake/GravitinoClient.java | 3 +- .../connector-seatunnel-e2e-base/pom.xml | 6 + .../engine/e2e/metalake/JdbcCase.java | 2 +- .../engine/e2e/metalake/MetalakeIT.java | 258 +++++++++++------- .../mysql_to_console_with_metalake.conf | 10 +- .../parse/MultipleTableJobConfigParser.java | 17 +- 6 files changed, 178 insertions(+), 118 deletions(-) diff --git a/seatunnel-api/src/main/java/org/apache/seatunnel/api/metalake/GravitinoClient.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/metalake/GravitinoClient.java index cde48f95ef7..cdb0563ddf8 100644 --- a/seatunnel-api/src/main/java/org/apache/seatunnel/api/metalake/GravitinoClient.java +++ b/seatunnel-api/src/main/java/org/apache/seatunnel/api/metalake/GravitinoClient.java @@ -55,7 +55,8 @@ public JsonNode getMetaInfo(String sourceId) throws IOException { Response response = client.newCall(request).execute(); ObjectMapper mapper = new ObjectMapper(); JsonNode rootNode = mapper.readTree(response.body().byteStream()); - JsonNode propertiesNode = rootNode.get("properties"); + JsonNode catalogNode = rootNode.get("catalog"); + JsonNode propertiesNode = catalogNode.get("properties"); return propertiesNode; } } diff --git a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/pom.xml b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/pom.xml index 08929ca74be..b1d84ce6bae 100644 --- a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/pom.xml +++ b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/pom.xml @@ -157,6 +157,12 @@ 1.17.6 test + + mysql + mysql-connector-java + 8.0.27 + test + diff --git a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/metalake/JdbcCase.java b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/metalake/JdbcCase.java index ea5c4a0bda4..68594cd5764 100644 --- a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/metalake/JdbcCase.java +++ b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/metalake/JdbcCase.java @@ -63,4 +63,4 @@ public class JdbcCase { // The full path of the table created when initializing data // According to whether jdbc api supports setting private String tablePathFullName; -} \ No newline at end of file +} diff --git a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/metalake/MetalakeIT.java b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/metalake/MetalakeIT.java index f7647a36658..3ad5920f9b4 100644 --- a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/metalake/MetalakeIT.java +++ b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/metalake/MetalakeIT.java @@ -17,21 +17,23 @@ package org.apache.seatunnel.engine.e2e.metalake; -import com.github.dockerjava.api.DockerClient; -import com.github.dockerjava.api.model.Image; -import io.restassured.RestAssured; -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.tuple.Pair; +import org.apache.seatunnel.shade.com.google.common.collect.Lists; + import org.apache.seatunnel.api.table.catalog.Catalog; import org.apache.seatunnel.api.table.type.SeaTunnelRow; -import org.apache.seatunnel.common.exception.SeaTunnelRuntimeException; -import org.apache.seatunnel.common.utils.ExceptionUtils; import org.apache.seatunnel.engine.e2e.SeaTunnelEngineContainer; -import org.apache.seatunnel.shade.com.google.common.collect.Lists; -import org.junit.jupiter.api.*; + +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.tuple.Pair; + +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; import org.testcontainers.DockerClientFactory; import org.testcontainers.containers.Container; import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.MySQLContainer; import org.testcontainers.containers.Network; import org.testcontainers.containers.output.Slf4jLogConsumer; import org.testcontainers.containers.wait.strategy.Wait; @@ -39,20 +41,32 @@ import org.testcontainers.lifecycle.Startables; import org.testcontainers.utility.DockerImageName; import org.testcontainers.utility.DockerLoggerFactory; -import org.testcontainers.containers.MySQLContainer; + +import com.github.dockerjava.api.DockerClient; +import io.restassured.RestAssured; +import org.testcontainers.utility.MountableFile; import java.io.IOException; import java.math.BigDecimal; -import java.sql.*; -import java.sql.Date; -import java.time.LocalDate; -import java.time.LocalDateTime; -import java.time.LocalTime; -import java.util.*; +import java.nio.file.Paths; +import java.sql.Connection; +import java.sql.Driver; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.sql.Statement; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.UUID; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import java.util.stream.Stream; +import static org.apache.seatunnel.e2e.common.util.ContainerUtil.PROJECT_ROOT_PATH; import static org.awaitility.Awaitility.given; public class MetalakeIT extends SeaTunnelEngineContainer { @@ -88,15 +102,14 @@ public class MetalakeIT extends SeaTunnelEngineContainer { private static final String DRIVER_CLASS = "com.mysql.cj.jdbc.Driver"; - Network NETWORK = - Network.builder() - .createNetworkCmdModifier(cmd -> cmd.withName("SEATUNNEL-" + UUID.randomUUID())) - .enableIpv6(false) - .build(); +// Network NETWORK = +// Network.builder() +// .createNetworkCmdModifier(cmd -> cmd.withName("SEATUNNEL-" + UUID.randomUUID())) +// .enableIpv6(false) +// .build(); private static final List CONFIG_FILE = - Lists.newArrayList( - "/mysql_to_mysql_with_metalake.conf"); + Lists.newArrayList("/mysql_to_mysql_with_metalake.conf"); private static final String CREATE_SQL = "CREATE TABLE IF NOT EXISTS %s\n" + "(\n" @@ -111,27 +124,62 @@ public class MetalakeIT extends SeaTunnelEngineContainer { @BeforeAll @Override - public void startUp() throws Exception{ - super.startUp(); + public void startUp() throws Exception { + //super.startUp(); + server = + new GenericContainer<>(getDockerImage()) + .withNetwork(NETWORK) + .withEnv("TZ", "UTC") + .withEnv("METALAKE_ENABLED", "true") + .withEnv("METALAKE_TYPE", "gravitino") + .withEnv("METALAKE_URL", "http://172.17.0.1:8090/api/metalakes/test_metalake/catalogs/") + .withCommand(buildStartCommand()) + .withNetworkAliases("server") + .withExposedPorts() + .withFileSystemBind("/tmp", "/opt/hive") + .withLogConsumer( + new Slf4jLogConsumer( + DockerLoggerFactory.getLogger( + "seatunnel-engine:" + JDK_DOCKER_IMAGE))) + .waitingFor(Wait.forLogMessage(".*received new worker register:.*", 1)); + copySeaTunnelStarterToContainer(server); + server.setPortBindings(Arrays.asList("5801:5801", "8080:8080")); + server.withCopyFileToContainer( + MountableFile.forHostPath( + PROJECT_ROOT_PATH + + "/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/"), + Paths.get(SEATUNNEL_HOME, "config").toString()); + + server.withCopyFileToContainer( + MountableFile.forHostPath( + PROJECT_ROOT_PATH + + "/seatunnel-shade/seatunnel-hadoop3-3.1.4-uber/target/seatunnel-hadoop3-3.1.4-uber.jar"), + Paths.get(SEATUNNEL_HOME, "lib/seatunnel-hadoop3-3.1.4-uber.jar").toString()); + // execute extra commands + executeExtraCommands(server); + server.start(); + server.execInContainer( "bash", "-c", - "export METALAKE_ENABLED=${METALAKE_ENABLED:-true} && " + - "export METALAKE_TYPE=${METALAKE_TYPE:-gravitino} && " + - "export METALAKE_URL=${METALAKE_URL:-http://localhost:8090/api/metalakes/test_metalake/catalogs/} && " + "mkdir -p /tmp/seatunnel/plugins/Jdbc/lib && cd /tmp/seatunnel/plugins/Jdbc/lib && wget " + driverUrl() + " --no-check-certificate"); - dbServer = initContainer().withImagePullPolicy(PullPolicy.alwaysPull()); - - gravitinoServer = new GenericContainer<>("apache/gravitino:latest") - .withExposedPorts(8090) - .withImagePullPolicy(PullPolicy.alwaysPull()) - .withNetwork(NETWORK) - .withNetworkAliases("gravitino") - .withLogConsumer(new Slf4jLogConsumer(DockerLoggerFactory.getLogger("apache/gravitino:latest"))); - - Startables.deepStart(Stream.of(dbServer, gravitinoServer)).join(); + dbServer = initContainer().withImagePullPolicy(PullPolicy.defaultPolicy()); + +// gravitinoServer = +// new GenericContainer<>("apache/gravitino:latest") +// .withExposedPorts(8090) +// .withExposedPorts(9001) +// .waitingFor(Wait.forListeningPort().withStartupTimeout(Duration.ofMinutes(2))) +// .withImagePullPolicy(PullPolicy.defaultPolicy()) +// .withNetwork(NETWORK) +// .withNetworkAliases("gravitino") +// .withLogConsumer( +// new Slf4jLogConsumer( +// DockerLoggerFactory.getLogger("apache/gravitino:latest"))); + + Startables.deepStart(Stream.of(dbServer)).join(); jdbcCase = getJdbcCase(); @@ -140,43 +188,52 @@ public void startUp() throws Exception{ .atMost(360, TimeUnit.SECONDS) .untilAsserted(() -> this.initializeJdbcConnection(jdbcCase.getJdbcUrl())); - given().ignoreExceptions() - .await() - .atMost(60, TimeUnit.SECONDS) - .untilAsserted(() -> { - String url = "http://" + gravitinoServer.getHost() + ":" + gravitinoServer.getMappedPort(8090); - int statusCode = RestAssured.get(url + "/health").statusCode(); - Assertions.assertEquals(200, statusCode); - }); - - String gravitinoUrl = "http://" + gravitinoServer.getHost() + ":" + gravitinoServer.getMappedPort(8090); - - String payload = "{" - + "\"name\": \"test_catalog\"," - + "\"type\": \"relational\"," - + "\"provider\": \"mysql\"," - + "\"comment\": \"Catalog for integration test\"," - + "\"properties\": {" - + " \"user\": \"root\"," - + " \"password\": \"Abc!@#135_seatunnel\"" - + " }" - + "}"; - - RestAssured.given() - .contentType("application/json") - .body(payload) - .when() - .post(gravitinoUrl + "/api/metalakes/test_metalake/catalogs/test_catalog") - .then() - .statusCode(200); +// given().ignoreExceptions() +// .await() +// .atMost(60, TimeUnit.SECONDS) +// .untilAsserted( +// () -> { +// String url = +// "http://" +// + gravitinoServer.getHost() +// + ":" +// + gravitinoServer.getMappedPort(8090); +// int statusCode = RestAssured.get(url + "/health").statusCode(); +// Assertions.assertEquals(200, statusCode); +// }); + +// String gravitinoUrl = +// "http://" + gravitinoServer.getHost() + ":" + gravitinoServer.getMappedPort(8090); +// +// String payload = +// "{" +// + "\"name\": \"test_catalog\"," +// + "\"type\": \"relational\"," +// + "\"provider\": \"mysql\"," +// + "\"comment\": \"Catalog for integration test\"," +// + "\"properties\": {" +// + " \"user\": \"root\"," +// + " \"password\": \"Abc!@#135_seatunnel\"" +// + " }" +// + "}"; + +// RestAssured.given() +// .contentType("application/json") +// .body(payload) +// .when() +// .post(gravitinoUrl + "/api/metalakes/test_metalake/catalogs/test_catalog") +// .then() +// .statusCode(200); createNeededTables(); insertTestData(); + System.out.println("prepare over"); } @AfterAll @Override public void tearDown() throws Exception { + System.out.println("test over"); if (catalog != null) { catalog.close(); } @@ -188,9 +245,9 @@ public void tearDown() throws Exception { if (dbServer != null) { dbServer.close(); try { - dockerClient.removeImageCmd(dbServer.getDockerImageName()).exec(); + //dockerClient.removeImageCmd(dbServer.getDockerImageName()).exec(); } catch (Exception ignored) { - //log.warn("Failed to delete the image. Another container may be in use", ignored); + // log.warn("Failed to delete the image. Another container may be in use", ignored); ignored.printStackTrace(); } } @@ -198,9 +255,9 @@ public void tearDown() throws Exception { if (gravitinoServer != null) { gravitinoServer.close(); try { - dockerClient.removeImageCmd(gravitinoServer.getDockerImageName()).exec(); + //dockerClient.removeImageCmd(gravitinoServer.getDockerImageName()).exec(); } catch (Exception ignored) { - //log.warn("Failed to delete the image. Another container may be in use", ignored); + // log.warn("Failed to delete the image. Another container may be in use", ignored); ignored.printStackTrace(); } } @@ -209,7 +266,9 @@ public void tearDown() throws Exception { @Test public void TestMetalake() throws IOException, InterruptedException { - Container.ExecResult execResult = executeSeaTunnelJob("/mysql_to_mysql_with_metalake.conf"); + System.out.println("begin to test"); + Container.ExecResult execResult = executeSeaTunnelJob("/mysql_to_console_with_metalake.conf"); + System.in.read(); Assertions.assertEquals(0, execResult.getExitCode()); } @@ -343,15 +402,15 @@ protected void createNeededTables() { connection.commit(); } catch (Exception exception) { - //log.error(ExceptionUtils.getMessage(exception)); - //throw new SeaTunnelRuntimeException(JdbcITErrorCode.CREATE_TABLE_FAILED, exception); + // log.error(ExceptionUtils.getMessage(exception)); + // throw new SeaTunnelRuntimeException(JdbcITErrorCode.CREATE_TABLE_FAILED, exception); exception.printStackTrace(); } } protected void insertTestData() { try (PreparedStatement preparedStatement = - connection.prepareStatement(jdbcCase.getInsertSql())) { + connection.prepareStatement(jdbcCase.getInsertSql())) { List rows = jdbcCase.getTestData().getValue(); @@ -366,8 +425,8 @@ protected void insertTestData() { connection.commit(); } catch (Exception exception) { - //log.error(ExceptionUtils.getMessage(exception)); - //throw new SeaTunnelRuntimeException(JdbcITErrorCode.INSERT_DATA_FAILED, exception); + // log.error(ExceptionUtils.getMessage(exception)); + // throw new SeaTunnelRuntimeException(JdbcITErrorCode.INSERT_DATA_FAILED, exception); exception.printStackTrace(); } } @@ -375,12 +434,7 @@ protected void insertTestData() { Pair> initTestData() { String[] fieldNames = new String[] { - "c-bit_1", - "c_bit_8", - "c_bit_16", - "c_bit_32", - "c_bit_64", - "c_bigint_30", + "c-bit_1", "c_bit_8", "c_bit_16", "c_bit_32", "c_bit_64", "c_bigint_30", }; List rows = new ArrayList<>(); @@ -393,30 +447,30 @@ Pair> initTestData() { row = new SeaTunnelRow( new Object[] { - null, - null, - null, - null, - null, - // https://github.com/apache/seatunnel/issues/5559 this value - // cannot set null, this null - // value column's row will be lost in - // jdbc_mysql_source_and_sink_parallel.conf,jdbc_mysql_source_and_sink_parallel_upper_lower.conf. - bigintValue.add(BigDecimal.valueOf(i)), + null, + null, + null, + null, + null, + // https://github.com/apache/seatunnel/issues/5559 this value + // cannot set null, this null + // value column's row will be lost in + // jdbc_mysql_source_and_sink_parallel.conf,jdbc_mysql_source_and_sink_parallel_upper_lower.conf. + bigintValue.add(BigDecimal.valueOf(i)), }); } else { row = new SeaTunnelRow( new Object[] { - i % 2 == 0 ? (byte) 1 : (byte) 0, - new byte[] {byteArr}, - new byte[] {byteArr, byteArr}, - new byte[] {byteArr, byteArr, byteArr, byteArr}, - new byte[] { - byteArr, byteArr, byteArr, byteArr, byteArr, byteArr, - byteArr, byteArr - }, - bigintValue.add(BigDecimal.valueOf(i)), + i % 2 == 0 ? (byte) 1 : (byte) 0, + new byte[] {byteArr}, + new byte[] {byteArr, byteArr}, + new byte[] {byteArr, byteArr, byteArr, byteArr}, + new byte[] { + byteArr, byteArr, byteArr, byteArr, byteArr, byteArr, + byteArr, byteArr + }, + bigintValue.add(BigDecimal.valueOf(i)), }); } rows.add(row); diff --git a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/mysql_to_console_with_metalake.conf b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/mysql_to_console_with_metalake.conf index 241e33722eb..47b3ebc649c 100644 --- a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/mysql_to_console_with_metalake.conf +++ b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/mysql_to_console_with_metalake.conf @@ -5,17 +5,13 @@ env { source { Jdbc { - url = "jdbc:mysql://mysql-e2e:3306/seatunnel" + url = "jdbc:mysql://mysql-e2e:3306/seatunnel?useSSL=false&serverTimezone=UTC&allowPublicKeyRetrieval=true" driver = "com.mysql.cj.jdbc.Driver" connection_check_timeout_sec = 100 sourceId = "test_catalog" - user = "${user}" - password = "${password}" + user = "${jdbc-user}" + password = "${jdbc-password}" query = "select * from source" - properties { - useSSL=false - rewriteBatchedStatements=true - } } } diff --git a/seatunnel-engine/seatunnel-engine-core/src/main/java/org/apache/seatunnel/engine/core/parse/MultipleTableJobConfigParser.java b/seatunnel-engine/seatunnel-engine-core/src/main/java/org/apache/seatunnel/engine/core/parse/MultipleTableJobConfigParser.java index 87b5b33a925..2c4b7b42015 100644 --- a/seatunnel-engine/seatunnel-engine-core/src/main/java/org/apache/seatunnel/engine/core/parse/MultipleTableJobConfigParser.java +++ b/seatunnel-engine/seatunnel-engine-core/src/main/java/org/apache/seatunnel/engine/core/parse/MultipleTableJobConfigParser.java @@ -22,8 +22,10 @@ import org.apache.seatunnel.shade.com.google.common.collect.Lists; import org.apache.seatunnel.shade.com.typesafe.config.Config; import org.apache.seatunnel.shade.com.typesafe.config.ConfigObject; +import org.apache.seatunnel.shade.com.typesafe.config.ConfigList; import org.apache.seatunnel.shade.com.typesafe.config.ConfigValue; import org.apache.seatunnel.shade.com.typesafe.config.ConfigValueType; +import org.apache.seatunnel.shade.com.typesafe.config.ConfigRenderOptions; import org.apache.seatunnel.api.common.PluginIdentifier; import org.apache.seatunnel.api.configuration.ReadonlyConfig; @@ -839,14 +841,15 @@ private Config getMetalakeConfig(Config jobConfigTmp) { MetalakeClient metalakeClient = MetalakeClientFactory.create(metalakeType, metalakeUrl); try { - ConfigObject sourceObj = jobConfigTmp.getObject("source"); - Set sourceKeys = sourceObj.keySet(); + ConfigList sourceList = jobConfigTmp.getList("source"); + //Set sourceKeys = sourceObj.keySet(); + log.info("Parsed config:\n{}", jobConfigTmp.root().render(ConfigRenderOptions.concise().setFormatted(true))); - for (String key : sourceKeys) { - if (jobConfigTmp.hasPath("source." + key + ".sourceId")) { - String sourceId = jobConfigTmp.getString("source." + key + ".sourceId"); + for (int i = 0; i < sourceList.size(); i++) { + if (jobConfigTmp.hasPath("source." + ".sourceId")) { + String sourceId = jobConfigTmp.getString("source." + ".sourceId"); JsonNode metalakeJson = metalakeClient.getMetaInfo(sourceId); - ConfigObject subConfig = jobConfigTmp.getObject("source." + key); + ConfigObject subConfig = jobConfigTmp.getObject("source."); for (Map.Entry entry : subConfig.entrySet()) { String subKey = entry.getKey(); ConfigValue value = entry.getValue(); @@ -858,7 +861,7 @@ private Config getMetalakeConfig(Config jobConfigTmp) { if (metalakeJson.has(placeholder)) { String replaced = metalakeJson.get(placeholder).asText(); - String finalPath = "source." + key + "." + subKey; + String finalPath = "source." + "." + subKey; metalakeConfigMap.put(finalPath, replaced); } } From a6d23273d60d7446a612b1e6a83e4cfabb14fa28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=90=B4=E5=A4=A9=E5=AE=87?= <1599646055@qq.com> Date: Wed, 27 Aug 2025 19:14:49 +0800 Subject: [PATCH 10/39] feat: apply spotless code style --- .../engine/e2e/metalake/MetalakeIT.java | 74 +++---------------- .../parse/MultipleTableJobConfigParser.java | 12 +-- 2 files changed, 16 insertions(+), 70 deletions(-) diff --git a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/metalake/MetalakeIT.java b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/metalake/MetalakeIT.java index 3ad5920f9b4..3751a48a8f7 100644 --- a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/metalake/MetalakeIT.java +++ b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/metalake/MetalakeIT.java @@ -34,17 +34,15 @@ import org.testcontainers.containers.Container; import org.testcontainers.containers.GenericContainer; import org.testcontainers.containers.MySQLContainer; -import org.testcontainers.containers.Network; import org.testcontainers.containers.output.Slf4jLogConsumer; import org.testcontainers.containers.wait.strategy.Wait; import org.testcontainers.images.PullPolicy; import org.testcontainers.lifecycle.Startables; import org.testcontainers.utility.DockerImageName; import org.testcontainers.utility.DockerLoggerFactory; +import org.testcontainers.utility.MountableFile; import com.github.dockerjava.api.DockerClient; -import io.restassured.RestAssured; -import org.testcontainers.utility.MountableFile; import java.io.IOException; import java.math.BigDecimal; @@ -54,14 +52,12 @@ import java.sql.PreparedStatement; import java.sql.SQLException; import java.sql.Statement; -import java.time.Duration; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Properties; -import java.util.UUID; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -102,12 +98,6 @@ public class MetalakeIT extends SeaTunnelEngineContainer { private static final String DRIVER_CLASS = "com.mysql.cj.jdbc.Driver"; -// Network NETWORK = -// Network.builder() -// .createNetworkCmdModifier(cmd -> cmd.withName("SEATUNNEL-" + UUID.randomUUID())) -// .enableIpv6(false) -// .build(); - private static final List CONFIG_FILE = Lists.newArrayList("/mysql_to_mysql_with_metalake.conf"); private static final String CREATE_SQL = @@ -125,14 +115,16 @@ public class MetalakeIT extends SeaTunnelEngineContainer { @BeforeAll @Override public void startUp() throws Exception { - //super.startUp(); + // super.startUp(); server = new GenericContainer<>(getDockerImage()) .withNetwork(NETWORK) .withEnv("TZ", "UTC") .withEnv("METALAKE_ENABLED", "true") .withEnv("METALAKE_TYPE", "gravitino") - .withEnv("METALAKE_URL", "http://172.17.0.1:8090/api/metalakes/test_metalake/catalogs/") + .withEnv( + "METALAKE_URL", + "http://172.17.0.1:8090/api/metalakes/test_metalake/catalogs/") .withCommand(buildStartCommand()) .withNetworkAliases("server") .withExposedPorts() @@ -167,18 +159,6 @@ public void startUp() throws Exception { + " --no-check-certificate"); dbServer = initContainer().withImagePullPolicy(PullPolicy.defaultPolicy()); -// gravitinoServer = -// new GenericContainer<>("apache/gravitino:latest") -// .withExposedPorts(8090) -// .withExposedPorts(9001) -// .waitingFor(Wait.forListeningPort().withStartupTimeout(Duration.ofMinutes(2))) -// .withImagePullPolicy(PullPolicy.defaultPolicy()) -// .withNetwork(NETWORK) -// .withNetworkAliases("gravitino") -// .withLogConsumer( -// new Slf4jLogConsumer( -// DockerLoggerFactory.getLogger("apache/gravitino:latest"))); - Startables.deepStart(Stream.of(dbServer)).join(); jdbcCase = getJdbcCase(); @@ -188,43 +168,6 @@ public void startUp() throws Exception { .atMost(360, TimeUnit.SECONDS) .untilAsserted(() -> this.initializeJdbcConnection(jdbcCase.getJdbcUrl())); -// given().ignoreExceptions() -// .await() -// .atMost(60, TimeUnit.SECONDS) -// .untilAsserted( -// () -> { -// String url = -// "http://" -// + gravitinoServer.getHost() -// + ":" -// + gravitinoServer.getMappedPort(8090); -// int statusCode = RestAssured.get(url + "/health").statusCode(); -// Assertions.assertEquals(200, statusCode); -// }); - -// String gravitinoUrl = -// "http://" + gravitinoServer.getHost() + ":" + gravitinoServer.getMappedPort(8090); -// -// String payload = -// "{" -// + "\"name\": \"test_catalog\"," -// + "\"type\": \"relational\"," -// + "\"provider\": \"mysql\"," -// + "\"comment\": \"Catalog for integration test\"," -// + "\"properties\": {" -// + " \"user\": \"root\"," -// + " \"password\": \"Abc!@#135_seatunnel\"" -// + " }" -// + "}"; - -// RestAssured.given() -// .contentType("application/json") -// .body(payload) -// .when() -// .post(gravitinoUrl + "/api/metalakes/test_metalake/catalogs/test_catalog") -// .then() -// .statusCode(200); - createNeededTables(); insertTestData(); System.out.println("prepare over"); @@ -245,7 +188,7 @@ public void tearDown() throws Exception { if (dbServer != null) { dbServer.close(); try { - //dockerClient.removeImageCmd(dbServer.getDockerImageName()).exec(); + // dockerClient.removeImageCmd(dbServer.getDockerImageName()).exec(); } catch (Exception ignored) { // log.warn("Failed to delete the image. Another container may be in use", ignored); ignored.printStackTrace(); @@ -255,7 +198,7 @@ public void tearDown() throws Exception { if (gravitinoServer != null) { gravitinoServer.close(); try { - //dockerClient.removeImageCmd(gravitinoServer.getDockerImageName()).exec(); + // dockerClient.removeImageCmd(gravitinoServer.getDockerImageName()).exec(); } catch (Exception ignored) { // log.warn("Failed to delete the image. Another container may be in use", ignored); ignored.printStackTrace(); @@ -267,7 +210,8 @@ public void tearDown() throws Exception { @Test public void TestMetalake() throws IOException, InterruptedException { System.out.println("begin to test"); - Container.ExecResult execResult = executeSeaTunnelJob("/mysql_to_console_with_metalake.conf"); + Container.ExecResult execResult = + executeSeaTunnelJob("/mysql_to_console_with_metalake.conf"); System.in.read(); Assertions.assertEquals(0, execResult.getExitCode()); } diff --git a/seatunnel-engine/seatunnel-engine-core/src/main/java/org/apache/seatunnel/engine/core/parse/MultipleTableJobConfigParser.java b/seatunnel-engine/seatunnel-engine-core/src/main/java/org/apache/seatunnel/engine/core/parse/MultipleTableJobConfigParser.java index 2c4b7b42015..1539ddaf78f 100644 --- a/seatunnel-engine/seatunnel-engine-core/src/main/java/org/apache/seatunnel/engine/core/parse/MultipleTableJobConfigParser.java +++ b/seatunnel-engine/seatunnel-engine-core/src/main/java/org/apache/seatunnel/engine/core/parse/MultipleTableJobConfigParser.java @@ -21,11 +21,11 @@ import org.apache.seatunnel.shade.com.google.common.base.Preconditions; import org.apache.seatunnel.shade.com.google.common.collect.Lists; import org.apache.seatunnel.shade.com.typesafe.config.Config; -import org.apache.seatunnel.shade.com.typesafe.config.ConfigObject; import org.apache.seatunnel.shade.com.typesafe.config.ConfigList; +import org.apache.seatunnel.shade.com.typesafe.config.ConfigObject; +import org.apache.seatunnel.shade.com.typesafe.config.ConfigRenderOptions; import org.apache.seatunnel.shade.com.typesafe.config.ConfigValue; import org.apache.seatunnel.shade.com.typesafe.config.ConfigValueType; -import org.apache.seatunnel.shade.com.typesafe.config.ConfigRenderOptions; import org.apache.seatunnel.api.common.PluginIdentifier; import org.apache.seatunnel.api.configuration.ReadonlyConfig; @@ -842,12 +842,14 @@ private Config getMetalakeConfig(Config jobConfigTmp) { try { ConfigList sourceList = jobConfigTmp.getList("source"); - //Set sourceKeys = sourceObj.keySet(); - log.info("Parsed config:\n{}", jobConfigTmp.root().render(ConfigRenderOptions.concise().setFormatted(true))); + // Set sourceKeys = sourceObj.keySet(); + log.info( + "metalake config:\n{}", + jobConfigTmp.root().render(ConfigRenderOptions.concise().setFormatted(true))); for (int i = 0; i < sourceList.size(); i++) { if (jobConfigTmp.hasPath("source." + ".sourceId")) { - String sourceId = jobConfigTmp.getString("source." + ".sourceId"); + String sourceId = jobConfigTmp.getString("source." + ".sourceId"); JsonNode metalakeJson = metalakeClient.getMetaInfo(sourceId); ConfigObject subConfig = jobConfigTmp.getObject("source."); for (Map.Entry entry : subConfig.entrySet()) { From a1148f7cd634d949dbab90e358c43a513a6ded8b Mon Sep 17 00:00:00 2001 From: wtybxqm <1599646055@qq.com> Date: Wed, 27 Aug 2025 21:14:02 +0800 Subject: [PATCH 11/39] fix: fix the error of config structure --- .../parse/MultipleTableJobConfigParser.java | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/seatunnel-engine/seatunnel-engine-core/src/main/java/org/apache/seatunnel/engine/core/parse/MultipleTableJobConfigParser.java b/seatunnel-engine/seatunnel-engine-core/src/main/java/org/apache/seatunnel/engine/core/parse/MultipleTableJobConfigParser.java index 1539ddaf78f..33567787488 100644 --- a/seatunnel-engine/seatunnel-engine-core/src/main/java/org/apache/seatunnel/engine/core/parse/MultipleTableJobConfigParser.java +++ b/seatunnel-engine/seatunnel-engine-core/src/main/java/org/apache/seatunnel/engine/core/parse/MultipleTableJobConfigParser.java @@ -842,17 +842,17 @@ private Config getMetalakeConfig(Config jobConfigTmp) { try { ConfigList sourceList = jobConfigTmp.getList("source"); - // Set sourceKeys = sourceObj.keySet(); - log.info( - "metalake config:\n{}", + + System.err.println("metalake"); + System.err.println( jobConfigTmp.root().render(ConfigRenderOptions.concise().setFormatted(true))); for (int i = 0; i < sourceList.size(); i++) { - if (jobConfigTmp.hasPath("source." + ".sourceId")) { - String sourceId = jobConfigTmp.getString("source." + ".sourceId"); + ConfigObject sourceObj = (ConfigObject) sourceList.get(i); + if (sourceObj.containsKey("sourceId")) { + String sourceId = sourceObj.toConfig().getString("sourceId"); JsonNode metalakeJson = metalakeClient.getMetaInfo(sourceId); - ConfigObject subConfig = jobConfigTmp.getObject("source."); - for (Map.Entry entry : subConfig.entrySet()) { + for (Map.Entry entry : sourceObj.entrySet()) { String subKey = entry.getKey(); ConfigValue value = entry.getValue(); @@ -863,7 +863,7 @@ private Config getMetalakeConfig(Config jobConfigTmp) { if (metalakeJson.has(placeholder)) { String replaced = metalakeJson.get(placeholder).asText(); - String finalPath = "source." + "." + subKey; + String finalPath = "source[" + i + "]." + subKey; metalakeConfigMap.put(finalPath, replaced); } } @@ -876,14 +876,14 @@ private Config getMetalakeConfig(Config jobConfigTmp) { } try { - ConfigObject sinkObj = jobConfigTmp.getObject("sink"); - Set sinkKeys = sinkObj.keySet(); - for (String key : sinkKeys) { - if (jobConfigTmp.hasPath("sink." + key + ".sourceId")) { - String sourceId = jobConfigTmp.getString("source." + key + ".sourceId"); + ConfigList sinkList = jobConfigTmp.getList("sink"); + + for (int i = 0; i < sinkList.size(); i++) { + ConfigObject sinkObj = (ConfigObject) sinkList.get(i); + if (sinkObj.containsKey("sourceId")) { + String sourceId = sinkObj.toConfig().getString("sourceId"); JsonNode metalakeJson = metalakeClient.getMetaInfo(sourceId); - ConfigObject subConfig = jobConfigTmp.getObject("sink." + key); - for (Map.Entry entry : subConfig.entrySet()) { + for (Map.Entry entry : sinkObj.entrySet()) { String subKey = entry.getKey(); ConfigValue value = entry.getValue(); @@ -894,7 +894,7 @@ private Config getMetalakeConfig(Config jobConfigTmp) { if (metalakeJson.has(placeholder)) { String replaced = metalakeJson.get(placeholder).asText(); - String finalPath = "sink." + key + "." + subKey; + String finalPath = "sink[" + i + "]." + subKey; metalakeConfigMap.put(finalPath, replaced); } } From c0b304ea3086cacca41eca4ad3ecce2bfd2122e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=90=B4=E5=A4=A9=E5=AE=87?= <1599646055@qq.com> Date: Wed, 27 Aug 2025 21:55:12 +0800 Subject: [PATCH 12/39] feat: place okhttp3 with apache httpclient --- seatunnel-api/pom.xml | 6 +-- .../api/metalake/GravitinoClient.java | 48 +++++++++++-------- .../parse/MultipleTableJobConfigParser.java | 7 ++- 3 files changed, 36 insertions(+), 25 deletions(-) diff --git a/seatunnel-api/pom.xml b/seatunnel-api/pom.xml index d68928899df..80c771f4309 100644 --- a/seatunnel-api/pom.xml +++ b/seatunnel-api/pom.xml @@ -42,9 +42,9 @@ optional - com.squareup.okhttp3 - okhttp - 4.12.0 + org.apache.httpcomponents + httpclient + 4.5.13 diff --git a/seatunnel-api/src/main/java/org/apache/seatunnel/api/metalake/GravitinoClient.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/metalake/GravitinoClient.java index cdb0563ddf8..5eb4bfd239a 100644 --- a/seatunnel-api/src/main/java/org/apache/seatunnel/api/metalake/GravitinoClient.java +++ b/seatunnel-api/src/main/java/org/apache/seatunnel/api/metalake/GravitinoClient.java @@ -20,11 +20,12 @@ import org.apache.seatunnel.shade.com.fasterxml.jackson.databind.JsonNode; import org.apache.seatunnel.shade.com.fasterxml.jackson.databind.ObjectMapper; -import okhttp3.MediaType; -import okhttp3.OkHttpClient; -import okhttp3.Request; -import okhttp3.RequestBody; -import okhttp3.Response; +import org.apache.http.HttpEntity; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.util.EntityUtils; import java.io.IOException; @@ -42,21 +43,26 @@ public String getType() { @Override public JsonNode getMetaInfo(String sourceId) throws IOException { - OkHttpClient client = new OkHttpClient().newBuilder().build(); - MediaType mediaType = MediaType.parse("text/plain"); - RequestBody body = RequestBody.create(mediaType, ""); - Request request = - new Request.Builder() - .url(this.metalakeUrl + sourceId) - .method("GET", body) - .addHeader("Accept", "application/vnd.gravitino.v1+json") - // .addHeader("Authorization", "Bearer ") - .build(); - Response response = client.newCall(request).execute(); - ObjectMapper mapper = new ObjectMapper(); - JsonNode rootNode = mapper.readTree(response.body().byteStream()); - JsonNode catalogNode = rootNode.get("catalog"); - JsonNode propertiesNode = catalogNode.get("properties"); - return propertiesNode; + try (CloseableHttpClient client = HttpClients.createDefault()) { + HttpGet request = new HttpGet(this.metalakeUrl + sourceId); + request.addHeader("Accept", "application/vnd.gravitino.v1+json"); + try (CloseableHttpResponse response = client.execute(request)) { + HttpEntity entity = response.getEntity(); + if (entity == null) { + throw new RuntimeException("No response entity"); + } + ObjectMapper mapper = new ObjectMapper(); + JsonNode rootNode = mapper.readTree(entity.getContent()); + System.err.println("gravitino return"); + System.err.println(rootNode.toString()); + EntityUtils.consume(entity); + JsonNode catalogNode = rootNode.get("catalog"); + if (catalogNode == null) { + throw new RuntimeException("Response JSON has no 'catalog' field"); + } + JsonNode propertiesNode = catalogNode.get("properties"); + return propertiesNode; + } + } } } diff --git a/seatunnel-engine/seatunnel-engine-core/src/main/java/org/apache/seatunnel/engine/core/parse/MultipleTableJobConfigParser.java b/seatunnel-engine/seatunnel-engine-core/src/main/java/org/apache/seatunnel/engine/core/parse/MultipleTableJobConfigParser.java index 33567787488..6a92e957d37 100644 --- a/seatunnel-engine/seatunnel-engine-core/src/main/java/org/apache/seatunnel/engine/core/parse/MultipleTableJobConfigParser.java +++ b/seatunnel-engine/seatunnel-engine-core/src/main/java/org/apache/seatunnel/engine/core/parse/MultipleTableJobConfigParser.java @@ -843,7 +843,7 @@ private Config getMetalakeConfig(Config jobConfigTmp) { try { ConfigList sourceList = jobConfigTmp.getList("source"); - System.err.println("metalake"); + System.err.println("original config"); System.err.println( jobConfigTmp.root().render(ConfigRenderOptions.concise().setFormatted(true))); @@ -905,6 +905,11 @@ private Config getMetalakeConfig(Config jobConfigTmp) { } catch (IOException e) { log.error("Fail to get MetaInfo, metalakeUrl: {}", metalakeUrl, e); } + System.err.println("metalake config"); + System.err.println( + ConfigBuilder.of(metalakeConfigMap) + .root() + .render(ConfigRenderOptions.concise().setFormatted(true))); return ConfigBuilder.of(metalakeConfigMap); } } From d66778e139fe3d97b1601e54784100edb7ee90f8 Mon Sep 17 00:00:00 2001 From: wtybxqm <1599646055@qq.com> Date: Thu, 28 Aug 2025 12:17:41 +0800 Subject: [PATCH 13/39] fix: fix the bug of matainfo replace and remove log info --- .../api/metalake/GravitinoClient.java | 2 - .../engine/e2e/metalake/MetalakeIT.java | 4 -- .../parse/MultipleTableJobConfigParser.java | 39 ++++++++++--------- 3 files changed, 21 insertions(+), 24 deletions(-) diff --git a/seatunnel-api/src/main/java/org/apache/seatunnel/api/metalake/GravitinoClient.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/metalake/GravitinoClient.java index 5eb4bfd239a..de95bf3a875 100644 --- a/seatunnel-api/src/main/java/org/apache/seatunnel/api/metalake/GravitinoClient.java +++ b/seatunnel-api/src/main/java/org/apache/seatunnel/api/metalake/GravitinoClient.java @@ -53,8 +53,6 @@ public JsonNode getMetaInfo(String sourceId) throws IOException { } ObjectMapper mapper = new ObjectMapper(); JsonNode rootNode = mapper.readTree(entity.getContent()); - System.err.println("gravitino return"); - System.err.println(rootNode.toString()); EntityUtils.consume(entity); JsonNode catalogNode = rootNode.get("catalog"); if (catalogNode == null) { diff --git a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/metalake/MetalakeIT.java b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/metalake/MetalakeIT.java index 3751a48a8f7..17049d7d0ab 100644 --- a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/metalake/MetalakeIT.java +++ b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/metalake/MetalakeIT.java @@ -170,13 +170,11 @@ public void startUp() throws Exception { createNeededTables(); insertTestData(); - System.out.println("prepare over"); } @AfterAll @Override public void tearDown() throws Exception { - System.out.println("test over"); if (catalog != null) { catalog.close(); } @@ -209,10 +207,8 @@ public void tearDown() throws Exception { @Test public void TestMetalake() throws IOException, InterruptedException { - System.out.println("begin to test"); Container.ExecResult execResult = executeSeaTunnelJob("/mysql_to_console_with_metalake.conf"); - System.in.read(); Assertions.assertEquals(0, execResult.getExitCode()); } diff --git a/seatunnel-engine/seatunnel-engine-core/src/main/java/org/apache/seatunnel/engine/core/parse/MultipleTableJobConfigParser.java b/seatunnel-engine/seatunnel-engine-core/src/main/java/org/apache/seatunnel/engine/core/parse/MultipleTableJobConfigParser.java index 6a92e957d37..95087dacf57 100644 --- a/seatunnel-engine/seatunnel-engine-core/src/main/java/org/apache/seatunnel/engine/core/parse/MultipleTableJobConfigParser.java +++ b/seatunnel-engine/seatunnel-engine-core/src/main/java/org/apache/seatunnel/engine/core/parse/MultipleTableJobConfigParser.java @@ -25,6 +25,7 @@ import org.apache.seatunnel.shade.com.typesafe.config.ConfigObject; import org.apache.seatunnel.shade.com.typesafe.config.ConfigRenderOptions; import org.apache.seatunnel.shade.com.typesafe.config.ConfigValue; +import org.apache.seatunnel.shade.com.typesafe.config.ConfigValueFactory; import org.apache.seatunnel.shade.com.typesafe.config.ConfigValueType; import org.apache.seatunnel.api.common.PluginIdentifier; @@ -171,9 +172,8 @@ public MultipleTableJobConfigParser( boolean metalakeEnabled = Boolean.parseBoolean(System.getenv().getOrDefault("METALAKE_ENABLED", "false")); if (metalakeEnabled) { - Config jobConfigTmp = ConfigBuilder.of(Paths.get(jobDefineFilePath), variables); - Config metalakeConfig = getMetalakeConfig(jobConfigTmp); - this.seaTunnelJobConfig = metalakeConfig.withFallback(jobConfigTmp); + this.seaTunnelJobConfig = + getMetalakeConfig(ConfigBuilder.of(Paths.get(jobDefineFilePath), variables)); } else { this.seaTunnelJobConfig = ConfigBuilder.of(Paths.get(jobDefineFilePath), variables); } @@ -834,7 +834,7 @@ private ChangeStreamTableSourceCheckpoint getSourceCheckpoint( } private Config getMetalakeConfig(Config jobConfigTmp) { - Map metalakeConfigMap = new LinkedHashMap<>(); + Config update = jobConfigTmp; String metalakeType = System.getenv("METALAKE_TYPE"); String metalakeUrl = System.getenv("METALAKE_URL"); @@ -842,14 +842,12 @@ private Config getMetalakeConfig(Config jobConfigTmp) { try { ConfigList sourceList = jobConfigTmp.getList("source"); - - System.err.println("original config"); - System.err.println( - jobConfigTmp.root().render(ConfigRenderOptions.concise().setFormatted(true))); + List newSourceList = new ArrayList<>(sourceList); for (int i = 0; i < sourceList.size(); i++) { ConfigObject sourceObj = (ConfigObject) sourceList.get(i); if (sourceObj.containsKey("sourceId")) { + ConfigObject tmp = sourceObj; String sourceId = sourceObj.toConfig().getString("sourceId"); JsonNode metalakeJson = metalakeClient.getMetaInfo(sourceId); for (Map.Entry entry : sourceObj.entrySet()) { @@ -863,24 +861,30 @@ private Config getMetalakeConfig(Config jobConfigTmp) { if (metalakeJson.has(placeholder)) { String replaced = metalakeJson.get(placeholder).asText(); - String finalPath = "source[" + i + "]." + subKey; - metalakeConfigMap.put(finalPath, replaced); + tmp = + tmp.withValue( + subKey, + ConfigValueFactory.fromAnyRef(replaced)); } } } } + newSourceList.set(i, tmp); } } + update = update.withValue("source", ConfigValueFactory.fromIterable(newSourceList)); } catch (IOException e) { log.error("Fail to get MetaInfo, metalakeUrl: {}", metalakeUrl, e); } try { ConfigList sinkList = jobConfigTmp.getList("sink"); + List newSinkList = new ArrayList<>(sinkList); for (int i = 0; i < sinkList.size(); i++) { ConfigObject sinkObj = (ConfigObject) sinkList.get(i); if (sinkObj.containsKey("sourceId")) { + ConfigObject tmp = sinkObj; String sourceId = sinkObj.toConfig().getString("sourceId"); JsonNode metalakeJson = metalakeClient.getMetaInfo(sourceId); for (Map.Entry entry : sinkObj.entrySet()) { @@ -894,22 +898,21 @@ private Config getMetalakeConfig(Config jobConfigTmp) { if (metalakeJson.has(placeholder)) { String replaced = metalakeJson.get(placeholder).asText(); - String finalPath = "sink[" + i + "]." + subKey; - metalakeConfigMap.put(finalPath, replaced); + tmp = + tmp.withValue( + subKey, + ConfigValueFactory.fromAnyRef(replaced)); } } } } + newSinkList.set(i, tmp); } } + update = update.withValue("sink", ConfigValueFactory.fromIterable(newSinkList)); } catch (IOException e) { log.error("Fail to get MetaInfo, metalakeUrl: {}", metalakeUrl, e); } - System.err.println("metalake config"); - System.err.println( - ConfigBuilder.of(metalakeConfigMap) - .root() - .render(ConfigRenderOptions.concise().setFormatted(true))); - return ConfigBuilder.of(metalakeConfigMap); + return update; } } From b74105289548e15c68d096ff9935450c289947a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=90=B4=E5=A4=A9=E5=AE=87?= <1599646055@qq.com> Date: Thu, 28 Aug 2025 13:24:12 +0800 Subject: [PATCH 14/39] feat: apply spotless code style and remove extra code --- .../seatunnel/engine/e2e/metalake/MetalakeIT.java | 10 ++-------- .../core/parse/MultipleTableJobConfigParser.java | 1 - 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/metalake/MetalakeIT.java b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/metalake/MetalakeIT.java index 17049d7d0ab..90b9be5c7c2 100644 --- a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/metalake/MetalakeIT.java +++ b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/metalake/MetalakeIT.java @@ -186,9 +186,8 @@ public void tearDown() throws Exception { if (dbServer != null) { dbServer.close(); try { - // dockerClient.removeImageCmd(dbServer.getDockerImageName()).exec(); + dockerClient.removeImageCmd(dbServer.getDockerImageName()).exec(); } catch (Exception ignored) { - // log.warn("Failed to delete the image. Another container may be in use", ignored); ignored.printStackTrace(); } } @@ -196,9 +195,8 @@ public void tearDown() throws Exception { if (gravitinoServer != null) { gravitinoServer.close(); try { - // dockerClient.removeImageCmd(gravitinoServer.getDockerImageName()).exec(); + dockerClient.removeImageCmd(gravitinoServer.getDockerImageName()).exec(); } catch (Exception ignored) { - // log.warn("Failed to delete the image. Another container may be in use", ignored); ignored.printStackTrace(); } } @@ -342,8 +340,6 @@ protected void createNeededTables() { connection.commit(); } catch (Exception exception) { - // log.error(ExceptionUtils.getMessage(exception)); - // throw new SeaTunnelRuntimeException(JdbcITErrorCode.CREATE_TABLE_FAILED, exception); exception.printStackTrace(); } } @@ -365,8 +361,6 @@ protected void insertTestData() { connection.commit(); } catch (Exception exception) { - // log.error(ExceptionUtils.getMessage(exception)); - // throw new SeaTunnelRuntimeException(JdbcITErrorCode.INSERT_DATA_FAILED, exception); exception.printStackTrace(); } } diff --git a/seatunnel-engine/seatunnel-engine-core/src/main/java/org/apache/seatunnel/engine/core/parse/MultipleTableJobConfigParser.java b/seatunnel-engine/seatunnel-engine-core/src/main/java/org/apache/seatunnel/engine/core/parse/MultipleTableJobConfigParser.java index 95087dacf57..046dc0b1de4 100644 --- a/seatunnel-engine/seatunnel-engine-core/src/main/java/org/apache/seatunnel/engine/core/parse/MultipleTableJobConfigParser.java +++ b/seatunnel-engine/seatunnel-engine-core/src/main/java/org/apache/seatunnel/engine/core/parse/MultipleTableJobConfigParser.java @@ -23,7 +23,6 @@ import org.apache.seatunnel.shade.com.typesafe.config.Config; import org.apache.seatunnel.shade.com.typesafe.config.ConfigList; import org.apache.seatunnel.shade.com.typesafe.config.ConfigObject; -import org.apache.seatunnel.shade.com.typesafe.config.ConfigRenderOptions; import org.apache.seatunnel.shade.com.typesafe.config.ConfigValue; import org.apache.seatunnel.shade.com.typesafe.config.ConfigValueFactory; import org.apache.seatunnel.shade.com.typesafe.config.ConfigValueType; From 50d015860ecfcc5225c9397b6c482eda16a7d7ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=90=B4=E5=A4=A9=E5=AE=87?= <1599646055@qq.com> Date: Fri, 29 Aug 2025 13:56:54 +0800 Subject: [PATCH 15/39] feat: Add docs of Metalake in zh and en --- docs/en/seatunnel-engine/metalake.md | 69 ++++++++++++++++++++++++++++ docs/zh/seatunnel-engine/metalake.md | 69 ++++++++++++++++++++++++++++ 2 files changed, 138 insertions(+) create mode 100644 docs/en/seatunnel-engine/metalake.md create mode 100644 docs/zh/seatunnel-engine/metalake.md diff --git a/docs/en/seatunnel-engine/metalake.md b/docs/en/seatunnel-engine/metalake.md new file mode 100644 index 00000000000..1e76c995483 --- /dev/null +++ b/docs/en/seatunnel-engine/metalake.md @@ -0,0 +1,69 @@ +# METALAKE + +Since Seatunnel requires database usernames, passwords, and other sensitive information to be written in plaintext within scripts when executing tasks, this may lead to information leakage and is also difficult to maintain. When data source information changes, manual modifications are often required. + +To address this, Metalake is introduced. Data source information can be stored in Metalake systems such as Apache Gravitino. Task scripts then use `sourceId` and placeholders instead of actual usernames and passwords. At runtime, the Seatunnel engine retrieves the information from Metalake via HTTP requests and replaces the placeholders accordingly. + +To enable Metalake, you first need to modify the environment variables in **seatunnel-env.sh**: + +* `METALAKE_ENABLED` +* `METALAKE_TYPE` +* `METALAKE_URL` + +Set `METALAKE_ENABLED` to `true`. Currently, `METALAKE_TYPE` only supports `gravitino`. + +For Apache Gravitino, set `METALAKE_URL` to: + +``` +http://host:port/api/metalakes/your_metalake_name/catalogs/ +``` + +--- + +## Usage Example + +First, create a catalog in Gravitino, for example: + +```bash +curl -L 'http://localhost:8090/api/metalakes/test_metalake/catalogs' \ +-H 'Content-Type: application/json' \ +-H 'Accept: application/vnd.gravitino.v1+json' \ +-d '{ + "name": "test_catalog", + "type": "relational", + "provider": "jdbc-mysql", + "comment": "for metalake test", + "properties": { + "jdbc-driver": "com.mysql.cj.jdbc.Driver", + "jdbc-url": "not used", + "jdbc-user": "root", + "jdbc-password": "Abc!@#135_seatunnel" + } +}' +``` + +This creates a `test_catalog` under `test_metalake` (note: `metalake` itself must be created in advance). + +Thus, `METALAKE_URL` can be set to: + +``` +http://localhost:8090/api/metalakes/test_metalake/catalogs/ +``` + +You can then define the source as: + +```hocon +source { + Jdbc { + url = "jdbc:mysql://mysql-e2e:3306/seatunnel?useSSL=false&serverTimezone=UTC&allowPublicKeyRetrieval=true" + driver = "${jdbc-driver}" + connection_check_timeout_sec = 100 + sourceId = "test_catalog" + user = "${jdbc-user}" + password = "${jdbc-password}" + query = "select * from source" + } +} +``` + +Here, `sourceId` refers to the catalog name, allowing other fields to use `${}` placeholders. At runtime, they will be automatically replaced. Note that in sinks, the same `sourceId` name is used, and placeholders must always start with `${` and end with `}`. \ No newline at end of file diff --git a/docs/zh/seatunnel-engine/metalake.md b/docs/zh/seatunnel-engine/metalake.md new file mode 100644 index 00000000000..f702680ea3f --- /dev/null +++ b/docs/zh/seatunnel-engine/metalake.md @@ -0,0 +1,69 @@ +# METALAKE + +由于seatunnel在执行任务时，需要将数据库用户名与密码等隐私信息明文写在脚本中，可能会导致信息泄露；并且维护较为困难，数据源信息发生变更时可能需要手动更改。 + +因此引入了metalake，将数据源的信息存储于Apache Gravitino等metalake中，任务脚本采用`sourId`和占位符的方法来代替原本的用户名和密码等信息，运行时seatunnel-engine通过http请求从metalake获取信息，根据占位符进行替换。 + +若要使用metalake，首先要修改**seatunnel-env.sh**中的环境变量： + +* `METALAKE_ENABLED` +* `METALAKE_TYPE` +* `METALAKE_URL` + +将`METALAKE_ENABLED`设为`true`，`METALAKE_TYPE`当前仅支持设为`gravitino`。 + +对于Apache Gravitino，`METALAKE_URL`设为 + +``` +http://host:port/api/metalakes/your_metalake_name/catalogs/ +``` + +--- + +## 使用示例： + +用户需要先在Gravitino中创建catalog，如 + +```bash +curl -L 'http://localhost:8090/api/metalakes/test_metalake/catalogs' +-H 'Content-Type: application/json' +-H 'Accept: application/vnd.gravitino.v1+json' +-d '{ + "name": "test_catalog", + "type": "relational", + "provider": "jdbc-mysql", + "comment": "for metalake test", + "properties": { + "jdbc-driver": "com.mysql.cj.jdbc.Driver", + "jdbc-url": "not used", + "jdbc-user": "root", + "jdbc-password": "Abc!@#135_seatunnel" + } +}' +``` + +这样便在`test_metalake`中创建了一个`test_catalog`（`metalake`需要提前创建） + +于是`METALAKE_URL`可以设为 + +``` +http://localhost:8090/api/metalakes/test_metalake/catalogs/ +``` + +source可以写为 + +``` +source { + Jdbc { + url = "jdbc:mysql://mysql-e2e:3306/seatunnel?useSSL=false&serverTimezone=UTC&allowPublicKeyRetrieval=true" + driver = "${jdbc-driver}" + connection_check_timeout_sec = 100 + sourceId = "test_catalog" + user = "${jdbc-user}" + password = "${jdbc-password}" + query = "select * from source" + } +} +``` + +其中`sourceId`指代catalog的名称，从而其他项可以使用`${}`占位符，运行时会自动替换。注意，在sink中使用时，同样叫`sourceId`；使用占位符时必须以`${`开头，以`}`结尾 \ No newline at end of file From 84a2b0b844e1d6bf52f417cd28990ba9aafd9b77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=90=B4=E5=A4=A9=E5=AE=87?= <1599646055@qq.com> Date: Mon, 1 Sep 2025 12:46:28 +0800 Subject: [PATCH 16/39] push an empty commit to trigger the workflow From 1acfb14f754bd5745763370637aaa78a4736ee04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=90=B4=E5=A4=A9=E5=AE=87?= <1599646055@qq.com> Date: Mon, 1 Sep 2025 17:13:28 +0800 Subject: [PATCH 17/39] fix: add license header to conf file --- .../mysql_to_console_with_metalake.conf | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/mysql_to_console_with_metalake.conf b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/mysql_to_console_with_metalake.conf index 47b3ebc649c..f12376a3b62 100644 --- a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/mysql_to_console_with_metalake.conf +++ b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/mysql_to_console_with_metalake.conf @@ -1,3 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + env { execution.parallelism = 1 job.mode = "BATCH" From cfeb84a7dcad07dfc9002bfe8587c7486c755ef7 Mon Sep 17 00:00:00 2001 From: wtybxqm <1599646055@qq.com> Date: Wed, 3 Sep 2025 01:09:40 +0800 Subject: [PATCH 18/39] feat: download gravitino in test container --- .../engine/e2e/metalake/MetalakeIT.java | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/metalake/MetalakeIT.java b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/metalake/MetalakeIT.java index 90b9be5c7c2..62290590b24 100644 --- a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/metalake/MetalakeIT.java +++ b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/metalake/MetalakeIT.java @@ -124,7 +124,7 @@ public void startUp() throws Exception { .withEnv("METALAKE_TYPE", "gravitino") .withEnv( "METALAKE_URL", - "http://172.17.0.1:8090/api/metalakes/test_metalake/catalogs/") + "http://127.0.0.1:8090/api/metalakes/test_metalake/catalogs/") .withCommand(buildStartCommand()) .withNetworkAliases("server") .withExposedPorts() @@ -156,7 +156,15 @@ public void startUp() throws Exception { "-c", "mkdir -p /tmp/seatunnel/plugins/Jdbc/lib && cd /tmp/seatunnel/plugins/Jdbc/lib && wget " + driverUrl() - + " --no-check-certificate"); + + " --no-check-certificate" + + "&& mkdir -p /tmp/gravitino && cd /tmp/gravitino && curl -C - --retry 5 -L -k -o gravitino-0.9.1-bin.tar.gz https://dlcdn.apache.org/gravitino/0.9.1/gravitino-0.9.1-bin.tar.gz && tar -zxvf gravitino-0.9.1-bin.tar.gz && cd /tmp/gravitino/gravitino-0.9.1-bin && ./bin/gravitino.sh start"); + + server.execInContainer( + "bash", + "-c", + "sleep 60 && curl -L 'http://127.0.0.1:8090/api/metalakes' -H 'Content-Type: application/json' -H 'Accept: application/vnd.gravitino.v1+json' -d '{\"name\":\"test_metalake\",\"comment\":\"for metalake test\",\"properties\":{}}'" + + "&& curl -L 'http://127.0.0.1:8090/api/metalakes/test_metalake/catalogs' -H 'Content-Type: application/json' -H 'Accept: application/vnd.gravitino.v1+json' -d '{\"name\":\"test_catalog\",\"type\":\"relational\",\"provider\":\"jdbc-mysql\",\"comment\":\"for metalake test\",\"properties\":{\"jdbc-driver\":\"com.mysql.cj.jdbc.Driver\",\"jdbc-url\":\"not used\",\"jdbc-user\":\"root\",\"jdbc-password\":\"Abc!@#135_seatunnel\"}}'"); + dbServer = initContainer().withImagePullPolicy(PullPolicy.defaultPolicy()); Startables.deepStart(Stream.of(dbServer)).join(); @@ -186,7 +194,7 @@ public void tearDown() throws Exception { if (dbServer != null) { dbServer.close(); try { - dockerClient.removeImageCmd(dbServer.getDockerImageName()).exec(); + //dockerClient.removeImageCmd(dbServer.getDockerImageName()).exec(); } catch (Exception ignored) { ignored.printStackTrace(); } @@ -195,7 +203,7 @@ public void tearDown() throws Exception { if (gravitinoServer != null) { gravitinoServer.close(); try { - dockerClient.removeImageCmd(gravitinoServer.getDockerImageName()).exec(); + //dockerClient.removeImageCmd(gravitinoServer.getDockerImageName()).exec(); } catch (Exception ignored) { ignored.printStackTrace(); } From ec25bb0f544e03378368cecd2cf67593384c3149 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=90=B4=E5=A4=A9=E5=AE=87?= <1599646055@qq.com> Date: Wed, 3 Sep 2025 01:16:09 +0800 Subject: [PATCH 19/39] feat: apply spotless codestyle and remove useless code --- .../seatunnel/engine/e2e/metalake/MetalakeIT.java | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/metalake/MetalakeIT.java b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/metalake/MetalakeIT.java index 62290590b24..f2a08c1c439 100644 --- a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/metalake/MetalakeIT.java +++ b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/metalake/MetalakeIT.java @@ -69,8 +69,6 @@ public class MetalakeIT extends SeaTunnelEngineContainer { protected GenericContainer dbServer; - protected GenericContainer gravitinoServer; - protected JdbcCase jdbcCase; protected Connection connection; @@ -165,7 +163,7 @@ public void startUp() throws Exception { "sleep 60 && curl -L 'http://127.0.0.1:8090/api/metalakes' -H 'Content-Type: application/json' -H 'Accept: application/vnd.gravitino.v1+json' -d '{\"name\":\"test_metalake\",\"comment\":\"for metalake test\",\"properties\":{}}'" + "&& curl -L 'http://127.0.0.1:8090/api/metalakes/test_metalake/catalogs' -H 'Content-Type: application/json' -H 'Accept: application/vnd.gravitino.v1+json' -d '{\"name\":\"test_catalog\",\"type\":\"relational\",\"provider\":\"jdbc-mysql\",\"comment\":\"for metalake test\",\"properties\":{\"jdbc-driver\":\"com.mysql.cj.jdbc.Driver\",\"jdbc-url\":\"not used\",\"jdbc-user\":\"root\",\"jdbc-password\":\"Abc!@#135_seatunnel\"}}'"); - dbServer = initContainer().withImagePullPolicy(PullPolicy.defaultPolicy()); + dbServer = initContainer().withImagePullPolicy(PullPolicy.alwaysPull()); Startables.deepStart(Stream.of(dbServer)).join(); @@ -194,20 +192,12 @@ public void tearDown() throws Exception { if (dbServer != null) { dbServer.close(); try { - //dockerClient.removeImageCmd(dbServer.getDockerImageName()).exec(); + dockerClient.removeImageCmd(dbServer.getDockerImageName()).exec(); } catch (Exception ignored) { ignored.printStackTrace(); } } - if (gravitinoServer != null) { - gravitinoServer.close(); - try { - //dockerClient.removeImageCmd(gravitinoServer.getDockerImageName()).exec(); - } catch (Exception ignored) { - ignored.printStackTrace(); - } - } super.tearDown(); } From 1b42050d322a050c03861d24b4fd85418be8113a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=90=B4=E5=A4=A9=E5=AE=87?= <1599646055@qq.com> Date: Sat, 6 Sep 2025 20:51:01 +0800 Subject: [PATCH 20/39] feat: support metalake for spark and flink engine; use assert connector instead of console in the conf file for test --- .../command/FlinkTaskExecuteCommand.java | 105 +++++++++++++++++- .../command/SparkTaskExecuteCommand.java | 105 +++++++++++++++++- .../mysql_to_console_with_metalake.conf | 43 ++++++- 3 files changed, 249 insertions(+), 4 deletions(-) diff --git a/seatunnel-core/seatunnel-flink-starter/seatunnel-flink-starter-common/src/main/java/org/apache/seatunnel/core/starter/flink/command/FlinkTaskExecuteCommand.java b/seatunnel-core/seatunnel-flink-starter/seatunnel-flink-starter-common/src/main/java/org/apache/seatunnel/core/starter/flink/command/FlinkTaskExecuteCommand.java index e831fb081bf..9621c3c182f 100644 --- a/seatunnel-core/seatunnel-flink-starter/seatunnel-flink-starter-common/src/main/java/org/apache/seatunnel/core/starter/flink/command/FlinkTaskExecuteCommand.java +++ b/seatunnel-core/seatunnel-flink-starter/seatunnel-flink-starter-common/src/main/java/org/apache/seatunnel/core/starter/flink/command/FlinkTaskExecuteCommand.java @@ -17,10 +17,17 @@ package org.apache.seatunnel.core.starter.flink.command; +import org.apache.seatunnel.shade.com.fasterxml.jackson.databind.JsonNode; import org.apache.seatunnel.shade.com.typesafe.config.Config; +import org.apache.seatunnel.shade.com.typesafe.config.ConfigList; +import org.apache.seatunnel.shade.com.typesafe.config.ConfigObject; import org.apache.seatunnel.shade.com.typesafe.config.ConfigUtil; +import org.apache.seatunnel.shade.com.typesafe.config.ConfigValue; import org.apache.seatunnel.shade.com.typesafe.config.ConfigValueFactory; +import org.apache.seatunnel.shade.com.typesafe.config.ConfigValueType; +import org.apache.seatunnel.api.metalake.MetalakeClient; +import org.apache.seatunnel.api.metalake.MetalakeClientFactory; import org.apache.seatunnel.common.Constants; import org.apache.seatunnel.core.starter.command.Command; import org.apache.seatunnel.core.starter.exception.CommandExecuteException; @@ -31,7 +38,11 @@ import lombok.extern.slf4j.Slf4j; +import java.io.IOException; import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; import static org.apache.seatunnel.core.starter.utils.FileUtils.checkConfigExist; @@ -48,7 +59,16 @@ public FlinkTaskExecuteCommand(FlinkCommandArgs flinkCommandArgs) { public void execute() throws CommandExecuteException { Path configFile = FileUtils.getConfigPath(flinkCommandArgs); checkConfigExist(configFile); - Config config = ConfigBuilder.of(configFile, flinkCommandArgs.getVariables()); + Config config = null; + boolean metalakeEnabled = + Boolean.parseBoolean(System.getenv().getOrDefault("METALAKE_ENABLED", "false")); + if (metalakeEnabled) { + config = + getMetalakeConfig( + ConfigBuilder.of(configFile, flinkCommandArgs.getVariables())); + } else { + config = ConfigBuilder.of(configFile, flinkCommandArgs.getVariables()); + } // if user specified job name using command line arguments, override config option if (!flinkCommandArgs.getJobName().equals(Constants.LOGO)) { config = @@ -63,4 +83,87 @@ public void execute() throws CommandExecuteException { throw new CommandExecuteException("Flink job executed failed", e); } } + + private Config getMetalakeConfig(Config jobConfigTmp) { + Config update = jobConfigTmp; + String metalakeType = System.getenv("METALAKE_TYPE"); + String metalakeUrl = System.getenv("METALAKE_URL"); + + MetalakeClient metalakeClient = MetalakeClientFactory.create(metalakeType, metalakeUrl); + + try { + ConfigList sourceList = jobConfigTmp.getList("source"); + List newSourceList = new ArrayList<>(sourceList); + + for (int i = 0; i < sourceList.size(); i++) { + ConfigObject sourceObj = (ConfigObject) sourceList.get(i); + if (sourceObj.containsKey("sourceId")) { + ConfigObject tmp = sourceObj; + String sourceId = sourceObj.toConfig().getString("sourceId"); + JsonNode metalakeJson = metalakeClient.getMetaInfo(sourceId); + for (Map.Entry entry : sourceObj.entrySet()) { + String subKey = entry.getKey(); + ConfigValue value = entry.getValue(); + + if (value.valueType() == ConfigValueType.STRING) { + String strValue = (String) value.unwrapped(); + if (strValue.startsWith("${") && strValue.endsWith("}")) { + String placeholder = strValue.substring(2, strValue.length() - 1); + + if (metalakeJson.has(placeholder)) { + String replaced = metalakeJson.get(placeholder).asText(); + tmp = + tmp.withValue( + subKey, + ConfigValueFactory.fromAnyRef(replaced)); + } + } + } + } + newSourceList.set(i, tmp); + } + } + update = update.withValue("source", ConfigValueFactory.fromIterable(newSourceList)); + } catch (IOException e) { + log.error("Fail to get MetaInfo, metalakeUrl: {}", metalakeUrl, e); + } + + try { + ConfigList sinkList = jobConfigTmp.getList("sink"); + List newSinkList = new ArrayList<>(sinkList); + + for (int i = 0; i < sinkList.size(); i++) { + ConfigObject sinkObj = (ConfigObject) sinkList.get(i); + if (sinkObj.containsKey("sourceId")) { + ConfigObject tmp = sinkObj; + String sourceId = sinkObj.toConfig().getString("sourceId"); + JsonNode metalakeJson = metalakeClient.getMetaInfo(sourceId); + for (Map.Entry entry : sinkObj.entrySet()) { + String subKey = entry.getKey(); + ConfigValue value = entry.getValue(); + + if (value.valueType() == ConfigValueType.STRING) { + String strValue = (String) value.unwrapped(); + if (strValue.startsWith("${") && strValue.endsWith("}")) { + String placeholder = strValue.substring(2, strValue.length() - 1); + + if (metalakeJson.has(placeholder)) { + String replaced = metalakeJson.get(placeholder).asText(); + tmp = + tmp.withValue( + subKey, + ConfigValueFactory.fromAnyRef(replaced)); + } + } + } + } + newSinkList.set(i, tmp); + } + } + update = update.withValue("sink", ConfigValueFactory.fromIterable(newSinkList)); + } catch (IOException e) { + log.error("Fail to get MetaInfo, metalakeUrl: {}", metalakeUrl, e); + } + return update; + } } diff --git a/seatunnel-core/seatunnel-spark-starter/seatunnel-spark-starter-common/src/main/java/org/apache/seatunnel/core/starter/spark/command/SparkTaskExecuteCommand.java b/seatunnel-core/seatunnel-spark-starter/seatunnel-spark-starter-common/src/main/java/org/apache/seatunnel/core/starter/spark/command/SparkTaskExecuteCommand.java index ea36bc0777a..fad440ef091 100644 --- a/seatunnel-core/seatunnel-spark-starter/seatunnel-spark-starter-common/src/main/java/org/apache/seatunnel/core/starter/spark/command/SparkTaskExecuteCommand.java +++ b/seatunnel-core/seatunnel-spark-starter/seatunnel-spark-starter-common/src/main/java/org/apache/seatunnel/core/starter/spark/command/SparkTaskExecuteCommand.java @@ -17,10 +17,17 @@ package org.apache.seatunnel.core.starter.spark.command; +import org.apache.seatunnel.shade.com.fasterxml.jackson.databind.JsonNode; import org.apache.seatunnel.shade.com.typesafe.config.Config; +import org.apache.seatunnel.shade.com.typesafe.config.ConfigList; +import org.apache.seatunnel.shade.com.typesafe.config.ConfigObject; import org.apache.seatunnel.shade.com.typesafe.config.ConfigUtil; +import org.apache.seatunnel.shade.com.typesafe.config.ConfigValue; import org.apache.seatunnel.shade.com.typesafe.config.ConfigValueFactory; +import org.apache.seatunnel.shade.com.typesafe.config.ConfigValueType; +import org.apache.seatunnel.api.metalake.MetalakeClient; +import org.apache.seatunnel.api.metalake.MetalakeClientFactory; import org.apache.seatunnel.common.Constants; import org.apache.seatunnel.core.starter.command.Command; import org.apache.seatunnel.core.starter.exception.CommandExecuteException; @@ -31,7 +38,11 @@ import lombok.extern.slf4j.Slf4j; +import java.io.IOException; import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; import static org.apache.seatunnel.core.starter.utils.FileUtils.checkConfigExist; @@ -48,7 +59,16 @@ public SparkTaskExecuteCommand(SparkCommandArgs sparkCommandArgs) { public void execute() throws CommandExecuteException { Path configFile = FileUtils.getConfigPath(sparkCommandArgs); checkConfigExist(configFile); - Config config = ConfigBuilder.of(configFile, sparkCommandArgs.getVariables()); + Config config = null; + boolean metalakeEnabled = + Boolean.parseBoolean(System.getenv().getOrDefault("METALAKE_ENABLED", "false")); + if (metalakeEnabled) { + config = + getMetalakeConfig( + ConfigBuilder.of(configFile, sparkCommandArgs.getVariables())); + } else { + config = ConfigBuilder.of(configFile, sparkCommandArgs.getVariables()); + } if (!sparkCommandArgs.getJobName().equals(Constants.LOGO)) { config = config.withValue( @@ -62,4 +82,87 @@ public void execute() throws CommandExecuteException { throw new CommandExecuteException("Run SeaTunnel on spark failed", e); } } + + private Config getMetalakeConfig(Config jobConfigTmp) { + Config update = jobConfigTmp; + String metalakeType = System.getenv("METALAKE_TYPE"); + String metalakeUrl = System.getenv("METALAKE_URL"); + + MetalakeClient metalakeClient = MetalakeClientFactory.create(metalakeType, metalakeUrl); + + try { + ConfigList sourceList = jobConfigTmp.getList("source"); + List newSourceList = new ArrayList<>(sourceList); + + for (int i = 0; i < sourceList.size(); i++) { + ConfigObject sourceObj = (ConfigObject) sourceList.get(i); + if (sourceObj.containsKey("sourceId")) { + ConfigObject tmp = sourceObj; + String sourceId = sourceObj.toConfig().getString("sourceId"); + JsonNode metalakeJson = metalakeClient.getMetaInfo(sourceId); + for (Map.Entry entry : sourceObj.entrySet()) { + String subKey = entry.getKey(); + ConfigValue value = entry.getValue(); + + if (value.valueType() == ConfigValueType.STRING) { + String strValue = (String) value.unwrapped(); + if (strValue.startsWith("${") && strValue.endsWith("}")) { + String placeholder = strValue.substring(2, strValue.length() - 1); + + if (metalakeJson.has(placeholder)) { + String replaced = metalakeJson.get(placeholder).asText(); + tmp = + tmp.withValue( + subKey, + ConfigValueFactory.fromAnyRef(replaced)); + } + } + } + } + newSourceList.set(i, tmp); + } + } + update = update.withValue("source", ConfigValueFactory.fromIterable(newSourceList)); + } catch (IOException e) { + log.error("Fail to get MetaInfo, metalakeUrl: {}", metalakeUrl, e); + } + + try { + ConfigList sinkList = jobConfigTmp.getList("sink"); + List newSinkList = new ArrayList<>(sinkList); + + for (int i = 0; i < sinkList.size(); i++) { + ConfigObject sinkObj = (ConfigObject) sinkList.get(i); + if (sinkObj.containsKey("sourceId")) { + ConfigObject tmp = sinkObj; + String sourceId = sinkObj.toConfig().getString("sourceId"); + JsonNode metalakeJson = metalakeClient.getMetaInfo(sourceId); + for (Map.Entry entry : sinkObj.entrySet()) { + String subKey = entry.getKey(); + ConfigValue value = entry.getValue(); + + if (value.valueType() == ConfigValueType.STRING) { + String strValue = (String) value.unwrapped(); + if (strValue.startsWith("${") && strValue.endsWith("}")) { + String placeholder = strValue.substring(2, strValue.length() - 1); + + if (metalakeJson.has(placeholder)) { + String replaced = metalakeJson.get(placeholder).asText(); + tmp = + tmp.withValue( + subKey, + ConfigValueFactory.fromAnyRef(replaced)); + } + } + } + } + newSinkList.set(i, tmp); + } + } + update = update.withValue("sink", ConfigValueFactory.fromIterable(newSinkList)); + } catch (IOException e) { + log.error("Fail to get MetaInfo, metalakeUrl: {}", metalakeUrl, e); + } + return update; + } } diff --git a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/mysql_to_console_with_metalake.conf b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/mysql_to_console_with_metalake.conf index f12376a3b62..95b1ea33a5e 100644 --- a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/mysql_to_console_with_metalake.conf +++ b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/mysql_to_console_with_metalake.conf @@ -36,6 +36,45 @@ transform { } sink { - Console { - } + Assert { + rules = + { + row_rules = [ + { + rule_type = MAX_ROW + rule_value = 101 + }, + { + rule_type = MIN_ROW + rule_value = 99 + } + ], + field_rules = [ + { + field_name = c-bit_1 + field_type = boolean + }, + { + field_name = c_bit_8 + field_type = bytes + }, + { + field_name = c_bit_16 + field_type = bytes + }, + { + field_name = c_bit_32 + field_type = bytes + }, + { + field_name = c_bit_64 + field_type = bytes + }, + { + field_name = c_bigbit_30 + field_type = "decimal(20,0)" + }, + ] + } + } } \ No newline at end of file From d67545f8fc6263b49f0623201a1d8546a7f72cda Mon Sep 17 00:00:00 2001 From: wtybxqm <1599646055@qq.com> Date: Sun, 7 Sep 2025 17:08:40 +0800 Subject: [PATCH 21/39] fix: fix the error of the assert connector --- .../engine/e2e/metalake/MetalakeIT.java | 13 +++++--- .../mysql_to_console_with_metalake.conf | 31 ++++++++++++++++--- 2 files changed, 34 insertions(+), 10 deletions(-) diff --git a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/metalake/MetalakeIT.java b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/metalake/MetalakeIT.java index f2a08c1c439..e198dbbf106 100644 --- a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/metalake/MetalakeIT.java +++ b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/metalake/MetalakeIT.java @@ -379,11 +379,14 @@ Pair> initTestData() { row = new SeaTunnelRow( new Object[] { - null, - null, - null, - null, - null, + (byte) 0, + new byte[] {byteArr}, + new byte[] {byteArr, byteArr}, + new byte[] {byteArr, byteArr, byteArr, byteArr}, + new byte[] { + byteArr, byteArr, byteArr, byteArr, byteArr, byteArr, + byteArr, byteArr + }, // https://github.com/apache/seatunnel/issues/5559 this value // cannot set null, this null // value column's row will be lost in diff --git a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/mysql_to_console_with_metalake.conf b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/mysql_to_console_with_metalake.conf index 95b1ea33a5e..0038bbd9dd6 100644 --- a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/mysql_to_console_with_metalake.conf +++ b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/mysql_to_console_with_metalake.conf @@ -50,29 +50,50 @@ sink { } ], field_rules = [ - { - field_name = c-bit_1 - field_type = boolean - }, { field_name = c_bit_8 field_type = bytes + field_value = [ + { + rule_type = NOT_NULL + } + ] }, { field_name = c_bit_16 field_type = bytes + field_value = [ + { + rule_type = NOT_NULL + } + ] }, { field_name = c_bit_32 field_type = bytes + field_value = [ + { + rule_type = NOT_NULL + } + ] }, { field_name = c_bit_64 field_type = bytes + field_value = [ + { + rule_type = NOT_NULL + } + ] }, { - field_name = c_bigbit_30 + field_name = c_bigint_30 field_type = "decimal(20,0)" + field_value = [ + { + rule_type = NOT_NULL + } + ] }, ] } From 4013ed82dc648630cf0d9ae706dd10c95f72041a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=90=B4=E5=A4=A9=E5=AE=87?= <1599646055@qq.com> Date: Sun, 7 Sep 2025 23:26:05 +0800 Subject: [PATCH 22/39] fix: move metalakeIT to seatunnel-connector-v2-e2e --- .../connectors/seatunnel/jdbc/MetalakeIT.java | 457 ++++++++++++++++++ ...l_source_to_assert_sink_with_metalake.conf | 101 ++++ 2 files changed, 558 insertions(+) create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-7/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/MetalakeIT.java create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-7/src/test/resources/jdbc_mysql_source_to_assert_sink_with_metalake.conf diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-7/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/MetalakeIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-7/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/MetalakeIT.java new file mode 100644 index 00000000000..7adde4271c9 --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-7/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/MetalakeIT.java @@ -0,0 +1,457 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.jdbc; + +import org.apache.seatunnel.e2e.common.container.seatunnel.SeaTunnelContainer; +import org.apache.seatunnel.shade.com.google.common.collect.Lists; + +import org.apache.seatunnel.api.table.catalog.Catalog; +import org.apache.seatunnel.api.table.type.SeaTunnelRow; + +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.tuple.Pair; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.testcontainers.DockerClientFactory; +import org.testcontainers.containers.Container; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.MySQLContainer; +import org.testcontainers.containers.output.Slf4jLogConsumer; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.images.PullPolicy; +import org.testcontainers.lifecycle.Startables; +import org.testcontainers.utility.DockerImageName; +import org.testcontainers.utility.DockerLoggerFactory; +import org.testcontainers.utility.MountableFile; + +import com.github.dockerjava.api.DockerClient; + +import java.io.IOException; +import java.math.BigDecimal; +import java.nio.file.Paths; +import java.sql.Connection; +import java.sql.Driver; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.apache.seatunnel.e2e.common.util.ContainerUtil.PROJECT_ROOT_PATH; +import static org.awaitility.Awaitility.given; + +public class MetalakeIT extends SeaTunnelContainer { + + protected GenericContainer dbServer; + + protected JdbcCase jdbcCase; + + protected Connection connection; + + protected Catalog catalog; + + protected DockerClient dockerClient = DockerClientFactory.lazyClient(); + + protected static final String HOST = "HOST"; + + private static final String MYSQL_IMAGE = "mysql:8.0"; + private static final String MYSQL_CONTAINER_HOST = "mysql-e2e"; + private static final String MYSQL_DATABASE = "seatunnel"; + private static final String MYSQL_SOURCE = "source"; + private static final String MYSQL_SINK = "sink"; + private static final String CATALOG_DATABASE = "catalog_database"; + + private static final String MYSQL_USERNAME = "root"; + private static final String MYSQL_PASSWORD = "Abc!@#135_seatunnel"; + private static final int MYSQL_PORT = 3306; + private static final String MYSQL_URL = "jdbc:mysql://" + HOST + ":%s/%s?useSSL=false"; + private static final String URL = "jdbc:mysql://" + HOST + ":3306/seatunnel"; + + private static final String SQL = "select * from seatunnel.source"; + + private static final String DRIVER_CLASS = "com.mysql.cj.jdbc.Driver"; + + private static final List CONFIG_FILE = + Lists.newArrayList("/mysql_to_mysql_with_metalake.conf"); + private static final String CREATE_SQL = + "CREATE TABLE IF NOT EXISTS %s\n" + + "(\n" + + " `c-bit_1` bit(1) DEFAULT NULL,\n" + + " `c_bit_8` bit(8) DEFAULT NULL,\n" + + " `c_bit_16` bit(16) DEFAULT NULL,\n" + + " `c_bit_32` bit(32) DEFAULT NULL,\n" + + " `c_bit_64` bit(64) DEFAULT NULL,\n" + + " `c_bigint_30` BIGINT(40) unsigned DEFAULT NULL,\n" + + " UNIQUE (c_bigint_30)\n" + + ");"; + + @BeforeEach + @Override + public void startUp() throws Exception { + // super.startUp(); + server = + new GenericContainer<>(getDockerImage()) + .withNetwork(NETWORK) + .withEnv("TZ", "UTC") + .withEnv("METALAKE_ENABLED", "true") + .withEnv("METALAKE_TYPE", "gravitino") + .withEnv( + "METALAKE_URL", + "http://127.0.0.1:8090/api/metalakes/test_metalake/catalogs/") + .withCommand(buildStartCommand()) + .withNetworkAliases("server") + .withExposedPorts() + .withFileSystemBind("/tmp", "/opt/hive") + .withLogConsumer( + new Slf4jLogConsumer( + DockerLoggerFactory.getLogger( + "seatunnel-engine:" + JDK_DOCKER_IMAGE))) + .waitingFor(Wait.forLogMessage(".*received new worker register:.*", 1)); + copySeaTunnelStarterToContainer(server); + server.setPortBindings(Arrays.asList("5801:5801", "8080:8080")); + server.withCopyFileToContainer( + MountableFile.forHostPath( + PROJECT_ROOT_PATH + + "/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/"), + Paths.get(SEATUNNEL_HOME, "config").toString()); + + server.withCopyFileToContainer( + MountableFile.forHostPath( + PROJECT_ROOT_PATH + + "/seatunnel-shade/seatunnel-hadoop3-3.1.4-uber/target/seatunnel-hadoop3-3.1.4-uber.jar"), + Paths.get(SEATUNNEL_HOME, "lib/seatunnel-hadoop3-3.1.4-uber.jar").toString()); + // execute extra commands + executeExtraCommands(server); + server.start(); + + server.execInContainer( + "bash", + "-c", + "mkdir -p /tmp/seatunnel/plugins/Jdbc/lib && cd /tmp/seatunnel/plugins/Jdbc/lib && wget " + + driverUrl() + + " --no-check-certificate" + + "&& mkdir -p /tmp/gravitino && cd /tmp/gravitino && curl -C - --retry 5 -L -k -o gravitino-0.9.1-bin.tar.gz https://dlcdn.apache.org/gravitino/0.9.1/gravitino-0.9.1-bin.tar.gz && tar -zxvf gravitino-0.9.1-bin.tar.gz && cd /tmp/gravitino/gravitino-0.9.1-bin && ./bin/gravitino.sh start"); + + server.execInContainer( + "bash", + "-c", + "sleep 60 && curl -L 'http://127.0.0.1:8090/api/metalakes' -H 'Content-Type: application/json' -H 'Accept: application/vnd.gravitino.v1+json' -d '{\"name\":\"test_metalake\",\"comment\":\"for metalake test\",\"properties\":{}}'" + + "&& curl -L 'http://127.0.0.1:8090/api/metalakes/test_metalake/catalogs' -H 'Content-Type: application/json' -H 'Accept: application/vnd.gravitino.v1+json' -d '{\"name\":\"test_catalog\",\"type\":\"relational\",\"provider\":\"jdbc-mysql\",\"comment\":\"for metalake test\",\"properties\":{\"jdbc-driver\":\"com.mysql.cj.jdbc.Driver\",\"jdbc-url\":\"not used\",\"jdbc-user\":\"root\",\"jdbc-password\":\"Abc!@#135_seatunnel\"}}'"); + + dbServer = initContainer().withImagePullPolicy(PullPolicy.alwaysPull()); + + Startables.deepStart(Stream.of(dbServer)).join(); + + jdbcCase = getJdbcCase(); + + given().ignoreExceptions() + .await() + .atMost(360, TimeUnit.SECONDS) + .untilAsserted(() -> this.initializeJdbcConnection(jdbcCase.getJdbcUrl())); + + createNeededTables(); + insertTestData(); + } + + @AfterEach + @Override + public void tearDown() throws Exception { + if (catalog != null) { + catalog.close(); + } + + if (connection != null) { + connection.close(); + } + + if (dbServer != null) { + dbServer.close(); + try { + dockerClient.removeImageCmd(dbServer.getDockerImageName()).exec(); + } catch (Exception ignored) { + ignored.printStackTrace(); + } + } + + super.tearDown(); + } + + @Test + public void TestMetalake() throws IOException, InterruptedException { + Container.ExecResult execResult = + executeJob("/mysql_to_console_with_metalake.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + } + + String driverUrl() { + return "https://repo1.maven.org/maven2/com/mysql/mysql-connector-j/8.0.32/mysql-connector-j-8.0.32.jar"; + } + + protected GenericContainer initContainer() { + DockerImageName imageName = DockerImageName.parse(MYSQL_IMAGE); + + GenericContainer container = + new MySQLContainer<>(imageName) + .withUsername(MYSQL_USERNAME) + .withPassword(MYSQL_PASSWORD) + .withDatabaseName(MYSQL_DATABASE) + .withNetwork(NETWORK) + .withNetworkAliases(MYSQL_CONTAINER_HOST) + .withExposedPorts(MYSQL_PORT) + .waitingFor(Wait.forHealthcheck()) + .withLogConsumer( + new Slf4jLogConsumer(DockerLoggerFactory.getLogger(MYSQL_IMAGE))); + + container.setPortBindings( + Lists.newArrayList(String.format("%s:%s", MYSQL_PORT, MYSQL_PORT))); + + return container; + } + + JdbcCase getJdbcCase() { + Map containerEnv = new HashMap<>(); + String jdbcUrl = String.format(MYSQL_URL, MYSQL_PORT, MYSQL_DATABASE); + Pair> testDataSet = initTestData(); + String[] fieldNames = testDataSet.getKey(); + + String insertSql = insertTable(MYSQL_DATABASE, MYSQL_SOURCE, fieldNames); + + return JdbcCase.builder() + .dockerImage(MYSQL_IMAGE) + .networkAliases(MYSQL_CONTAINER_HOST) + .containerEnv(containerEnv) + .driverClass(DRIVER_CLASS) + .host(HOST) + .port(MYSQL_PORT) + .localPort(MYSQL_PORT) + .jdbcTemplate(MYSQL_URL) + .jdbcUrl(jdbcUrl) + .userName(MYSQL_USERNAME) + .password(MYSQL_PASSWORD) + .database(MYSQL_DATABASE) + .sourceTable(MYSQL_SOURCE) + .sinkTable(MYSQL_SINK) + .createSql(CREATE_SQL) + .configFile(CONFIG_FILE) + .insertSql(insertSql) + .testData(testDataSet) + .catalogDatabase(CATALOG_DATABASE) + .catalogTable(MYSQL_SINK) + .tablePathFullName(MYSQL_DATABASE + "." + MYSQL_SOURCE) + .build(); + } + + protected void initializeJdbcConnection(String jdbcUrl) + throws SQLException, InstantiationException, IllegalAccessException { + Driver driver = (Driver) loadDriverClass().newInstance(); + Properties props = new Properties(); + + if (StringUtils.isNotBlank(jdbcCase.getUserName())) { + props.put("user", jdbcCase.getUserName()); + } + + if (StringUtils.isNotBlank(jdbcCase.getPassword())) { + props.put("password", jdbcCase.getPassword()); + } + + if (dbServer != null) { + jdbcUrl = jdbcUrl.replace(HOST, dbServer.getHost()); + } + + this.connection = driver.connect(jdbcUrl, props); + connection.setAutoCommit(false); + } + + protected void createNeededTables() { + try (Statement statement = connection.createStatement()) { + String createTemplate = jdbcCase.getCreateSql(); + + String createSource = + String.format( + createTemplate, + buildTableInfoWithSchema( + jdbcCase.getDatabase(), + jdbcCase.getSchema(), + jdbcCase.getSourceTable())); + statement.execute(createSource); + + if (jdbcCase.getAdditionalSqlOnSource() != null) { + String additionalSql = + String.format( + jdbcCase.getAdditionalSqlOnSource(), + buildTableInfoWithSchema( + jdbcCase.getDatabase(), + jdbcCase.getSchema(), + jdbcCase.getSourceTable())); + statement.execute(additionalSql); + } + + if (!jdbcCase.isUseSaveModeCreateTable()) { + if (jdbcCase.getSinkCreateSql() != null) { + createTemplate = jdbcCase.getSinkCreateSql(); + } + String createSink = + String.format( + createTemplate, + buildTableInfoWithSchema( + jdbcCase.getDatabase(), + jdbcCase.getSchema(), + jdbcCase.getSinkTable())); + statement.execute(createSink); + } + + if (jdbcCase.getAdditionalSqlOnSink() != null) { + String additionalSql = + String.format( + jdbcCase.getAdditionalSqlOnSink(), + buildTableInfoWithSchema( + jdbcCase.getDatabase(), + jdbcCase.getSchema(), + jdbcCase.getSinkTable())); + statement.execute(additionalSql); + } + + connection.commit(); + } catch (Exception exception) { + exception.printStackTrace(); + } + } + + protected void insertTestData() { + try (PreparedStatement preparedStatement = + connection.prepareStatement(jdbcCase.getInsertSql())) { + + List rows = jdbcCase.getTestData().getValue(); + + for (SeaTunnelRow row : rows) { + for (int index = 0; index < row.getArity(); index++) { + preparedStatement.setObject(index + 1, row.getField(index)); + } + preparedStatement.addBatch(); + } + + preparedStatement.executeBatch(); + + connection.commit(); + } catch (Exception exception) { + exception.printStackTrace(); + } + } + + Pair> initTestData() { + String[] fieldNames = + new String[] { + "c-bit_1", "c_bit_8", "c_bit_16", "c_bit_32", "c_bit_64", "c_bigint_30", + }; + + List rows = new ArrayList<>(); + BigDecimal bigintValue = new BigDecimal("2844674407371055000"); + BigDecimal decimalValue = new BigDecimal("999999999999999999999999999899"); + for (int i = 0; i < 100; i++) { + byte byteArr = Integer.valueOf(i).byteValue(); + SeaTunnelRow row; + if (i == 99) { + row = + new SeaTunnelRow( + new Object[] { + (byte) 0, + new byte[] {byteArr}, + new byte[] {byteArr, byteArr}, + new byte[] {byteArr, byteArr, byteArr, byteArr}, + new byte[] { + byteArr, byteArr, byteArr, byteArr, byteArr, byteArr, + byteArr, byteArr + }, + // https://github.com/apache/seatunnel/issues/5559 this value + // cannot set null, this null + // value column's row will be lost in + // jdbc_mysql_source_and_sink_parallel.conf,jdbc_mysql_source_and_sink_parallel_upper_lower.conf. + bigintValue.add(BigDecimal.valueOf(i)), + }); + } else { + row = + new SeaTunnelRow( + new Object[] { + i % 2 == 0 ? (byte) 1 : (byte) 0, + new byte[] {byteArr}, + new byte[] {byteArr, byteArr}, + new byte[] {byteArr, byteArr, byteArr, byteArr}, + new byte[] { + byteArr, byteArr, byteArr, byteArr, byteArr, byteArr, + byteArr, byteArr + }, + bigintValue.add(BigDecimal.valueOf(i)), + }); + } + rows.add(row); + } + + return Pair.of(fieldNames, rows); + } + + public String insertTable(String schema, String table, String... fields) { + String columns = + Arrays.stream(fields).map(this::quoteIdentifier).collect(Collectors.joining(", ")); + String placeholders = Arrays.stream(fields).map(f -> "?").collect(Collectors.joining(", ")); + + return "INSERT INTO " + + buildTableInfoWithSchema(schema, table) + + " (" + + columns + + " )" + + " VALUES (" + + placeholders + + ")"; + } + + protected Class loadDriverClass() { + try { + return Class.forName(jdbcCase.getDriverClass()); + } catch (Exception e) { + throw new RuntimeException( + "Failed to load driver class: " + jdbcCase.getDriverClass(), e); + } + } + + protected String buildTableInfoWithSchema(String database, String schema, String table) { + return buildTableInfoWithSchema(database, table); + } + + public String buildTableInfoWithSchema(String schema, String table) { + if (StringUtils.isNotBlank(schema)) { + return quoteIdentifier(schema) + "." + quoteIdentifier(table); + } else { + return quoteIdentifier(table); + } + } + + public String quoteIdentifier(String field) { + return "`" + field + "`"; + } +} + diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-7/src/test/resources/jdbc_mysql_source_to_assert_sink_with_metalake.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-7/src/test/resources/jdbc_mysql_source_to_assert_sink_with_metalake.conf new file mode 100644 index 00000000000..0038bbd9dd6 --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-7/src/test/resources/jdbc_mysql_source_to_assert_sink_with_metalake.conf @@ -0,0 +1,101 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +env { + execution.parallelism = 1 + job.mode = "BATCH" +} + +source { + Jdbc { + url = "jdbc:mysql://mysql-e2e:3306/seatunnel?useSSL=false&serverTimezone=UTC&allowPublicKeyRetrieval=true" + driver = "com.mysql.cj.jdbc.Driver" + connection_check_timeout_sec = 100 + sourceId = "test_catalog" + user = "${jdbc-user}" + password = "${jdbc-password}" + query = "select * from source" + } +} + +transform { +} + +sink { + Assert { + rules = + { + row_rules = [ + { + rule_type = MAX_ROW + rule_value = 101 + }, + { + rule_type = MIN_ROW + rule_value = 99 + } + ], + field_rules = [ + { + field_name = c_bit_8 + field_type = bytes + field_value = [ + { + rule_type = NOT_NULL + } + ] + }, + { + field_name = c_bit_16 + field_type = bytes + field_value = [ + { + rule_type = NOT_NULL + } + ] + }, + { + field_name = c_bit_32 + field_type = bytes + field_value = [ + { + rule_type = NOT_NULL + } + ] + }, + { + field_name = c_bit_64 + field_type = bytes + field_value = [ + { + rule_type = NOT_NULL + } + ] + }, + { + field_name = c_bigint_30 + field_type = "decimal(20,0)" + field_value = [ + { + rule_type = NOT_NULL + } + ] + }, + ] + } + } +} \ No newline at end of file From d0b1ff2b392bb16a86f03ccd538deff238445c89 Mon Sep 17 00:00:00 2001 From: wtybxqm <1599646055@qq.com> Date: Mon, 8 Sep 2025 10:16:05 +0800 Subject: [PATCH 23/39] fix: apply spotless code style --- .../connectors/seatunnel/jdbc/MetalakeIT.java | 53 +++++++++---------- .../engine/e2e/metalake/MetalakeIT.java | 4 +- 2 files changed, 28 insertions(+), 29 deletions(-) diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-7/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/MetalakeIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-7/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/MetalakeIT.java index 7adde4271c9..45ff2538b4c 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-7/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/MetalakeIT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-7/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/MetalakeIT.java @@ -17,11 +17,11 @@ package org.apache.seatunnel.connectors.seatunnel.jdbc; -import org.apache.seatunnel.e2e.common.container.seatunnel.SeaTunnelContainer; import org.apache.seatunnel.shade.com.google.common.collect.Lists; import org.apache.seatunnel.api.table.catalog.Catalog; import org.apache.seatunnel.api.table.type.SeaTunnelRow; +import org.apache.seatunnel.e2e.common.container.seatunnel.SeaTunnelContainer; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; @@ -204,7 +204,7 @@ public void tearDown() throws Exception { @Test public void TestMetalake() throws IOException, InterruptedException { Container.ExecResult execResult = - executeJob("/mysql_to_console_with_metalake.conf"); + executeJob("/jdbc_mysql_source_to_assert_sink_with_metalake.conf"); Assertions.assertEquals(0, execResult.getExitCode()); } @@ -344,7 +344,7 @@ protected void createNeededTables() { protected void insertTestData() { try (PreparedStatement preparedStatement = - connection.prepareStatement(jdbcCase.getInsertSql())) { + connection.prepareStatement(jdbcCase.getInsertSql())) { List rows = jdbcCase.getTestData().getValue(); @@ -366,7 +366,7 @@ protected void insertTestData() { Pair> initTestData() { String[] fieldNames = new String[] { - "c-bit_1", "c_bit_8", "c_bit_16", "c_bit_32", "c_bit_64", "c_bigint_30", + "c-bit_1", "c_bit_8", "c_bit_16", "c_bit_32", "c_bit_64", "c_bigint_30", }; List rows = new ArrayList<>(); @@ -379,33 +379,33 @@ Pair> initTestData() { row = new SeaTunnelRow( new Object[] { - (byte) 0, - new byte[] {byteArr}, - new byte[] {byteArr, byteArr}, - new byte[] {byteArr, byteArr, byteArr, byteArr}, - new byte[] { - byteArr, byteArr, byteArr, byteArr, byteArr, byteArr, - byteArr, byteArr - }, - // https://github.com/apache/seatunnel/issues/5559 this value - // cannot set null, this null - // value column's row will be lost in - // jdbc_mysql_source_and_sink_parallel.conf,jdbc_mysql_source_and_sink_parallel_upper_lower.conf. - bigintValue.add(BigDecimal.valueOf(i)), + (byte) 0, + new byte[] {byteArr}, + new byte[] {byteArr, byteArr}, + new byte[] {byteArr, byteArr, byteArr, byteArr}, + new byte[] { + byteArr, byteArr, byteArr, byteArr, byteArr, byteArr, + byteArr, byteArr + }, + // https://github.com/apache/seatunnel/issues/5559 this value + // cannot set null, this null + // value column's row will be lost in + // jdbc_mysql_source_and_sink_parallel.conf,jdbc_mysql_source_and_sink_parallel_upper_lower.conf. + bigintValue.add(BigDecimal.valueOf(i)), }); } else { row = new SeaTunnelRow( new Object[] { - i % 2 == 0 ? (byte) 1 : (byte) 0, - new byte[] {byteArr}, - new byte[] {byteArr, byteArr}, - new byte[] {byteArr, byteArr, byteArr, byteArr}, - new byte[] { - byteArr, byteArr, byteArr, byteArr, byteArr, byteArr, - byteArr, byteArr - }, - bigintValue.add(BigDecimal.valueOf(i)), + i % 2 == 0 ? (byte) 1 : (byte) 0, + new byte[] {byteArr}, + new byte[] {byteArr, byteArr}, + new byte[] {byteArr, byteArr, byteArr, byteArr}, + new byte[] { + byteArr, byteArr, byteArr, byteArr, byteArr, byteArr, + byteArr, byteArr + }, + bigintValue.add(BigDecimal.valueOf(i)), }); } rows.add(row); @@ -454,4 +454,3 @@ public String quoteIdentifier(String field) { return "`" + field + "`"; } } - diff --git a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/metalake/MetalakeIT.java b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/metalake/MetalakeIT.java index e198dbbf106..36aabf8a749 100644 --- a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/metalake/MetalakeIT.java +++ b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/metalake/MetalakeIT.java @@ -384,8 +384,8 @@ Pair> initTestData() { new byte[] {byteArr, byteArr}, new byte[] {byteArr, byteArr, byteArr, byteArr}, new byte[] { - byteArr, byteArr, byteArr, byteArr, byteArr, byteArr, - byteArr, byteArr + byteArr, byteArr, byteArr, byteArr, byteArr, byteArr, + byteArr, byteArr }, // https://github.com/apache/seatunnel/issues/5559 this value // cannot set null, this null From 618fa288c2e3a319f71fba5656f503fcc2c434a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=90=B4=E5=A4=A9=E5=AE=87?= <1599646055@qq.com> Date: Mon, 8 Sep 2025 10:20:41 +0800 Subject: [PATCH 24/39] fix: remove metalakeIT in seatunnel-engine-e2e --- .../connector-seatunnel-e2e-base/pom.xml | 12 - .../engine/e2e/metalake/JdbcCase.java | 66 --- .../engine/e2e/metalake/MetalakeIT.java | 456 ------------------ .../mysql_to_console_with_metalake.conf | 101 ---- 4 files changed, 635 deletions(-) delete mode 100644 seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/metalake/JdbcCase.java delete mode 100644 seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/metalake/MetalakeIT.java delete mode 100644 seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/mysql_to_console_with_metalake.conf diff --git a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/pom.xml b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/pom.xml index b1d84ce6bae..d8f18347ec0 100644 --- a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/pom.xml +++ b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/pom.xml @@ -151,18 +151,6 @@ ${project.version} test - - org.testcontainers - mysql - 1.17.6 - test - - - mysql - mysql-connector-java - 8.0.27 - test - diff --git a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/metalake/JdbcCase.java b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/metalake/JdbcCase.java deleted file mode 100644 index 68594cd5764..00000000000 --- a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/metalake/JdbcCase.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.engine.e2e.metalake; - -import org.apache.seatunnel.api.table.type.SeaTunnelRow; - -import org.apache.commons.lang3.tuple.Pair; - -import lombok.Builder; -import lombok.Getter; -import lombok.Setter; - -import java.util.List; -import java.util.Map; - -@Builder -@Setter -@Getter -public class JdbcCase { - private String dockerImage; - private String networkAliases; - private String driverClass; - private String host; - private String userName; - private String password; - private int port; - private int localPort; - private String database; - private String schema; - private String sourceTable; - private String sinkTable; - private String jdbcTemplate; - private String jdbcUrl; - private String createSql; - private String sinkCreateSql; - private String additionalSqlOnSource; - private String additionalSqlOnSink; - private String insertSql; - private List configFile; - private Pair> testData; - private Map containerEnv; - private boolean useSaveModeCreateTable; - - private String catalogDatabase; - private String catalogSchema; - private String catalogTable; - - // The full path of the table created when initializing data - // According to whether jdbc api supports setting - private String tablePathFullName; -} diff --git a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/metalake/MetalakeIT.java b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/metalake/MetalakeIT.java deleted file mode 100644 index 36aabf8a749..00000000000 --- a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/java/org/apache/seatunnel/engine/e2e/metalake/MetalakeIT.java +++ /dev/null @@ -1,456 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.engine.e2e.metalake; - -import org.apache.seatunnel.shade.com.google.common.collect.Lists; - -import org.apache.seatunnel.api.table.catalog.Catalog; -import org.apache.seatunnel.api.table.type.SeaTunnelRow; -import org.apache.seatunnel.engine.e2e.SeaTunnelEngineContainer; - -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.tuple.Pair; - -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; -import org.testcontainers.DockerClientFactory; -import org.testcontainers.containers.Container; -import org.testcontainers.containers.GenericContainer; -import org.testcontainers.containers.MySQLContainer; -import org.testcontainers.containers.output.Slf4jLogConsumer; -import org.testcontainers.containers.wait.strategy.Wait; -import org.testcontainers.images.PullPolicy; -import org.testcontainers.lifecycle.Startables; -import org.testcontainers.utility.DockerImageName; -import org.testcontainers.utility.DockerLoggerFactory; -import org.testcontainers.utility.MountableFile; - -import com.github.dockerjava.api.DockerClient; - -import java.io.IOException; -import java.math.BigDecimal; -import java.nio.file.Paths; -import java.sql.Connection; -import java.sql.Driver; -import java.sql.PreparedStatement; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Properties; -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import static org.apache.seatunnel.e2e.common.util.ContainerUtil.PROJECT_ROOT_PATH; -import static org.awaitility.Awaitility.given; - -public class MetalakeIT extends SeaTunnelEngineContainer { - - protected GenericContainer dbServer; - - protected JdbcCase jdbcCase; - - protected Connection connection; - - protected Catalog catalog; - - protected DockerClient dockerClient = DockerClientFactory.lazyClient(); - - protected static final String HOST = "HOST"; - - private static final String MYSQL_IMAGE = "mysql:8.0"; - private static final String MYSQL_CONTAINER_HOST = "mysql-e2e"; - private static final String MYSQL_DATABASE = "seatunnel"; - private static final String MYSQL_SOURCE = "source"; - private static final String MYSQL_SINK = "sink"; - private static final String CATALOG_DATABASE = "catalog_database"; - - private static final String MYSQL_USERNAME = "root"; - private static final String MYSQL_PASSWORD = "Abc!@#135_seatunnel"; - private static final int MYSQL_PORT = 3306; - private static final String MYSQL_URL = "jdbc:mysql://" + HOST + ":%s/%s?useSSL=false"; - private static final String URL = "jdbc:mysql://" + HOST + ":3306/seatunnel"; - - private static final String SQL = "select * from seatunnel.source"; - - private static final String DRIVER_CLASS = "com.mysql.cj.jdbc.Driver"; - - private static final List CONFIG_FILE = - Lists.newArrayList("/mysql_to_mysql_with_metalake.conf"); - private static final String CREATE_SQL = - "CREATE TABLE IF NOT EXISTS %s\n" - + "(\n" - + " `c-bit_1` bit(1) DEFAULT NULL,\n" - + " `c_bit_8` bit(8) DEFAULT NULL,\n" - + " `c_bit_16` bit(16) DEFAULT NULL,\n" - + " `c_bit_32` bit(32) DEFAULT NULL,\n" - + " `c_bit_64` bit(64) DEFAULT NULL,\n" - + " `c_bigint_30` BIGINT(40) unsigned DEFAULT NULL,\n" - + " UNIQUE (c_bigint_30)\n" - + ");"; - - @BeforeAll - @Override - public void startUp() throws Exception { - // super.startUp(); - server = - new GenericContainer<>(getDockerImage()) - .withNetwork(NETWORK) - .withEnv("TZ", "UTC") - .withEnv("METALAKE_ENABLED", "true") - .withEnv("METALAKE_TYPE", "gravitino") - .withEnv( - "METALAKE_URL", - "http://127.0.0.1:8090/api/metalakes/test_metalake/catalogs/") - .withCommand(buildStartCommand()) - .withNetworkAliases("server") - .withExposedPorts() - .withFileSystemBind("/tmp", "/opt/hive") - .withLogConsumer( - new Slf4jLogConsumer( - DockerLoggerFactory.getLogger( - "seatunnel-engine:" + JDK_DOCKER_IMAGE))) - .waitingFor(Wait.forLogMessage(".*received new worker register:.*", 1)); - copySeaTunnelStarterToContainer(server); - server.setPortBindings(Arrays.asList("5801:5801", "8080:8080")); - server.withCopyFileToContainer( - MountableFile.forHostPath( - PROJECT_ROOT_PATH - + "/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/"), - Paths.get(SEATUNNEL_HOME, "config").toString()); - - server.withCopyFileToContainer( - MountableFile.forHostPath( - PROJECT_ROOT_PATH - + "/seatunnel-shade/seatunnel-hadoop3-3.1.4-uber/target/seatunnel-hadoop3-3.1.4-uber.jar"), - Paths.get(SEATUNNEL_HOME, "lib/seatunnel-hadoop3-3.1.4-uber.jar").toString()); - // execute extra commands - executeExtraCommands(server); - server.start(); - - server.execInContainer( - "bash", - "-c", - "mkdir -p /tmp/seatunnel/plugins/Jdbc/lib && cd /tmp/seatunnel/plugins/Jdbc/lib && wget " - + driverUrl() - + " --no-check-certificate" - + "&& mkdir -p /tmp/gravitino && cd /tmp/gravitino && curl -C - --retry 5 -L -k -o gravitino-0.9.1-bin.tar.gz https://dlcdn.apache.org/gravitino/0.9.1/gravitino-0.9.1-bin.tar.gz && tar -zxvf gravitino-0.9.1-bin.tar.gz && cd /tmp/gravitino/gravitino-0.9.1-bin && ./bin/gravitino.sh start"); - - server.execInContainer( - "bash", - "-c", - "sleep 60 && curl -L 'http://127.0.0.1:8090/api/metalakes' -H 'Content-Type: application/json' -H 'Accept: application/vnd.gravitino.v1+json' -d '{\"name\":\"test_metalake\",\"comment\":\"for metalake test\",\"properties\":{}}'" - + "&& curl -L 'http://127.0.0.1:8090/api/metalakes/test_metalake/catalogs' -H 'Content-Type: application/json' -H 'Accept: application/vnd.gravitino.v1+json' -d '{\"name\":\"test_catalog\",\"type\":\"relational\",\"provider\":\"jdbc-mysql\",\"comment\":\"for metalake test\",\"properties\":{\"jdbc-driver\":\"com.mysql.cj.jdbc.Driver\",\"jdbc-url\":\"not used\",\"jdbc-user\":\"root\",\"jdbc-password\":\"Abc!@#135_seatunnel\"}}'"); - - dbServer = initContainer().withImagePullPolicy(PullPolicy.alwaysPull()); - - Startables.deepStart(Stream.of(dbServer)).join(); - - jdbcCase = getJdbcCase(); - - given().ignoreExceptions() - .await() - .atMost(360, TimeUnit.SECONDS) - .untilAsserted(() -> this.initializeJdbcConnection(jdbcCase.getJdbcUrl())); - - createNeededTables(); - insertTestData(); - } - - @AfterAll - @Override - public void tearDown() throws Exception { - if (catalog != null) { - catalog.close(); - } - - if (connection != null) { - connection.close(); - } - - if (dbServer != null) { - dbServer.close(); - try { - dockerClient.removeImageCmd(dbServer.getDockerImageName()).exec(); - } catch (Exception ignored) { - ignored.printStackTrace(); - } - } - - super.tearDown(); - } - - @Test - public void TestMetalake() throws IOException, InterruptedException { - Container.ExecResult execResult = - executeSeaTunnelJob("/mysql_to_console_with_metalake.conf"); - Assertions.assertEquals(0, execResult.getExitCode()); - } - - String driverUrl() { - return "https://repo1.maven.org/maven2/com/mysql/mysql-connector-j/8.0.32/mysql-connector-j-8.0.32.jar"; - } - - protected GenericContainer initContainer() { - DockerImageName imageName = DockerImageName.parse(MYSQL_IMAGE); - - GenericContainer container = - new MySQLContainer<>(imageName) - .withUsername(MYSQL_USERNAME) - .withPassword(MYSQL_PASSWORD) - .withDatabaseName(MYSQL_DATABASE) - .withNetwork(NETWORK) - .withNetworkAliases(MYSQL_CONTAINER_HOST) - .withExposedPorts(MYSQL_PORT) - .waitingFor(Wait.forHealthcheck()) - .withLogConsumer( - new Slf4jLogConsumer(DockerLoggerFactory.getLogger(MYSQL_IMAGE))); - - container.setPortBindings( - Lists.newArrayList(String.format("%s:%s", MYSQL_PORT, MYSQL_PORT))); - - return container; - } - - JdbcCase getJdbcCase() { - Map containerEnv = new HashMap<>(); - String jdbcUrl = String.format(MYSQL_URL, MYSQL_PORT, MYSQL_DATABASE); - Pair> testDataSet = initTestData(); - String[] fieldNames = testDataSet.getKey(); - - String insertSql = insertTable(MYSQL_DATABASE, MYSQL_SOURCE, fieldNames); - - return JdbcCase.builder() - .dockerImage(MYSQL_IMAGE) - .networkAliases(MYSQL_CONTAINER_HOST) - .containerEnv(containerEnv) - .driverClass(DRIVER_CLASS) - .host(HOST) - .port(MYSQL_PORT) - .localPort(MYSQL_PORT) - .jdbcTemplate(MYSQL_URL) - .jdbcUrl(jdbcUrl) - .userName(MYSQL_USERNAME) - .password(MYSQL_PASSWORD) - .database(MYSQL_DATABASE) - .sourceTable(MYSQL_SOURCE) - .sinkTable(MYSQL_SINK) - .createSql(CREATE_SQL) - .configFile(CONFIG_FILE) - .insertSql(insertSql) - .testData(testDataSet) - .catalogDatabase(CATALOG_DATABASE) - .catalogTable(MYSQL_SINK) - .tablePathFullName(MYSQL_DATABASE + "." + MYSQL_SOURCE) - .build(); - } - - protected void initializeJdbcConnection(String jdbcUrl) - throws SQLException, InstantiationException, IllegalAccessException { - Driver driver = (Driver) loadDriverClass().newInstance(); - Properties props = new Properties(); - - if (StringUtils.isNotBlank(jdbcCase.getUserName())) { - props.put("user", jdbcCase.getUserName()); - } - - if (StringUtils.isNotBlank(jdbcCase.getPassword())) { - props.put("password", jdbcCase.getPassword()); - } - - if (dbServer != null) { - jdbcUrl = jdbcUrl.replace(HOST, dbServer.getHost()); - } - - this.connection = driver.connect(jdbcUrl, props); - connection.setAutoCommit(false); - } - - protected void createNeededTables() { - try (Statement statement = connection.createStatement()) { - String createTemplate = jdbcCase.getCreateSql(); - - String createSource = - String.format( - createTemplate, - buildTableInfoWithSchema( - jdbcCase.getDatabase(), - jdbcCase.getSchema(), - jdbcCase.getSourceTable())); - statement.execute(createSource); - - if (jdbcCase.getAdditionalSqlOnSource() != null) { - String additionalSql = - String.format( - jdbcCase.getAdditionalSqlOnSource(), - buildTableInfoWithSchema( - jdbcCase.getDatabase(), - jdbcCase.getSchema(), - jdbcCase.getSourceTable())); - statement.execute(additionalSql); - } - - if (!jdbcCase.isUseSaveModeCreateTable()) { - if (jdbcCase.getSinkCreateSql() != null) { - createTemplate = jdbcCase.getSinkCreateSql(); - } - String createSink = - String.format( - createTemplate, - buildTableInfoWithSchema( - jdbcCase.getDatabase(), - jdbcCase.getSchema(), - jdbcCase.getSinkTable())); - statement.execute(createSink); - } - - if (jdbcCase.getAdditionalSqlOnSink() != null) { - String additionalSql = - String.format( - jdbcCase.getAdditionalSqlOnSink(), - buildTableInfoWithSchema( - jdbcCase.getDatabase(), - jdbcCase.getSchema(), - jdbcCase.getSinkTable())); - statement.execute(additionalSql); - } - - connection.commit(); - } catch (Exception exception) { - exception.printStackTrace(); - } - } - - protected void insertTestData() { - try (PreparedStatement preparedStatement = - connection.prepareStatement(jdbcCase.getInsertSql())) { - - List rows = jdbcCase.getTestData().getValue(); - - for (SeaTunnelRow row : rows) { - for (int index = 0; index < row.getArity(); index++) { - preparedStatement.setObject(index + 1, row.getField(index)); - } - preparedStatement.addBatch(); - } - - preparedStatement.executeBatch(); - - connection.commit(); - } catch (Exception exception) { - exception.printStackTrace(); - } - } - - Pair> initTestData() { - String[] fieldNames = - new String[] { - "c-bit_1", "c_bit_8", "c_bit_16", "c_bit_32", "c_bit_64", "c_bigint_30", - }; - - List rows = new ArrayList<>(); - BigDecimal bigintValue = new BigDecimal("2844674407371055000"); - BigDecimal decimalValue = new BigDecimal("999999999999999999999999999899"); - for (int i = 0; i < 100; i++) { - byte byteArr = Integer.valueOf(i).byteValue(); - SeaTunnelRow row; - if (i == 99) { - row = - new SeaTunnelRow( - new Object[] { - (byte) 0, - new byte[] {byteArr}, - new byte[] {byteArr, byteArr}, - new byte[] {byteArr, byteArr, byteArr, byteArr}, - new byte[] { - byteArr, byteArr, byteArr, byteArr, byteArr, byteArr, - byteArr, byteArr - }, - // https://github.com/apache/seatunnel/issues/5559 this value - // cannot set null, this null - // value column's row will be lost in - // jdbc_mysql_source_and_sink_parallel.conf,jdbc_mysql_source_and_sink_parallel_upper_lower.conf. - bigintValue.add(BigDecimal.valueOf(i)), - }); - } else { - row = - new SeaTunnelRow( - new Object[] { - i % 2 == 0 ? (byte) 1 : (byte) 0, - new byte[] {byteArr}, - new byte[] {byteArr, byteArr}, - new byte[] {byteArr, byteArr, byteArr, byteArr}, - new byte[] { - byteArr, byteArr, byteArr, byteArr, byteArr, byteArr, - byteArr, byteArr - }, - bigintValue.add(BigDecimal.valueOf(i)), - }); - } - rows.add(row); - } - - return Pair.of(fieldNames, rows); - } - - public String insertTable(String schema, String table, String... fields) { - String columns = - Arrays.stream(fields).map(this::quoteIdentifier).collect(Collectors.joining(", ")); - String placeholders = Arrays.stream(fields).map(f -> "?").collect(Collectors.joining(", ")); - - return "INSERT INTO " - + buildTableInfoWithSchema(schema, table) - + " (" - + columns - + " )" - + " VALUES (" - + placeholders - + ")"; - } - - protected Class loadDriverClass() { - try { - return Class.forName(jdbcCase.getDriverClass()); - } catch (Exception e) { - throw new RuntimeException( - "Failed to load driver class: " + jdbcCase.getDriverClass(), e); - } - } - - protected String buildTableInfoWithSchema(String database, String schema, String table) { - return buildTableInfoWithSchema(database, table); - } - - public String buildTableInfoWithSchema(String schema, String table) { - if (StringUtils.isNotBlank(schema)) { - return quoteIdentifier(schema) + "." + quoteIdentifier(table); - } else { - return quoteIdentifier(table); - } - } - - public String quoteIdentifier(String field) { - return "`" + field + "`"; - } -} diff --git a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/mysql_to_console_with_metalake.conf b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/mysql_to_console_with_metalake.conf deleted file mode 100644 index 0038bbd9dd6..00000000000 --- a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/mysql_to_console_with_metalake.conf +++ /dev/null @@ -1,101 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -env { - execution.parallelism = 1 - job.mode = "BATCH" -} - -source { - Jdbc { - url = "jdbc:mysql://mysql-e2e:3306/seatunnel?useSSL=false&serverTimezone=UTC&allowPublicKeyRetrieval=true" - driver = "com.mysql.cj.jdbc.Driver" - connection_check_timeout_sec = 100 - sourceId = "test_catalog" - user = "${jdbc-user}" - password = "${jdbc-password}" - query = "select * from source" - } -} - -transform { -} - -sink { - Assert { - rules = - { - row_rules = [ - { - rule_type = MAX_ROW - rule_value = 101 - }, - { - rule_type = MIN_ROW - rule_value = 99 - } - ], - field_rules = [ - { - field_name = c_bit_8 - field_type = bytes - field_value = [ - { - rule_type = NOT_NULL - } - ] - }, - { - field_name = c_bit_16 - field_type = bytes - field_value = [ - { - rule_type = NOT_NULL - } - ] - }, - { - field_name = c_bit_32 - field_type = bytes - field_value = [ - { - rule_type = NOT_NULL - } - ] - }, - { - field_name = c_bit_64 - field_type = bytes - field_value = [ - { - rule_type = NOT_NULL - } - ] - }, - { - field_name = c_bigint_30 - field_type = "decimal(20,0)" - field_value = [ - { - rule_type = NOT_NULL - } - ] - }, - ] - } - } -} \ No newline at end of file From dc45daa283fde445043e62569dc9c7384aae7c1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=90=B4=E5=A4=A9=E5=AE=87?= <1599646055@qq.com> Date: Tue, 9 Sep 2025 16:44:58 +0800 Subject: [PATCH 25/39] feat: add metalake integration test for spark and flink engine --- .../seatunnel/jdbc/MetalakeFlinkIT.java | 461 ++++++++++++++++++ .../seatunnel/jdbc/MetalakeSparkIT.java | 454 +++++++++++++++++ 2 files changed, 915 insertions(+) create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-7/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/MetalakeFlinkIT.java create mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-7/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/MetalakeSparkIT.java diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-7/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/MetalakeFlinkIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-7/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/MetalakeFlinkIT.java new file mode 100644 index 00000000000..18ab3410eec --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-7/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/MetalakeFlinkIT.java @@ -0,0 +1,461 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.jdbc; + +import org.apache.seatunnel.shade.com.google.common.collect.Lists; + +import org.apache.seatunnel.api.table.catalog.Catalog; +import org.apache.seatunnel.api.table.type.SeaTunnelRow; +import org.apache.seatunnel.common.utils.FileUtils; +import org.apache.seatunnel.e2e.common.container.flink.Flink13Container; + +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.tuple.Pair; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.testcontainers.DockerClientFactory; +import org.testcontainers.containers.BindMode; +import org.testcontainers.containers.Container; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.MySQLContainer; +import org.testcontainers.containers.output.Slf4jLogConsumer; +import org.testcontainers.containers.wait.strategy.LogMessageWaitStrategy; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.images.PullPolicy; +import org.testcontainers.lifecycle.Startables; +import org.testcontainers.utility.DockerImageName; +import org.testcontainers.utility.DockerLoggerFactory; + +import com.github.dockerjava.api.DockerClient; + +import java.io.IOException; +import java.math.BigDecimal; +import java.sql.Connection; +import java.sql.Driver; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.sql.Statement; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.awaitility.Awaitility.given; + +public class MetalakeFlinkIT extends Flink13Container { + protected GenericContainer dbServer; + + protected JdbcCase jdbcCase; + + protected Connection connection; + + protected Catalog catalog; + + protected DockerClient dockerClient = DockerClientFactory.lazyClient(); + + protected static final String HOST = "HOST"; + + private static final String MYSQL_IMAGE = "mysql:8.0"; + private static final String MYSQL_CONTAINER_HOST = "mysql-e2e"; + private static final String MYSQL_DATABASE = "seatunnel"; + private static final String MYSQL_SOURCE = "source"; + private static final String MYSQL_SINK = "sink"; + private static final String CATALOG_DATABASE = "catalog_database"; + + private static final String MYSQL_USERNAME = "root"; + private static final String MYSQL_PASSWORD = "Abc!@#135_seatunnel"; + private static final int MYSQL_PORT = 3306; + private static final String MYSQL_URL = "jdbc:mysql://" + HOST + ":%s/%s?useSSL=false"; + private static final String URL = "jdbc:mysql://" + HOST + ":3306/seatunnel"; + + private static final String SQL = "select * from seatunnel.source"; + + private static final String DRIVER_CLASS = "com.mysql.cj.jdbc.Driver"; + + private static final List CONFIG_FILE = + Lists.newArrayList("/mysql_to_mysql_with_metalake.conf"); + private static final String CREATE_SQL = + "CREATE TABLE IF NOT EXISTS %s\n" + + "(\n" + + " `c-bit_1` bit(1) DEFAULT NULL,\n" + + " `c_bit_8` bit(8) DEFAULT NULL,\n" + + " `c_bit_16` bit(16) DEFAULT NULL,\n" + + " `c_bit_32` bit(32) DEFAULT NULL,\n" + + " `c_bit_64` bit(64) DEFAULT NULL,\n" + + " `c_bigint_30` BIGINT(40) unsigned DEFAULT NULL,\n" + + " UNIQUE (c_bigint_30)\n" + + ");"; + + @BeforeEach + @Override + public void startUp() throws Exception { + FileUtils.createNewDir(HOST_VOLUME_MOUNT_PATH); + final String dockerImage = getDockerImage(); + final String properties = String.join("\n", getFlinkProperties()); + jobManager = + new GenericContainer<>(dockerImage) + .withCommand("jobmanager") + .withNetwork(NETWORK) + .withNetworkAliases("jobmanager") + .withExposedPorts() + .withEnv("FLINK_PROPERTIES", properties) + .withEnv("METALAKE_ENABLED", "true") + .withEnv("METALAKE_TYPE", "gravitino") + .withEnv( + "METALAKE_URL", + "http://127.0.0.1:8090/api/metalakes/test_metalake/catalogs/") + .withLogConsumer( + new Slf4jLogConsumer( + DockerLoggerFactory.getLogger(dockerImage + ":jobmanager"))) + .waitingFor( + new LogMessageWaitStrategy() + .withRegEx(".*Starting the resource manager.*") + .withStartupTimeout(Duration.ofMinutes(2))) + .withFileSystemBind( + HOST_VOLUME_MOUNT_PATH, + CONTAINER_VOLUME_MOUNT_PATH, + BindMode.READ_WRITE); + copySeaTunnelStarterToContainer(jobManager); + copySeaTunnelStarterLoggingToContainer(jobManager); + jobManager.setPortBindings(Lists.newArrayList(String.format("%s:%s", 8081, 8081))); + + taskManager = + new GenericContainer<>(dockerImage) + .withCommand("taskmanager") + .withNetwork(NETWORK) + .withNetworkAliases("taskmanager") + .withEnv("FLINK_PROPERTIES", properties) + .dependsOn(jobManager) + .withLogConsumer( + new Slf4jLogConsumer( + DockerLoggerFactory.getLogger( + dockerImage + ":taskmanager"))) + .waitingFor( + new LogMessageWaitStrategy() + .withRegEx( + ".*Successful registration at resource manager.*") + .withStartupTimeout(Duration.ofMinutes(2))) + .withFileSystemBind( + HOST_VOLUME_MOUNT_PATH, + CONTAINER_VOLUME_MOUNT_PATH, + BindMode.READ_WRITE); + + Startables.deepStart(Stream.of(jobManager)).join(); + Startables.deepStart(Stream.of(taskManager)).join(); + executeExtraCommands(jobManager); + + dbServer = initContainer().withImagePullPolicy(PullPolicy.alwaysPull()); + + Startables.deepStart(Stream.of(dbServer)).join(); + + jdbcCase = getJdbcCase(); + + given().ignoreExceptions() + .await() + .atMost(360, TimeUnit.SECONDS) + .untilAsserted(() -> this.initializeJdbcConnection(jdbcCase.getJdbcUrl())); + + createNeededTables(); + insertTestData(); + } + + @AfterEach + @Override + public void tearDown() throws Exception { + if (catalog != null) { + catalog.close(); + } + + if (connection != null) { + connection.close(); + } + + if (dbServer != null) { + dbServer.close(); + try { + dockerClient.removeImageCmd(dbServer.getDockerImageName()).exec(); + } catch (Exception ignored) { + ignored.printStackTrace(); + } + } + + super.tearDown(); + } + + @Test + public void TestMetalake() throws IOException, InterruptedException { + Container.ExecResult execResult = + executeJob("/jdbc_mysql_source_to_assert_sink_with_metalake.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + } + + String driverUrl() { + return "https://repo1.maven.org/maven2/com/mysql/mysql-connector-j/8.0.32/mysql-connector-j-8.0.32.jar"; + } + + protected GenericContainer initContainer() { + DockerImageName imageName = DockerImageName.parse(MYSQL_IMAGE); + + GenericContainer container = + new MySQLContainer<>(imageName) + .withUsername(MYSQL_USERNAME) + .withPassword(MYSQL_PASSWORD) + .withDatabaseName(MYSQL_DATABASE) + .withNetwork(NETWORK) + .withNetworkAliases(MYSQL_CONTAINER_HOST) + .withExposedPorts(MYSQL_PORT) + .waitingFor(Wait.forHealthcheck()) + .withLogConsumer( + new Slf4jLogConsumer(DockerLoggerFactory.getLogger(MYSQL_IMAGE))); + + container.setPortBindings( + Lists.newArrayList(String.format("%s:%s", MYSQL_PORT, MYSQL_PORT))); + + return container; + } + + JdbcCase getJdbcCase() { + Map containerEnv = new HashMap<>(); + String jdbcUrl = String.format(MYSQL_URL, MYSQL_PORT, MYSQL_DATABASE); + Pair> testDataSet = initTestData(); + String[] fieldNames = testDataSet.getKey(); + + String insertSql = insertTable(MYSQL_DATABASE, MYSQL_SOURCE, fieldNames); + + return JdbcCase.builder() + .dockerImage(MYSQL_IMAGE) + .networkAliases(MYSQL_CONTAINER_HOST) + .containerEnv(containerEnv) + .driverClass(DRIVER_CLASS) + .host(HOST) + .port(MYSQL_PORT) + .localPort(MYSQL_PORT) + .jdbcTemplate(MYSQL_URL) + .jdbcUrl(jdbcUrl) + .userName(MYSQL_USERNAME) + .password(MYSQL_PASSWORD) + .database(MYSQL_DATABASE) + .sourceTable(MYSQL_SOURCE) + .sinkTable(MYSQL_SINK) + .createSql(CREATE_SQL) + .configFile(CONFIG_FILE) + .insertSql(insertSql) + .testData(testDataSet) + .catalogDatabase(CATALOG_DATABASE) + .catalogTable(MYSQL_SINK) + .tablePathFullName(MYSQL_DATABASE + "." + MYSQL_SOURCE) + .build(); + } + + protected void initializeJdbcConnection(String jdbcUrl) + throws SQLException, InstantiationException, IllegalAccessException { + Driver driver = (Driver) loadDriverClass().newInstance(); + Properties props = new Properties(); + + if (StringUtils.isNotBlank(jdbcCase.getUserName())) { + props.put("user", jdbcCase.getUserName()); + } + + if (StringUtils.isNotBlank(jdbcCase.getPassword())) { + props.put("password", jdbcCase.getPassword()); + } + + if (dbServer != null) { + jdbcUrl = jdbcUrl.replace(HOST, dbServer.getHost()); + } + + this.connection = driver.connect(jdbcUrl, props); + connection.setAutoCommit(false); + } + + protected void createNeededTables() { + try (Statement statement = connection.createStatement()) { + String createTemplate = jdbcCase.getCreateSql(); + + String createSource = + String.format( + createTemplate, + buildTableInfoWithSchema( + jdbcCase.getDatabase(), + jdbcCase.getSchema(), + jdbcCase.getSourceTable())); + statement.execute(createSource); + + if (jdbcCase.getAdditionalSqlOnSource() != null) { + String additionalSql = + String.format( + jdbcCase.getAdditionalSqlOnSource(), + buildTableInfoWithSchema( + jdbcCase.getDatabase(), + jdbcCase.getSchema(), + jdbcCase.getSourceTable())); + statement.execute(additionalSql); + } + + if (!jdbcCase.isUseSaveModeCreateTable()) { + if (jdbcCase.getSinkCreateSql() != null) { + createTemplate = jdbcCase.getSinkCreateSql(); + } + String createSink = + String.format( + createTemplate, + buildTableInfoWithSchema( + jdbcCase.getDatabase(), + jdbcCase.getSchema(), + jdbcCase.getSinkTable())); + statement.execute(createSink); + } + + if (jdbcCase.getAdditionalSqlOnSink() != null) { + String additionalSql = + String.format( + jdbcCase.getAdditionalSqlOnSink(), + buildTableInfoWithSchema( + jdbcCase.getDatabase(), + jdbcCase.getSchema(), + jdbcCase.getSinkTable())); + statement.execute(additionalSql); + } + + connection.commit(); + } catch (Exception exception) { + exception.printStackTrace(); + } + } + + protected void insertTestData() { + try (PreparedStatement preparedStatement = + connection.prepareStatement(jdbcCase.getInsertSql())) { + + List rows = jdbcCase.getTestData().getValue(); + + for (SeaTunnelRow row : rows) { + for (int index = 0; index < row.getArity(); index++) { + preparedStatement.setObject(index + 1, row.getField(index)); + } + preparedStatement.addBatch(); + } + + preparedStatement.executeBatch(); + + connection.commit(); + } catch (Exception exception) { + exception.printStackTrace(); + } + } + + Pair> initTestData() { + String[] fieldNames = + new String[] { + "c-bit_1", "c_bit_8", "c_bit_16", "c_bit_32", "c_bit_64", "c_bigint_30", + }; + + List rows = new ArrayList<>(); + BigDecimal bigintValue = new BigDecimal("2844674407371055000"); + BigDecimal decimalValue = new BigDecimal("999999999999999999999999999899"); + for (int i = 0; i < 100; i++) { + byte byteArr = Integer.valueOf(i).byteValue(); + SeaTunnelRow row; + if (i == 99) { + row = + new SeaTunnelRow( + new Object[] { + (byte) 0, + new byte[] {byteArr}, + new byte[] {byteArr, byteArr}, + new byte[] {byteArr, byteArr, byteArr, byteArr}, + new byte[] { + byteArr, byteArr, byteArr, byteArr, byteArr, byteArr, + byteArr, byteArr + }, + // https://github.com/apache/seatunnel/issues/5559 this value + // cannot set null, this null + // value column's row will be lost in + // jdbc_mysql_source_and_sink_parallel.conf,jdbc_mysql_source_and_sink_parallel_upper_lower.conf. + bigintValue.add(BigDecimal.valueOf(i)), + }); + } else { + row = + new SeaTunnelRow( + new Object[] { + i % 2 == 0 ? (byte) 1 : (byte) 0, + new byte[] {byteArr}, + new byte[] {byteArr, byteArr}, + new byte[] {byteArr, byteArr, byteArr, byteArr}, + new byte[] { + byteArr, byteArr, byteArr, byteArr, byteArr, byteArr, + byteArr, byteArr + }, + bigintValue.add(BigDecimal.valueOf(i)), + }); + } + rows.add(row); + } + + return Pair.of(fieldNames, rows); + } + + public String insertTable(String schema, String table, String... fields) { + String columns = + Arrays.stream(fields).map(this::quoteIdentifier).collect(Collectors.joining(", ")); + String placeholders = Arrays.stream(fields).map(f -> "?").collect(Collectors.joining(", ")); + + return "INSERT INTO " + + buildTableInfoWithSchema(schema, table) + + " (" + + columns + + " )" + + " VALUES (" + + placeholders + + ")"; + } + + protected Class loadDriverClass() { + try { + return Class.forName(jdbcCase.getDriverClass()); + } catch (Exception e) { + throw new RuntimeException( + "Failed to load driver class: " + jdbcCase.getDriverClass(), e); + } + } + + protected String buildTableInfoWithSchema(String database, String schema, String table) { + return buildTableInfoWithSchema(database, table); + } + + public String buildTableInfoWithSchema(String schema, String table) { + if (StringUtils.isNotBlank(schema)) { + return quoteIdentifier(schema) + "." + quoteIdentifier(table); + } else { + return quoteIdentifier(table); + } + } + + public String quoteIdentifier(String field) { + return "`" + field + "`"; + } +} diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-7/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/MetalakeSparkIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-7/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/MetalakeSparkIT.java new file mode 100644 index 00000000000..4e928627f8f --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-7/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/MetalakeSparkIT.java @@ -0,0 +1,454 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.jdbc; + +import org.apache.seatunnel.shade.com.google.common.collect.Lists; + +import org.apache.seatunnel.api.table.catalog.Catalog; +import org.apache.seatunnel.api.table.type.SeaTunnelRow; +import org.apache.seatunnel.common.utils.FileUtils; +import org.apache.seatunnel.e2e.common.container.spark.Spark2Container; + +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.tuple.Pair; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.testcontainers.DockerClientFactory; +import org.testcontainers.containers.BindMode; +import org.testcontainers.containers.Container; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.MySQLContainer; +import org.testcontainers.containers.output.Slf4jLogConsumer; +import org.testcontainers.containers.wait.strategy.LogMessageWaitStrategy; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.images.PullPolicy; +import org.testcontainers.lifecycle.Startables; +import org.testcontainers.utility.DockerImageName; +import org.testcontainers.utility.DockerLoggerFactory; + +import com.github.dockerjava.api.DockerClient; + +import java.io.IOException; +import java.math.BigDecimal; +import java.sql.Connection; +import java.sql.Driver; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.sql.Statement; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.awaitility.Awaitility.given; + +public class MetalakeSparkIT extends Spark2Container { + protected GenericContainer dbServer; + + protected JdbcCase jdbcCase; + + protected Connection connection; + + protected Catalog catalog; + + protected DockerClient dockerClient = DockerClientFactory.lazyClient(); + + protected static final String HOST = "HOST"; + + private static final String MYSQL_IMAGE = "mysql:8.0"; + private static final String MYSQL_CONTAINER_HOST = "mysql-e2e"; + private static final String MYSQL_DATABASE = "seatunnel"; + private static final String MYSQL_SOURCE = "source"; + private static final String MYSQL_SINK = "sink"; + private static final String CATALOG_DATABASE = "catalog_database"; + + private static final String MYSQL_USERNAME = "root"; + private static final String MYSQL_PASSWORD = "Abc!@#135_seatunnel"; + private static final int MYSQL_PORT = 3306; + private static final String MYSQL_URL = "jdbc:mysql://" + HOST + ":%s/%s?useSSL=false"; + private static final String URL = "jdbc:mysql://" + HOST + ":3306/seatunnel"; + + private static final String SQL = "select * from seatunnel.source"; + + private static final String DRIVER_CLASS = "com.mysql.cj.jdbc.Driver"; + + private static final List CONFIG_FILE = + Lists.newArrayList("/mysql_to_mysql_with_metalake.conf"); + private static final String CREATE_SQL = + "CREATE TABLE IF NOT EXISTS %s\n" + + "(\n" + + " `c-bit_1` bit(1) DEFAULT NULL,\n" + + " `c_bit_8` bit(8) DEFAULT NULL,\n" + + " `c_bit_16` bit(16) DEFAULT NULL,\n" + + " `c_bit_32` bit(32) DEFAULT NULL,\n" + + " `c_bit_64` bit(64) DEFAULT NULL,\n" + + " `c_bigint_30` BIGINT(40) unsigned DEFAULT NULL,\n" + + " UNIQUE (c_bigint_30)\n" + + ");"; + + @BeforeEach + @Override + public void startUp() throws Exception { + FileUtils.createNewDir(HOST_VOLUME_MOUNT_PATH); + master = + new GenericContainer<>(getDockerImage()) + .withNetwork(NETWORK) + .withNetworkAliases("spark-master") + .withExposedPorts() + .withEnv("SPARK_MODE", "master") + .withEnv("METALAKE_ENABLED", "true") + .withEnv("METALAKE_TYPE", "gravitino") + .withEnv( + "METALAKE_URL", + "http://127.0.0.1:8090/api/metalakes/test_metalake/catalogs/") + .withLogConsumer( + new Slf4jLogConsumer( + DockerLoggerFactory.getLogger(getDockerImage()))) + .withCreateContainerCmdModifier(cmd -> cmd.withUser("root")) + .withFileSystemBind( + HOST_VOLUME_MOUNT_PATH, + CONTAINER_VOLUME_MOUNT_PATH, + BindMode.READ_WRITE) + .waitingFor( + new LogMessageWaitStrategy() + .withRegEx(".*Master: Starting Spark master at.*") + .withStartupTimeout(Duration.ofMinutes(2))); + copySeaTunnelStarterToContainer(master); + copySeaTunnelStarterLoggingToContainer(master); + + // In most case we can just use standalone mode to execute a spark job, if we want to use + // cluster mode, we need to + // start a worker. + Startables.deepStart(Stream.of(master)).join(); + // execute extra commands + executeExtraCommands(master); + + master.execInContainer( + "bash", + "-c", + "mkdir -p /tmp/seatunnel/plugins/Jdbc/lib && cd /tmp/seatunnel/plugins/Jdbc/lib && wget " + + driverUrl() + + " --no-check-certificate" + + "&& mkdir -p /tmp/gravitino && cd /tmp/gravitino && curl -C - --retry 5 -L -k -o gravitino-0.9.1-bin.tar.gz https://dlcdn.apache.org/gravitino/0.9.1/gravitino-0.9.1-bin.tar.gz && tar -zxvf gravitino-0.9.1-bin.tar.gz && cd /tmp/gravitino/gravitino-0.9.1-bin && ./bin/gravitino.sh start"); + + master.execInContainer( + "bash", + "-c", + "sleep 60 && curl -L 'http://127.0.0.1:8090/api/metalakes' -H 'Content-Type: application/json' -H 'Accept: application/vnd.gravitino.v1+json' -d '{\"name\":\"test_metalake\",\"comment\":\"for metalake test\",\"properties\":{}}'" + + "&& curl -L 'http://127.0.0.1:8090/api/metalakes/test_metalake/catalogs' -H 'Content-Type: application/json' -H 'Accept: application/vnd.gravitino.v1+json' -d '{\"name\":\"test_catalog\",\"type\":\"relational\",\"provider\":\"jdbc-mysql\",\"comment\":\"for metalake test\",\"properties\":{\"jdbc-driver\":\"com.mysql.cj.jdbc.Driver\",\"jdbc-url\":\"not used\",\"jdbc-user\":\"root\",\"jdbc-password\":\"Abc!@#135_seatunnel\"}}'"); + + dbServer = initContainer().withImagePullPolicy(PullPolicy.alwaysPull()); + + Startables.deepStart(Stream.of(dbServer)).join(); + + jdbcCase = getJdbcCase(); + + given().ignoreExceptions() + .await() + .atMost(360, TimeUnit.SECONDS) + .untilAsserted(() -> this.initializeJdbcConnection(jdbcCase.getJdbcUrl())); + + createNeededTables(); + insertTestData(); + } + + @AfterEach + @Override + public void tearDown() throws Exception { + if (catalog != null) { + catalog.close(); + } + + if (connection != null) { + connection.close(); + } + + if (dbServer != null) { + dbServer.close(); + try { + dockerClient.removeImageCmd(dbServer.getDockerImageName()).exec(); + } catch (Exception ignored) { + ignored.printStackTrace(); + } + } + + super.tearDown(); + } + + @Test + public void TestMetalake() throws IOException, InterruptedException { + Container.ExecResult execResult = + executeJob("/jdbc_mysql_source_to_assert_sink_with_metalake.conf"); + Assertions.assertEquals(0, execResult.getExitCode()); + } + + String driverUrl() { + return "https://repo1.maven.org/maven2/com/mysql/mysql-connector-j/8.0.32/mysql-connector-j-8.0.32.jar"; + } + + protected GenericContainer initContainer() { + DockerImageName imageName = DockerImageName.parse(MYSQL_IMAGE); + + GenericContainer container = + new MySQLContainer<>(imageName) + .withUsername(MYSQL_USERNAME) + .withPassword(MYSQL_PASSWORD) + .withDatabaseName(MYSQL_DATABASE) + .withNetwork(NETWORK) + .withNetworkAliases(MYSQL_CONTAINER_HOST) + .withExposedPorts(MYSQL_PORT) + .waitingFor(Wait.forHealthcheck()) + .withLogConsumer( + new Slf4jLogConsumer(DockerLoggerFactory.getLogger(MYSQL_IMAGE))); + + container.setPortBindings( + Lists.newArrayList(String.format("%s:%s", MYSQL_PORT, MYSQL_PORT))); + + return container; + } + + JdbcCase getJdbcCase() { + Map containerEnv = new HashMap<>(); + String jdbcUrl = String.format(MYSQL_URL, MYSQL_PORT, MYSQL_DATABASE); + Pair> testDataSet = initTestData(); + String[] fieldNames = testDataSet.getKey(); + + String insertSql = insertTable(MYSQL_DATABASE, MYSQL_SOURCE, fieldNames); + + return JdbcCase.builder() + .dockerImage(MYSQL_IMAGE) + .networkAliases(MYSQL_CONTAINER_HOST) + .containerEnv(containerEnv) + .driverClass(DRIVER_CLASS) + .host(HOST) + .port(MYSQL_PORT) + .localPort(MYSQL_PORT) + .jdbcTemplate(MYSQL_URL) + .jdbcUrl(jdbcUrl) + .userName(MYSQL_USERNAME) + .password(MYSQL_PASSWORD) + .database(MYSQL_DATABASE) + .sourceTable(MYSQL_SOURCE) + .sinkTable(MYSQL_SINK) + .createSql(CREATE_SQL) + .configFile(CONFIG_FILE) + .insertSql(insertSql) + .testData(testDataSet) + .catalogDatabase(CATALOG_DATABASE) + .catalogTable(MYSQL_SINK) + .tablePathFullName(MYSQL_DATABASE + "." + MYSQL_SOURCE) + .build(); + } + + protected void initializeJdbcConnection(String jdbcUrl) + throws SQLException, InstantiationException, IllegalAccessException { + Driver driver = (Driver) loadDriverClass().newInstance(); + Properties props = new Properties(); + + if (StringUtils.isNotBlank(jdbcCase.getUserName())) { + props.put("user", jdbcCase.getUserName()); + } + + if (StringUtils.isNotBlank(jdbcCase.getPassword())) { + props.put("password", jdbcCase.getPassword()); + } + + if (dbServer != null) { + jdbcUrl = jdbcUrl.replace(HOST, dbServer.getHost()); + } + + this.connection = driver.connect(jdbcUrl, props); + connection.setAutoCommit(false); + } + + protected void createNeededTables() { + try (Statement statement = connection.createStatement()) { + String createTemplate = jdbcCase.getCreateSql(); + + String createSource = + String.format( + createTemplate, + buildTableInfoWithSchema( + jdbcCase.getDatabase(), + jdbcCase.getSchema(), + jdbcCase.getSourceTable())); + statement.execute(createSource); + + if (jdbcCase.getAdditionalSqlOnSource() != null) { + String additionalSql = + String.format( + jdbcCase.getAdditionalSqlOnSource(), + buildTableInfoWithSchema( + jdbcCase.getDatabase(), + jdbcCase.getSchema(), + jdbcCase.getSourceTable())); + statement.execute(additionalSql); + } + + if (!jdbcCase.isUseSaveModeCreateTable()) { + if (jdbcCase.getSinkCreateSql() != null) { + createTemplate = jdbcCase.getSinkCreateSql(); + } + String createSink = + String.format( + createTemplate, + buildTableInfoWithSchema( + jdbcCase.getDatabase(), + jdbcCase.getSchema(), + jdbcCase.getSinkTable())); + statement.execute(createSink); + } + + if (jdbcCase.getAdditionalSqlOnSink() != null) { + String additionalSql = + String.format( + jdbcCase.getAdditionalSqlOnSink(), + buildTableInfoWithSchema( + jdbcCase.getDatabase(), + jdbcCase.getSchema(), + jdbcCase.getSinkTable())); + statement.execute(additionalSql); + } + + connection.commit(); + } catch (Exception exception) { + exception.printStackTrace(); + } + } + + protected void insertTestData() { + try (PreparedStatement preparedStatement = + connection.prepareStatement(jdbcCase.getInsertSql())) { + + List rows = jdbcCase.getTestData().getValue(); + + for (SeaTunnelRow row : rows) { + for (int index = 0; index < row.getArity(); index++) { + preparedStatement.setObject(index + 1, row.getField(index)); + } + preparedStatement.addBatch(); + } + + preparedStatement.executeBatch(); + + connection.commit(); + } catch (Exception exception) { + exception.printStackTrace(); + } + } + + Pair> initTestData() { + String[] fieldNames = + new String[] { + "c-bit_1", "c_bit_8", "c_bit_16", "c_bit_32", "c_bit_64", "c_bigint_30", + }; + + List rows = new ArrayList<>(); + BigDecimal bigintValue = new BigDecimal("2844674407371055000"); + BigDecimal decimalValue = new BigDecimal("999999999999999999999999999899"); + for (int i = 0; i < 100; i++) { + byte byteArr = Integer.valueOf(i).byteValue(); + SeaTunnelRow row; + if (i == 99) { + row = + new SeaTunnelRow( + new Object[] { + (byte) 0, + new byte[] {byteArr}, + new byte[] {byteArr, byteArr}, + new byte[] {byteArr, byteArr, byteArr, byteArr}, + new byte[] { + byteArr, byteArr, byteArr, byteArr, byteArr, byteArr, + byteArr, byteArr + }, + // https://github.com/apache/seatunnel/issues/5559 this value + // cannot set null, this null + // value column's row will be lost in + // jdbc_mysql_source_and_sink_parallel.conf,jdbc_mysql_source_and_sink_parallel_upper_lower.conf. + bigintValue.add(BigDecimal.valueOf(i)), + }); + } else { + row = + new SeaTunnelRow( + new Object[] { + i % 2 == 0 ? (byte) 1 : (byte) 0, + new byte[] {byteArr}, + new byte[] {byteArr, byteArr}, + new byte[] {byteArr, byteArr, byteArr, byteArr}, + new byte[] { + byteArr, byteArr, byteArr, byteArr, byteArr, byteArr, + byteArr, byteArr + }, + bigintValue.add(BigDecimal.valueOf(i)), + }); + } + rows.add(row); + } + + return Pair.of(fieldNames, rows); + } + + public String insertTable(String schema, String table, String... fields) { + String columns = + Arrays.stream(fields).map(this::quoteIdentifier).collect(Collectors.joining(", ")); + String placeholders = Arrays.stream(fields).map(f -> "?").collect(Collectors.joining(", ")); + + return "INSERT INTO " + + buildTableInfoWithSchema(schema, table) + + " (" + + columns + + " )" + + " VALUES (" + + placeholders + + ")"; + } + + protected Class loadDriverClass() { + try { + return Class.forName(jdbcCase.getDriverClass()); + } catch (Exception e) { + throw new RuntimeException( + "Failed to load driver class: " + jdbcCase.getDriverClass(), e); + } + } + + protected String buildTableInfoWithSchema(String database, String schema, String table) { + return buildTableInfoWithSchema(database, table); + } + + public String buildTableInfoWithSchema(String schema, String table) { + if (StringUtils.isNotBlank(schema)) { + return quoteIdentifier(schema) + "." + quoteIdentifier(table); + } else { + return quoteIdentifier(table); + } + } + + public String quoteIdentifier(String field) { + return "`" + field + "`"; + } +} From 73a4d0053824374714ee7c4edbd0261e9cca4805 Mon Sep 17 00:00:00 2001 From: wtybxqm <1599646055@qq.com> Date: Tue, 9 Sep 2025 18:39:54 +0800 Subject: [PATCH 26/39] fix: download gravitino in flink container --- .../connectors/seatunnel/jdbc/MetalakeFlinkIT.java | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-7/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/MetalakeFlinkIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-7/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/MetalakeFlinkIT.java index 18ab3410eec..d8ff8cade11 100644 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-7/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/MetalakeFlinkIT.java +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-7/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/MetalakeFlinkIT.java @@ -168,6 +168,20 @@ public void startUp() throws Exception { Startables.deepStart(Stream.of(taskManager)).join(); executeExtraCommands(jobManager); + jobManager.execInContainer( + "bash", + "-c", + "mkdir -p /tmp/seatunnel/plugins/Jdbc/lib && cd /tmp/seatunnel/plugins/Jdbc/lib && wget " + + driverUrl() + + " --no-check-certificate" + + "&& mkdir -p /tmp/gravitino && cd /tmp/gravitino && curl -C - --retry 5 -L -k -o gravitino-0.9.1-bin.tar.gz https://dlcdn.apache.org/gravitino/0.9.1/gravitino-0.9.1-bin.tar.gz && tar -zxvf gravitino-0.9.1-bin.tar.gz && cd /tmp/gravitino/gravitino-0.9.1-bin && ./bin/gravitino.sh start"); + + jobManager.execInContainer( + "bash", + "-c", + "sleep 60 && curl -L 'http://127.0.0.1:8090/api/metalakes' -H 'Content-Type: application/json' -H 'Accept: application/vnd.gravitino.v1+json' -d '{\"name\":\"test_metalake\",\"comment\":\"for metalake test\",\"properties\":{}}'" + + "&& curl -L 'http://127.0.0.1:8090/api/metalakes/test_metalake/catalogs' -H 'Content-Type: application/json' -H 'Accept: application/vnd.gravitino.v1+json' -d '{\"name\":\"test_catalog\",\"type\":\"relational\",\"provider\":\"jdbc-mysql\",\"comment\":\"for metalake test\",\"properties\":{\"jdbc-driver\":\"com.mysql.cj.jdbc.Driver\",\"jdbc-url\":\"not used\",\"jdbc-user\":\"root\",\"jdbc-password\":\"Abc!@#135_seatunnel\"}}'"); + dbServer = initContainer().withImagePullPolicy(PullPolicy.alwaysPull()); Startables.deepStart(Stream.of(dbServer)).join(); From b528a98e8ca038d86a35aeb08e8d843020b047a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=90=B4=E5=A4=A9=E5=AE=87?= <1599646055@qq.com> Date: Wed, 10 Sep 2025 20:11:48 +0800 Subject: [PATCH 27/39] fix: move the docs to concept directory; remove extra test cases for spark and flink --- .../{seatunnel-engine => concept}/metalake.md | 0 .../{seatunnel-engine => concept}/metalake.md | 0 .../seatunnel/jdbc/MetalakeFlinkIT.java | 475 ------------------ .../seatunnel/jdbc/MetalakeSparkIT.java | 454 ----------------- 4 files changed, 929 deletions(-) rename docs/en/{seatunnel-engine => concept}/metalake.md (100%) rename docs/zh/{seatunnel-engine => concept}/metalake.md (100%) delete mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-7/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/MetalakeFlinkIT.java delete mode 100644 seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-7/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/MetalakeSparkIT.java diff --git a/docs/en/seatunnel-engine/metalake.md b/docs/en/concept/metalake.md similarity index 100% rename from docs/en/seatunnel-engine/metalake.md rename to docs/en/concept/metalake.md diff --git a/docs/zh/seatunnel-engine/metalake.md b/docs/zh/concept/metalake.md similarity index 100% rename from docs/zh/seatunnel-engine/metalake.md rename to docs/zh/concept/metalake.md diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-7/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/MetalakeFlinkIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-7/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/MetalakeFlinkIT.java deleted file mode 100644 index d8ff8cade11..00000000000 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-7/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/MetalakeFlinkIT.java +++ /dev/null @@ -1,475 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.jdbc; - -import org.apache.seatunnel.shade.com.google.common.collect.Lists; - -import org.apache.seatunnel.api.table.catalog.Catalog; -import org.apache.seatunnel.api.table.type.SeaTunnelRow; -import org.apache.seatunnel.common.utils.FileUtils; -import org.apache.seatunnel.e2e.common.container.flink.Flink13Container; - -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.tuple.Pair; - -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.testcontainers.DockerClientFactory; -import org.testcontainers.containers.BindMode; -import org.testcontainers.containers.Container; -import org.testcontainers.containers.GenericContainer; -import org.testcontainers.containers.MySQLContainer; -import org.testcontainers.containers.output.Slf4jLogConsumer; -import org.testcontainers.containers.wait.strategy.LogMessageWaitStrategy; -import org.testcontainers.containers.wait.strategy.Wait; -import org.testcontainers.images.PullPolicy; -import org.testcontainers.lifecycle.Startables; -import org.testcontainers.utility.DockerImageName; -import org.testcontainers.utility.DockerLoggerFactory; - -import com.github.dockerjava.api.DockerClient; - -import java.io.IOException; -import java.math.BigDecimal; -import java.sql.Connection; -import java.sql.Driver; -import java.sql.PreparedStatement; -import java.sql.SQLException; -import java.sql.Statement; -import java.time.Duration; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Properties; -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import static org.awaitility.Awaitility.given; - -public class MetalakeFlinkIT extends Flink13Container { - protected GenericContainer dbServer; - - protected JdbcCase jdbcCase; - - protected Connection connection; - - protected Catalog catalog; - - protected DockerClient dockerClient = DockerClientFactory.lazyClient(); - - protected static final String HOST = "HOST"; - - private static final String MYSQL_IMAGE = "mysql:8.0"; - private static final String MYSQL_CONTAINER_HOST = "mysql-e2e"; - private static final String MYSQL_DATABASE = "seatunnel"; - private static final String MYSQL_SOURCE = "source"; - private static final String MYSQL_SINK = "sink"; - private static final String CATALOG_DATABASE = "catalog_database"; - - private static final String MYSQL_USERNAME = "root"; - private static final String MYSQL_PASSWORD = "Abc!@#135_seatunnel"; - private static final int MYSQL_PORT = 3306; - private static final String MYSQL_URL = "jdbc:mysql://" + HOST + ":%s/%s?useSSL=false"; - private static final String URL = "jdbc:mysql://" + HOST + ":3306/seatunnel"; - - private static final String SQL = "select * from seatunnel.source"; - - private static final String DRIVER_CLASS = "com.mysql.cj.jdbc.Driver"; - - private static final List CONFIG_FILE = - Lists.newArrayList("/mysql_to_mysql_with_metalake.conf"); - private static final String CREATE_SQL = - "CREATE TABLE IF NOT EXISTS %s\n" - + "(\n" - + " `c-bit_1` bit(1) DEFAULT NULL,\n" - + " `c_bit_8` bit(8) DEFAULT NULL,\n" - + " `c_bit_16` bit(16) DEFAULT NULL,\n" - + " `c_bit_32` bit(32) DEFAULT NULL,\n" - + " `c_bit_64` bit(64) DEFAULT NULL,\n" - + " `c_bigint_30` BIGINT(40) unsigned DEFAULT NULL,\n" - + " UNIQUE (c_bigint_30)\n" - + ");"; - - @BeforeEach - @Override - public void startUp() throws Exception { - FileUtils.createNewDir(HOST_VOLUME_MOUNT_PATH); - final String dockerImage = getDockerImage(); - final String properties = String.join("\n", getFlinkProperties()); - jobManager = - new GenericContainer<>(dockerImage) - .withCommand("jobmanager") - .withNetwork(NETWORK) - .withNetworkAliases("jobmanager") - .withExposedPorts() - .withEnv("FLINK_PROPERTIES", properties) - .withEnv("METALAKE_ENABLED", "true") - .withEnv("METALAKE_TYPE", "gravitino") - .withEnv( - "METALAKE_URL", - "http://127.0.0.1:8090/api/metalakes/test_metalake/catalogs/") - .withLogConsumer( - new Slf4jLogConsumer( - DockerLoggerFactory.getLogger(dockerImage + ":jobmanager"))) - .waitingFor( - new LogMessageWaitStrategy() - .withRegEx(".*Starting the resource manager.*") - .withStartupTimeout(Duration.ofMinutes(2))) - .withFileSystemBind( - HOST_VOLUME_MOUNT_PATH, - CONTAINER_VOLUME_MOUNT_PATH, - BindMode.READ_WRITE); - copySeaTunnelStarterToContainer(jobManager); - copySeaTunnelStarterLoggingToContainer(jobManager); - jobManager.setPortBindings(Lists.newArrayList(String.format("%s:%s", 8081, 8081))); - - taskManager = - new GenericContainer<>(dockerImage) - .withCommand("taskmanager") - .withNetwork(NETWORK) - .withNetworkAliases("taskmanager") - .withEnv("FLINK_PROPERTIES", properties) - .dependsOn(jobManager) - .withLogConsumer( - new Slf4jLogConsumer( - DockerLoggerFactory.getLogger( - dockerImage + ":taskmanager"))) - .waitingFor( - new LogMessageWaitStrategy() - .withRegEx( - ".*Successful registration at resource manager.*") - .withStartupTimeout(Duration.ofMinutes(2))) - .withFileSystemBind( - HOST_VOLUME_MOUNT_PATH, - CONTAINER_VOLUME_MOUNT_PATH, - BindMode.READ_WRITE); - - Startables.deepStart(Stream.of(jobManager)).join(); - Startables.deepStart(Stream.of(taskManager)).join(); - executeExtraCommands(jobManager); - - jobManager.execInContainer( - "bash", - "-c", - "mkdir -p /tmp/seatunnel/plugins/Jdbc/lib && cd /tmp/seatunnel/plugins/Jdbc/lib && wget " - + driverUrl() - + " --no-check-certificate" - + "&& mkdir -p /tmp/gravitino && cd /tmp/gravitino && curl -C - --retry 5 -L -k -o gravitino-0.9.1-bin.tar.gz https://dlcdn.apache.org/gravitino/0.9.1/gravitino-0.9.1-bin.tar.gz && tar -zxvf gravitino-0.9.1-bin.tar.gz && cd /tmp/gravitino/gravitino-0.9.1-bin && ./bin/gravitino.sh start"); - - jobManager.execInContainer( - "bash", - "-c", - "sleep 60 && curl -L 'http://127.0.0.1:8090/api/metalakes' -H 'Content-Type: application/json' -H 'Accept: application/vnd.gravitino.v1+json' -d '{\"name\":\"test_metalake\",\"comment\":\"for metalake test\",\"properties\":{}}'" - + "&& curl -L 'http://127.0.0.1:8090/api/metalakes/test_metalake/catalogs' -H 'Content-Type: application/json' -H 'Accept: application/vnd.gravitino.v1+json' -d '{\"name\":\"test_catalog\",\"type\":\"relational\",\"provider\":\"jdbc-mysql\",\"comment\":\"for metalake test\",\"properties\":{\"jdbc-driver\":\"com.mysql.cj.jdbc.Driver\",\"jdbc-url\":\"not used\",\"jdbc-user\":\"root\",\"jdbc-password\":\"Abc!@#135_seatunnel\"}}'"); - - dbServer = initContainer().withImagePullPolicy(PullPolicy.alwaysPull()); - - Startables.deepStart(Stream.of(dbServer)).join(); - - jdbcCase = getJdbcCase(); - - given().ignoreExceptions() - .await() - .atMost(360, TimeUnit.SECONDS) - .untilAsserted(() -> this.initializeJdbcConnection(jdbcCase.getJdbcUrl())); - - createNeededTables(); - insertTestData(); - } - - @AfterEach - @Override - public void tearDown() throws Exception { - if (catalog != null) { - catalog.close(); - } - - if (connection != null) { - connection.close(); - } - - if (dbServer != null) { - dbServer.close(); - try { - dockerClient.removeImageCmd(dbServer.getDockerImageName()).exec(); - } catch (Exception ignored) { - ignored.printStackTrace(); - } - } - - super.tearDown(); - } - - @Test - public void TestMetalake() throws IOException, InterruptedException { - Container.ExecResult execResult = - executeJob("/jdbc_mysql_source_to_assert_sink_with_metalake.conf"); - Assertions.assertEquals(0, execResult.getExitCode()); - } - - String driverUrl() { - return "https://repo1.maven.org/maven2/com/mysql/mysql-connector-j/8.0.32/mysql-connector-j-8.0.32.jar"; - } - - protected GenericContainer initContainer() { - DockerImageName imageName = DockerImageName.parse(MYSQL_IMAGE); - - GenericContainer container = - new MySQLContainer<>(imageName) - .withUsername(MYSQL_USERNAME) - .withPassword(MYSQL_PASSWORD) - .withDatabaseName(MYSQL_DATABASE) - .withNetwork(NETWORK) - .withNetworkAliases(MYSQL_CONTAINER_HOST) - .withExposedPorts(MYSQL_PORT) - .waitingFor(Wait.forHealthcheck()) - .withLogConsumer( - new Slf4jLogConsumer(DockerLoggerFactory.getLogger(MYSQL_IMAGE))); - - container.setPortBindings( - Lists.newArrayList(String.format("%s:%s", MYSQL_PORT, MYSQL_PORT))); - - return container; - } - - JdbcCase getJdbcCase() { - Map containerEnv = new HashMap<>(); - String jdbcUrl = String.format(MYSQL_URL, MYSQL_PORT, MYSQL_DATABASE); - Pair> testDataSet = initTestData(); - String[] fieldNames = testDataSet.getKey(); - - String insertSql = insertTable(MYSQL_DATABASE, MYSQL_SOURCE, fieldNames); - - return JdbcCase.builder() - .dockerImage(MYSQL_IMAGE) - .networkAliases(MYSQL_CONTAINER_HOST) - .containerEnv(containerEnv) - .driverClass(DRIVER_CLASS) - .host(HOST) - .port(MYSQL_PORT) - .localPort(MYSQL_PORT) - .jdbcTemplate(MYSQL_URL) - .jdbcUrl(jdbcUrl) - .userName(MYSQL_USERNAME) - .password(MYSQL_PASSWORD) - .database(MYSQL_DATABASE) - .sourceTable(MYSQL_SOURCE) - .sinkTable(MYSQL_SINK) - .createSql(CREATE_SQL) - .configFile(CONFIG_FILE) - .insertSql(insertSql) - .testData(testDataSet) - .catalogDatabase(CATALOG_DATABASE) - .catalogTable(MYSQL_SINK) - .tablePathFullName(MYSQL_DATABASE + "." + MYSQL_SOURCE) - .build(); - } - - protected void initializeJdbcConnection(String jdbcUrl) - throws SQLException, InstantiationException, IllegalAccessException { - Driver driver = (Driver) loadDriverClass().newInstance(); - Properties props = new Properties(); - - if (StringUtils.isNotBlank(jdbcCase.getUserName())) { - props.put("user", jdbcCase.getUserName()); - } - - if (StringUtils.isNotBlank(jdbcCase.getPassword())) { - props.put("password", jdbcCase.getPassword()); - } - - if (dbServer != null) { - jdbcUrl = jdbcUrl.replace(HOST, dbServer.getHost()); - } - - this.connection = driver.connect(jdbcUrl, props); - connection.setAutoCommit(false); - } - - protected void createNeededTables() { - try (Statement statement = connection.createStatement()) { - String createTemplate = jdbcCase.getCreateSql(); - - String createSource = - String.format( - createTemplate, - buildTableInfoWithSchema( - jdbcCase.getDatabase(), - jdbcCase.getSchema(), - jdbcCase.getSourceTable())); - statement.execute(createSource); - - if (jdbcCase.getAdditionalSqlOnSource() != null) { - String additionalSql = - String.format( - jdbcCase.getAdditionalSqlOnSource(), - buildTableInfoWithSchema( - jdbcCase.getDatabase(), - jdbcCase.getSchema(), - jdbcCase.getSourceTable())); - statement.execute(additionalSql); - } - - if (!jdbcCase.isUseSaveModeCreateTable()) { - if (jdbcCase.getSinkCreateSql() != null) { - createTemplate = jdbcCase.getSinkCreateSql(); - } - String createSink = - String.format( - createTemplate, - buildTableInfoWithSchema( - jdbcCase.getDatabase(), - jdbcCase.getSchema(), - jdbcCase.getSinkTable())); - statement.execute(createSink); - } - - if (jdbcCase.getAdditionalSqlOnSink() != null) { - String additionalSql = - String.format( - jdbcCase.getAdditionalSqlOnSink(), - buildTableInfoWithSchema( - jdbcCase.getDatabase(), - jdbcCase.getSchema(), - jdbcCase.getSinkTable())); - statement.execute(additionalSql); - } - - connection.commit(); - } catch (Exception exception) { - exception.printStackTrace(); - } - } - - protected void insertTestData() { - try (PreparedStatement preparedStatement = - connection.prepareStatement(jdbcCase.getInsertSql())) { - - List rows = jdbcCase.getTestData().getValue(); - - for (SeaTunnelRow row : rows) { - for (int index = 0; index < row.getArity(); index++) { - preparedStatement.setObject(index + 1, row.getField(index)); - } - preparedStatement.addBatch(); - } - - preparedStatement.executeBatch(); - - connection.commit(); - } catch (Exception exception) { - exception.printStackTrace(); - } - } - - Pair> initTestData() { - String[] fieldNames = - new String[] { - "c-bit_1", "c_bit_8", "c_bit_16", "c_bit_32", "c_bit_64", "c_bigint_30", - }; - - List rows = new ArrayList<>(); - BigDecimal bigintValue = new BigDecimal("2844674407371055000"); - BigDecimal decimalValue = new BigDecimal("999999999999999999999999999899"); - for (int i = 0; i < 100; i++) { - byte byteArr = Integer.valueOf(i).byteValue(); - SeaTunnelRow row; - if (i == 99) { - row = - new SeaTunnelRow( - new Object[] { - (byte) 0, - new byte[] {byteArr}, - new byte[] {byteArr, byteArr}, - new byte[] {byteArr, byteArr, byteArr, byteArr}, - new byte[] { - byteArr, byteArr, byteArr, byteArr, byteArr, byteArr, - byteArr, byteArr - }, - // https://github.com/apache/seatunnel/issues/5559 this value - // cannot set null, this null - // value column's row will be lost in - // jdbc_mysql_source_and_sink_parallel.conf,jdbc_mysql_source_and_sink_parallel_upper_lower.conf. - bigintValue.add(BigDecimal.valueOf(i)), - }); - } else { - row = - new SeaTunnelRow( - new Object[] { - i % 2 == 0 ? (byte) 1 : (byte) 0, - new byte[] {byteArr}, - new byte[] {byteArr, byteArr}, - new byte[] {byteArr, byteArr, byteArr, byteArr}, - new byte[] { - byteArr, byteArr, byteArr, byteArr, byteArr, byteArr, - byteArr, byteArr - }, - bigintValue.add(BigDecimal.valueOf(i)), - }); - } - rows.add(row); - } - - return Pair.of(fieldNames, rows); - } - - public String insertTable(String schema, String table, String... fields) { - String columns = - Arrays.stream(fields).map(this::quoteIdentifier).collect(Collectors.joining(", ")); - String placeholders = Arrays.stream(fields).map(f -> "?").collect(Collectors.joining(", ")); - - return "INSERT INTO " - + buildTableInfoWithSchema(schema, table) - + " (" - + columns - + " )" - + " VALUES (" - + placeholders - + ")"; - } - - protected Class loadDriverClass() { - try { - return Class.forName(jdbcCase.getDriverClass()); - } catch (Exception e) { - throw new RuntimeException( - "Failed to load driver class: " + jdbcCase.getDriverClass(), e); - } - } - - protected String buildTableInfoWithSchema(String database, String schema, String table) { - return buildTableInfoWithSchema(database, table); - } - - public String buildTableInfoWithSchema(String schema, String table) { - if (StringUtils.isNotBlank(schema)) { - return quoteIdentifier(schema) + "." + quoteIdentifier(table); - } else { - return quoteIdentifier(table); - } - } - - public String quoteIdentifier(String field) { - return "`" + field + "`"; - } -} diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-7/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/MetalakeSparkIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-7/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/MetalakeSparkIT.java deleted file mode 100644 index 4e928627f8f..00000000000 --- a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-jdbc-e2e/connector-jdbc-e2e-part-7/src/test/java/org/apache/seatunnel/connectors/seatunnel/jdbc/MetalakeSparkIT.java +++ /dev/null @@ -1,454 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.seatunnel.connectors.seatunnel.jdbc; - -import org.apache.seatunnel.shade.com.google.common.collect.Lists; - -import org.apache.seatunnel.api.table.catalog.Catalog; -import org.apache.seatunnel.api.table.type.SeaTunnelRow; -import org.apache.seatunnel.common.utils.FileUtils; -import org.apache.seatunnel.e2e.common.container.spark.Spark2Container; - -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.tuple.Pair; - -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.testcontainers.DockerClientFactory; -import org.testcontainers.containers.BindMode; -import org.testcontainers.containers.Container; -import org.testcontainers.containers.GenericContainer; -import org.testcontainers.containers.MySQLContainer; -import org.testcontainers.containers.output.Slf4jLogConsumer; -import org.testcontainers.containers.wait.strategy.LogMessageWaitStrategy; -import org.testcontainers.containers.wait.strategy.Wait; -import org.testcontainers.images.PullPolicy; -import org.testcontainers.lifecycle.Startables; -import org.testcontainers.utility.DockerImageName; -import org.testcontainers.utility.DockerLoggerFactory; - -import com.github.dockerjava.api.DockerClient; - -import java.io.IOException; -import java.math.BigDecimal; -import java.sql.Connection; -import java.sql.Driver; -import java.sql.PreparedStatement; -import java.sql.SQLException; -import java.sql.Statement; -import java.time.Duration; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Properties; -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -import static org.awaitility.Awaitility.given; - -public class MetalakeSparkIT extends Spark2Container { - protected GenericContainer dbServer; - - protected JdbcCase jdbcCase; - - protected Connection connection; - - protected Catalog catalog; - - protected DockerClient dockerClient = DockerClientFactory.lazyClient(); - - protected static final String HOST = "HOST"; - - private static final String MYSQL_IMAGE = "mysql:8.0"; - private static final String MYSQL_CONTAINER_HOST = "mysql-e2e"; - private static final String MYSQL_DATABASE = "seatunnel"; - private static final String MYSQL_SOURCE = "source"; - private static final String MYSQL_SINK = "sink"; - private static final String CATALOG_DATABASE = "catalog_database"; - - private static final String MYSQL_USERNAME = "root"; - private static final String MYSQL_PASSWORD = "Abc!@#135_seatunnel"; - private static final int MYSQL_PORT = 3306; - private static final String MYSQL_URL = "jdbc:mysql://" + HOST + ":%s/%s?useSSL=false"; - private static final String URL = "jdbc:mysql://" + HOST + ":3306/seatunnel"; - - private static final String SQL = "select * from seatunnel.source"; - - private static final String DRIVER_CLASS = "com.mysql.cj.jdbc.Driver"; - - private static final List CONFIG_FILE = - Lists.newArrayList("/mysql_to_mysql_with_metalake.conf"); - private static final String CREATE_SQL = - "CREATE TABLE IF NOT EXISTS %s\n" - + "(\n" - + " `c-bit_1` bit(1) DEFAULT NULL,\n" - + " `c_bit_8` bit(8) DEFAULT NULL,\n" - + " `c_bit_16` bit(16) DEFAULT NULL,\n" - + " `c_bit_32` bit(32) DEFAULT NULL,\n" - + " `c_bit_64` bit(64) DEFAULT NULL,\n" - + " `c_bigint_30` BIGINT(40) unsigned DEFAULT NULL,\n" - + " UNIQUE (c_bigint_30)\n" - + ");"; - - @BeforeEach - @Override - public void startUp() throws Exception { - FileUtils.createNewDir(HOST_VOLUME_MOUNT_PATH); - master = - new GenericContainer<>(getDockerImage()) - .withNetwork(NETWORK) - .withNetworkAliases("spark-master") - .withExposedPorts() - .withEnv("SPARK_MODE", "master") - .withEnv("METALAKE_ENABLED", "true") - .withEnv("METALAKE_TYPE", "gravitino") - .withEnv( - "METALAKE_URL", - "http://127.0.0.1:8090/api/metalakes/test_metalake/catalogs/") - .withLogConsumer( - new Slf4jLogConsumer( - DockerLoggerFactory.getLogger(getDockerImage()))) - .withCreateContainerCmdModifier(cmd -> cmd.withUser("root")) - .withFileSystemBind( - HOST_VOLUME_MOUNT_PATH, - CONTAINER_VOLUME_MOUNT_PATH, - BindMode.READ_WRITE) - .waitingFor( - new LogMessageWaitStrategy() - .withRegEx(".*Master: Starting Spark master at.*") - .withStartupTimeout(Duration.ofMinutes(2))); - copySeaTunnelStarterToContainer(master); - copySeaTunnelStarterLoggingToContainer(master); - - // In most case we can just use standalone mode to execute a spark job, if we want to use - // cluster mode, we need to - // start a worker. - Startables.deepStart(Stream.of(master)).join(); - // execute extra commands - executeExtraCommands(master); - - master.execInContainer( - "bash", - "-c", - "mkdir -p /tmp/seatunnel/plugins/Jdbc/lib && cd /tmp/seatunnel/plugins/Jdbc/lib && wget " - + driverUrl() - + " --no-check-certificate" - + "&& mkdir -p /tmp/gravitino && cd /tmp/gravitino && curl -C - --retry 5 -L -k -o gravitino-0.9.1-bin.tar.gz https://dlcdn.apache.org/gravitino/0.9.1/gravitino-0.9.1-bin.tar.gz && tar -zxvf gravitino-0.9.1-bin.tar.gz && cd /tmp/gravitino/gravitino-0.9.1-bin && ./bin/gravitino.sh start"); - - master.execInContainer( - "bash", - "-c", - "sleep 60 && curl -L 'http://127.0.0.1:8090/api/metalakes' -H 'Content-Type: application/json' -H 'Accept: application/vnd.gravitino.v1+json' -d '{\"name\":\"test_metalake\",\"comment\":\"for metalake test\",\"properties\":{}}'" - + "&& curl -L 'http://127.0.0.1:8090/api/metalakes/test_metalake/catalogs' -H 'Content-Type: application/json' -H 'Accept: application/vnd.gravitino.v1+json' -d '{\"name\":\"test_catalog\",\"type\":\"relational\",\"provider\":\"jdbc-mysql\",\"comment\":\"for metalake test\",\"properties\":{\"jdbc-driver\":\"com.mysql.cj.jdbc.Driver\",\"jdbc-url\":\"not used\",\"jdbc-user\":\"root\",\"jdbc-password\":\"Abc!@#135_seatunnel\"}}'"); - - dbServer = initContainer().withImagePullPolicy(PullPolicy.alwaysPull()); - - Startables.deepStart(Stream.of(dbServer)).join(); - - jdbcCase = getJdbcCase(); - - given().ignoreExceptions() - .await() - .atMost(360, TimeUnit.SECONDS) - .untilAsserted(() -> this.initializeJdbcConnection(jdbcCase.getJdbcUrl())); - - createNeededTables(); - insertTestData(); - } - - @AfterEach - @Override - public void tearDown() throws Exception { - if (catalog != null) { - catalog.close(); - } - - if (connection != null) { - connection.close(); - } - - if (dbServer != null) { - dbServer.close(); - try { - dockerClient.removeImageCmd(dbServer.getDockerImageName()).exec(); - } catch (Exception ignored) { - ignored.printStackTrace(); - } - } - - super.tearDown(); - } - - @Test - public void TestMetalake() throws IOException, InterruptedException { - Container.ExecResult execResult = - executeJob("/jdbc_mysql_source_to_assert_sink_with_metalake.conf"); - Assertions.assertEquals(0, execResult.getExitCode()); - } - - String driverUrl() { - return "https://repo1.maven.org/maven2/com/mysql/mysql-connector-j/8.0.32/mysql-connector-j-8.0.32.jar"; - } - - protected GenericContainer initContainer() { - DockerImageName imageName = DockerImageName.parse(MYSQL_IMAGE); - - GenericContainer container = - new MySQLContainer<>(imageName) - .withUsername(MYSQL_USERNAME) - .withPassword(MYSQL_PASSWORD) - .withDatabaseName(MYSQL_DATABASE) - .withNetwork(NETWORK) - .withNetworkAliases(MYSQL_CONTAINER_HOST) - .withExposedPorts(MYSQL_PORT) - .waitingFor(Wait.forHealthcheck()) - .withLogConsumer( - new Slf4jLogConsumer(DockerLoggerFactory.getLogger(MYSQL_IMAGE))); - - container.setPortBindings( - Lists.newArrayList(String.format("%s:%s", MYSQL_PORT, MYSQL_PORT))); - - return container; - } - - JdbcCase getJdbcCase() { - Map containerEnv = new HashMap<>(); - String jdbcUrl = String.format(MYSQL_URL, MYSQL_PORT, MYSQL_DATABASE); - Pair> testDataSet = initTestData(); - String[] fieldNames = testDataSet.getKey(); - - String insertSql = insertTable(MYSQL_DATABASE, MYSQL_SOURCE, fieldNames); - - return JdbcCase.builder() - .dockerImage(MYSQL_IMAGE) - .networkAliases(MYSQL_CONTAINER_HOST) - .containerEnv(containerEnv) - .driverClass(DRIVER_CLASS) - .host(HOST) - .port(MYSQL_PORT) - .localPort(MYSQL_PORT) - .jdbcTemplate(MYSQL_URL) - .jdbcUrl(jdbcUrl) - .userName(MYSQL_USERNAME) - .password(MYSQL_PASSWORD) - .database(MYSQL_DATABASE) - .sourceTable(MYSQL_SOURCE) - .sinkTable(MYSQL_SINK) - .createSql(CREATE_SQL) - .configFile(CONFIG_FILE) - .insertSql(insertSql) - .testData(testDataSet) - .catalogDatabase(CATALOG_DATABASE) - .catalogTable(MYSQL_SINK) - .tablePathFullName(MYSQL_DATABASE + "." + MYSQL_SOURCE) - .build(); - } - - protected void initializeJdbcConnection(String jdbcUrl) - throws SQLException, InstantiationException, IllegalAccessException { - Driver driver = (Driver) loadDriverClass().newInstance(); - Properties props = new Properties(); - - if (StringUtils.isNotBlank(jdbcCase.getUserName())) { - props.put("user", jdbcCase.getUserName()); - } - - if (StringUtils.isNotBlank(jdbcCase.getPassword())) { - props.put("password", jdbcCase.getPassword()); - } - - if (dbServer != null) { - jdbcUrl = jdbcUrl.replace(HOST, dbServer.getHost()); - } - - this.connection = driver.connect(jdbcUrl, props); - connection.setAutoCommit(false); - } - - protected void createNeededTables() { - try (Statement statement = connection.createStatement()) { - String createTemplate = jdbcCase.getCreateSql(); - - String createSource = - String.format( - createTemplate, - buildTableInfoWithSchema( - jdbcCase.getDatabase(), - jdbcCase.getSchema(), - jdbcCase.getSourceTable())); - statement.execute(createSource); - - if (jdbcCase.getAdditionalSqlOnSource() != null) { - String additionalSql = - String.format( - jdbcCase.getAdditionalSqlOnSource(), - buildTableInfoWithSchema( - jdbcCase.getDatabase(), - jdbcCase.getSchema(), - jdbcCase.getSourceTable())); - statement.execute(additionalSql); - } - - if (!jdbcCase.isUseSaveModeCreateTable()) { - if (jdbcCase.getSinkCreateSql() != null) { - createTemplate = jdbcCase.getSinkCreateSql(); - } - String createSink = - String.format( - createTemplate, - buildTableInfoWithSchema( - jdbcCase.getDatabase(), - jdbcCase.getSchema(), - jdbcCase.getSinkTable())); - statement.execute(createSink); - } - - if (jdbcCase.getAdditionalSqlOnSink() != null) { - String additionalSql = - String.format( - jdbcCase.getAdditionalSqlOnSink(), - buildTableInfoWithSchema( - jdbcCase.getDatabase(), - jdbcCase.getSchema(), - jdbcCase.getSinkTable())); - statement.execute(additionalSql); - } - - connection.commit(); - } catch (Exception exception) { - exception.printStackTrace(); - } - } - - protected void insertTestData() { - try (PreparedStatement preparedStatement = - connection.prepareStatement(jdbcCase.getInsertSql())) { - - List rows = jdbcCase.getTestData().getValue(); - - for (SeaTunnelRow row : rows) { - for (int index = 0; index < row.getArity(); index++) { - preparedStatement.setObject(index + 1, row.getField(index)); - } - preparedStatement.addBatch(); - } - - preparedStatement.executeBatch(); - - connection.commit(); - } catch (Exception exception) { - exception.printStackTrace(); - } - } - - Pair> initTestData() { - String[] fieldNames = - new String[] { - "c-bit_1", "c_bit_8", "c_bit_16", "c_bit_32", "c_bit_64", "c_bigint_30", - }; - - List rows = new ArrayList<>(); - BigDecimal bigintValue = new BigDecimal("2844674407371055000"); - BigDecimal decimalValue = new BigDecimal("999999999999999999999999999899"); - for (int i = 0; i < 100; i++) { - byte byteArr = Integer.valueOf(i).byteValue(); - SeaTunnelRow row; - if (i == 99) { - row = - new SeaTunnelRow( - new Object[] { - (byte) 0, - new byte[] {byteArr}, - new byte[] {byteArr, byteArr}, - new byte[] {byteArr, byteArr, byteArr, byteArr}, - new byte[] { - byteArr, byteArr, byteArr, byteArr, byteArr, byteArr, - byteArr, byteArr - }, - // https://github.com/apache/seatunnel/issues/5559 this value - // cannot set null, this null - // value column's row will be lost in - // jdbc_mysql_source_and_sink_parallel.conf,jdbc_mysql_source_and_sink_parallel_upper_lower.conf. - bigintValue.add(BigDecimal.valueOf(i)), - }); - } else { - row = - new SeaTunnelRow( - new Object[] { - i % 2 == 0 ? (byte) 1 : (byte) 0, - new byte[] {byteArr}, - new byte[] {byteArr, byteArr}, - new byte[] {byteArr, byteArr, byteArr, byteArr}, - new byte[] { - byteArr, byteArr, byteArr, byteArr, byteArr, byteArr, - byteArr, byteArr - }, - bigintValue.add(BigDecimal.valueOf(i)), - }); - } - rows.add(row); - } - - return Pair.of(fieldNames, rows); - } - - public String insertTable(String schema, String table, String... fields) { - String columns = - Arrays.stream(fields).map(this::quoteIdentifier).collect(Collectors.joining(", ")); - String placeholders = Arrays.stream(fields).map(f -> "?").collect(Collectors.joining(", ")); - - return "INSERT INTO " - + buildTableInfoWithSchema(schema, table) - + " (" - + columns - + " )" - + " VALUES (" - + placeholders - + ")"; - } - - protected Class loadDriverClass() { - try { - return Class.forName(jdbcCase.getDriverClass()); - } catch (Exception e) { - throw new RuntimeException( - "Failed to load driver class: " + jdbcCase.getDriverClass(), e); - } - } - - protected String buildTableInfoWithSchema(String database, String schema, String table) { - return buildTableInfoWithSchema(database, table); - } - - public String buildTableInfoWithSchema(String schema, String table) { - if (StringUtils.isNotBlank(schema)) { - return quoteIdentifier(schema) + "." + quoteIdentifier(table); - } else { - return quoteIdentifier(table); - } - } - - public String quoteIdentifier(String field) { - return "`" + field + "`"; - } -} From 4f3f00634ab8682ca217ee14acab6e23d2ff0836 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=90=B4=E5=A4=A9=E5=AE=87?= <1599646055@qq.com> Date: Thu, 11 Sep 2025 20:08:53 +0800 Subject: [PATCH 28/39] fix: add httpcore dependency in known-dependencies.txt --- tools/dependencies/known-dependencies.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/dependencies/known-dependencies.txt b/tools/dependencies/known-dependencies.txt index 7620152ce7f..0bd82cb2783 100755 --- a/tools/dependencies/known-dependencies.txt +++ b/tools/dependencies/known-dependencies.txt @@ -9,7 +9,9 @@ disruptor-3.4.4.jar guava-27.0-jre.jar hazelcast-5.1.jar httpclient-4.5.13.jar +httpcore-4.4.13.jar httpcore-4.4.16.jar +httpcore-4.4.4.jar jackson-annotations-2.13.3.jar jackson-core-2.13.3.jar jackson-databind-2.13.3.jar From bba310afe7697ca9006d1d8646ed3ed68632ca0e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=90=B4=E5=A4=A9=E5=AE=87?= <1599646055@qq.com> Date: Sat, 13 Sep 2025 14:14:29 +0800 Subject: [PATCH 29/39] fix: reuse PlaceholderUtils and refactor the getMetalakeConfig method; fix some errors in the docs --- docs/en/concept/metalake.md | 2 +- docs/zh/concept/metalake.md | 6 +- .../api/metalake/MetalakeConfigUtils.java | 117 ++++++++++++++++++ .../command/FlinkTaskExecuteCommand.java | 97 +-------------- .../command/SparkTaskExecuteCommand.java | 97 +-------------- .../parse/MultipleTableJobConfigParser.java | 96 +------------- 6 files changed, 128 insertions(+), 287 deletions(-) create mode 100644 seatunnel-api/src/main/java/org/apache/seatunnel/api/metalake/MetalakeConfigUtils.java diff --git a/docs/en/concept/metalake.md b/docs/en/concept/metalake.md index 1e76c995483..916d76b4513 100644 --- a/docs/en/concept/metalake.md +++ b/docs/en/concept/metalake.md @@ -66,4 +66,4 @@ source { } ``` -Here, `sourceId` refers to the catalog name, allowing other fields to use `${}` placeholders. At runtime, they will be automatically replaced. Note that in sinks, the same `sourceId` name is used, and placeholders must always start with `${` and end with `}`. \ No newline at end of file +Here, `sourceId` refers to the catalog name, allowing other fields to use `${}` placeholders. At runtime, they will be automatically replaced. Note that in sinks, the same `sourceId` name is used, and placeholders must always start with `${` and end with `}`. Each item can contain at most one placeholder, and there can be content outside the placeholder as well. \ No newline at end of file diff --git a/docs/zh/concept/metalake.md b/docs/zh/concept/metalake.md index f702680ea3f..d3b52eb5520 100644 --- a/docs/zh/concept/metalake.md +++ b/docs/zh/concept/metalake.md @@ -1,8 +1,8 @@ # METALAKE -由于seatunnel在执行任务时，需要将数据库用户名与密码等隐私信息明文写在脚本中，可能会导致信息泄露；并且维护较为困难，数据源信息发生变更时可能需要手动更改。 +由于Seatunnel在执行任务时，需要将数据库用户名与密码等隐私信息明文写在脚本中，可能会导致信息泄露；并且维护较为困难，数据源信息发生变更时可能需要手动更改。 -因此引入了metalake，将数据源的信息存储于Apache Gravitino等metalake中，任务脚本采用`sourId`和占位符的方法来代替原本的用户名和密码等信息，运行时seatunnel-engine通过http请求从metalake获取信息，根据占位符进行替换。 +因此引入了metalake，将数据源的信息存储于Apache Gravitino等metalake中，任务脚本采用`sourceId`和占位符的方法来代替原本的用户名和密码等信息，运行时seatunnel-engine通过http请求从metalake获取信息，根据占位符进行替换。 若要使用metalake，首先要修改**seatunnel-env.sh**中的环境变量： @@ -66,4 +66,4 @@ source { } ``` -其中`sourceId`指代catalog的名称，从而其他项可以使用`${}`占位符，运行时会自动替换。注意，在sink中使用时，同样叫`sourceId`；使用占位符时必须以`${`开头，以`}`结尾 \ No newline at end of file +其中`sourceId`指代catalog的名称，从而其他项可以使用`${}`占位符，运行时会自动替换。注意，在sink中使用时，同样叫`sourceId`；使用占位符时必须以`${`开头，以`}`结尾，每一项最多只能包含一个占位符，占位符以外也可以有内容 \ No newline at end of file diff --git a/seatunnel-api/src/main/java/org/apache/seatunnel/api/metalake/MetalakeConfigUtils.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/metalake/MetalakeConfigUtils.java new file mode 100644 index 00000000000..e837cfc6b12 --- /dev/null +++ b/seatunnel-api/src/main/java/org/apache/seatunnel/api/metalake/MetalakeConfigUtils.java @@ -0,0 +1,117 @@ +package org.apache.seatunnel.api.metalake; + +import org.apache.seatunnel.shade.com.fasterxml.jackson.databind.JsonNode; +import org.apache.seatunnel.shade.com.typesafe.config.Config; +import org.apache.seatunnel.shade.com.typesafe.config.ConfigList; +import org.apache.seatunnel.shade.com.typesafe.config.ConfigObject; +import org.apache.seatunnel.shade.com.typesafe.config.ConfigValue; +import org.apache.seatunnel.shade.com.typesafe.config.ConfigValueFactory; +import org.apache.seatunnel.shade.com.typesafe.config.ConfigValueType; + +import org.apache.seatunnel.common.utils.PlaceholderUtils; + +import lombok.extern.slf4j.Slf4j; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +@Slf4j +public class MetalakeConfigUtils { + + private static final Pattern pattern = Pattern.compile("\\$\\{[^}]*\\}"); + + public static Config getMetalakeConfig(Config jobConfigTmp) { + Config update = jobConfigTmp; + String metalakeType = System.getenv("METALAKE_TYPE"); + String metalakeUrl = System.getenv("METALAKE_URL"); + + MetalakeClient metalakeClient = MetalakeClientFactory.create(metalakeType, metalakeUrl); + + try { + ConfigList sourceList = jobConfigTmp.getList("source"); + List newSourceList = new ArrayList<>(sourceList); + + for (int i = 0; i < sourceList.size(); i++) { + ConfigObject sourceObj = (ConfigObject) sourceList.get(i); + if (sourceObj.containsKey("sourceId")) { + ConfigObject tmp = sourceObj; + String sourceId = sourceObj.toConfig().getString("sourceId"); + JsonNode metalakeJson = metalakeClient.getMetaInfo(sourceId); + for (Map.Entry entry : sourceObj.entrySet()) { + String subKey = entry.getKey(); + ConfigValue value = entry.getValue(); + + if (value.valueType() == ConfigValueType.STRING) { + String strValue = (String) value.unwrapped(); + Matcher matcher = pattern.matcher(strValue); + if (matcher.find()) { + String placeholder = matcher.group(1); + + if (metalakeJson.has(placeholder)) { + String replaced = metalakeJson.get(placeholder).asText(); + String newValue = + PlaceholderUtils.replacePlaceholders( + strValue, placeholder, replaced); + tmp = + tmp.withValue( + subKey, + ConfigValueFactory.fromAnyRef(newValue)); + } + } + } + } + newSourceList.set(i, tmp); + } + } + update = update.withValue("source", ConfigValueFactory.fromIterable(newSourceList)); + } catch (IOException e) { + log.error("Fail to get MetaInfo, metalakeUrl: {}", metalakeUrl, e); + } + + try { + ConfigList sinkList = jobConfigTmp.getList("sink"); + List newSinkList = new ArrayList<>(sinkList); + + for (int i = 0; i < sinkList.size(); i++) { + ConfigObject sinkObj = (ConfigObject) sinkList.get(i); + if (sinkObj.containsKey("sourceId")) { + ConfigObject tmp = sinkObj; + String sourceId = sinkObj.toConfig().getString("sourceId"); + JsonNode metalakeJson = metalakeClient.getMetaInfo(sourceId); + for (Map.Entry entry : sinkObj.entrySet()) { + String subKey = entry.getKey(); + ConfigValue value = entry.getValue(); + + if (value.valueType() == ConfigValueType.STRING) { + String strValue = (String) value.unwrapped(); + Matcher matcher = pattern.matcher(strValue); + if (matcher.find()) { + String placeholder = matcher.group(1); + + if (metalakeJson.has(placeholder)) { + String replaced = metalakeJson.get(placeholder).asText(); + String newValue = + PlaceholderUtils.replacePlaceholders( + strValue, placeholder, replaced); + tmp = + tmp.withValue( + subKey, + ConfigValueFactory.fromAnyRef(newValue)); + } + } + } + } + newSinkList.set(i, tmp); + } + } + update = update.withValue("sink", ConfigValueFactory.fromIterable(newSinkList)); + } catch (IOException e) { + log.error("Fail to get MetaInfo, metalakeUrl: {}", metalakeUrl, e); + } + return update; + } +} diff --git a/seatunnel-core/seatunnel-flink-starter/seatunnel-flink-starter-common/src/main/java/org/apache/seatunnel/core/starter/flink/command/FlinkTaskExecuteCommand.java b/seatunnel-core/seatunnel-flink-starter/seatunnel-flink-starter-common/src/main/java/org/apache/seatunnel/core/starter/flink/command/FlinkTaskExecuteCommand.java index 9621c3c182f..015695a12e5 100644 --- a/seatunnel-core/seatunnel-flink-starter/seatunnel-flink-starter-common/src/main/java/org/apache/seatunnel/core/starter/flink/command/FlinkTaskExecuteCommand.java +++ b/seatunnel-core/seatunnel-flink-starter/seatunnel-flink-starter-common/src/main/java/org/apache/seatunnel/core/starter/flink/command/FlinkTaskExecuteCommand.java @@ -17,17 +17,11 @@ package org.apache.seatunnel.core.starter.flink.command; -import org.apache.seatunnel.shade.com.fasterxml.jackson.databind.JsonNode; import org.apache.seatunnel.shade.com.typesafe.config.Config; -import org.apache.seatunnel.shade.com.typesafe.config.ConfigList; -import org.apache.seatunnel.shade.com.typesafe.config.ConfigObject; import org.apache.seatunnel.shade.com.typesafe.config.ConfigUtil; -import org.apache.seatunnel.shade.com.typesafe.config.ConfigValue; import org.apache.seatunnel.shade.com.typesafe.config.ConfigValueFactory; -import org.apache.seatunnel.shade.com.typesafe.config.ConfigValueType; -import org.apache.seatunnel.api.metalake.MetalakeClient; -import org.apache.seatunnel.api.metalake.MetalakeClientFactory; +import org.apache.seatunnel.api.metalake.MetalakeConfigUtils; import org.apache.seatunnel.common.Constants; import org.apache.seatunnel.core.starter.command.Command; import org.apache.seatunnel.core.starter.exception.CommandExecuteException; @@ -38,11 +32,7 @@ import lombok.extern.slf4j.Slf4j; -import java.io.IOException; import java.nio.file.Path; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; import static org.apache.seatunnel.core.starter.utils.FileUtils.checkConfigExist; @@ -64,7 +54,7 @@ public void execute() throws CommandExecuteException { Boolean.parseBoolean(System.getenv().getOrDefault("METALAKE_ENABLED", "false")); if (metalakeEnabled) { config = - getMetalakeConfig( + MetalakeConfigUtils.getMetalakeConfig( ConfigBuilder.of(configFile, flinkCommandArgs.getVariables())); } else { config = ConfigBuilder.of(configFile, flinkCommandArgs.getVariables()); @@ -83,87 +73,4 @@ public void execute() throws CommandExecuteException { throw new CommandExecuteException("Flink job executed failed", e); } } - - private Config getMetalakeConfig(Config jobConfigTmp) { - Config update = jobConfigTmp; - String metalakeType = System.getenv("METALAKE_TYPE"); - String metalakeUrl = System.getenv("METALAKE_URL"); - - MetalakeClient metalakeClient = MetalakeClientFactory.create(metalakeType, metalakeUrl); - - try { - ConfigList sourceList = jobConfigTmp.getList("source"); - List newSourceList = new ArrayList<>(sourceList); - - for (int i = 0; i < sourceList.size(); i++) { - ConfigObject sourceObj = (ConfigObject) sourceList.get(i); - if (sourceObj.containsKey("sourceId")) { - ConfigObject tmp = sourceObj; - String sourceId = sourceObj.toConfig().getString("sourceId"); - JsonNode metalakeJson = metalakeClient.getMetaInfo(sourceId); - for (Map.Entry entry : sourceObj.entrySet()) { - String subKey = entry.getKey(); - ConfigValue value = entry.getValue(); - - if (value.valueType() == ConfigValueType.STRING) { - String strValue = (String) value.unwrapped(); - if (strValue.startsWith("${") && strValue.endsWith("}")) { - String placeholder = strValue.substring(2, strValue.length() - 1); - - if (metalakeJson.has(placeholder)) { - String replaced = metalakeJson.get(placeholder).asText(); - tmp = - tmp.withValue( - subKey, - ConfigValueFactory.fromAnyRef(replaced)); - } - } - } - } - newSourceList.set(i, tmp); - } - } - update = update.withValue("source", ConfigValueFactory.fromIterable(newSourceList)); - } catch (IOException e) { - log.error("Fail to get MetaInfo, metalakeUrl: {}", metalakeUrl, e); - } - - try { - ConfigList sinkList = jobConfigTmp.getList("sink"); - List newSinkList = new ArrayList<>(sinkList); - - for (int i = 0; i < sinkList.size(); i++) { - ConfigObject sinkObj = (ConfigObject) sinkList.get(i); - if (sinkObj.containsKey("sourceId")) { - ConfigObject tmp = sinkObj; - String sourceId = sinkObj.toConfig().getString("sourceId"); - JsonNode metalakeJson = metalakeClient.getMetaInfo(sourceId); - for (Map.Entry entry : sinkObj.entrySet()) { - String subKey = entry.getKey(); - ConfigValue value = entry.getValue(); - - if (value.valueType() == ConfigValueType.STRING) { - String strValue = (String) value.unwrapped(); - if (strValue.startsWith("${") && strValue.endsWith("}")) { - String placeholder = strValue.substring(2, strValue.length() - 1); - - if (metalakeJson.has(placeholder)) { - String replaced = metalakeJson.get(placeholder).asText(); - tmp = - tmp.withValue( - subKey, - ConfigValueFactory.fromAnyRef(replaced)); - } - } - } - } - newSinkList.set(i, tmp); - } - } - update = update.withValue("sink", ConfigValueFactory.fromIterable(newSinkList)); - } catch (IOException e) { - log.error("Fail to get MetaInfo, metalakeUrl: {}", metalakeUrl, e); - } - return update; - } } diff --git a/seatunnel-core/seatunnel-spark-starter/seatunnel-spark-starter-common/src/main/java/org/apache/seatunnel/core/starter/spark/command/SparkTaskExecuteCommand.java b/seatunnel-core/seatunnel-spark-starter/seatunnel-spark-starter-common/src/main/java/org/apache/seatunnel/core/starter/spark/command/SparkTaskExecuteCommand.java index fad440ef091..c5ad4243a91 100644 --- a/seatunnel-core/seatunnel-spark-starter/seatunnel-spark-starter-common/src/main/java/org/apache/seatunnel/core/starter/spark/command/SparkTaskExecuteCommand.java +++ b/seatunnel-core/seatunnel-spark-starter/seatunnel-spark-starter-common/src/main/java/org/apache/seatunnel/core/starter/spark/command/SparkTaskExecuteCommand.java @@ -17,17 +17,11 @@ package org.apache.seatunnel.core.starter.spark.command; -import org.apache.seatunnel.shade.com.fasterxml.jackson.databind.JsonNode; import org.apache.seatunnel.shade.com.typesafe.config.Config; -import org.apache.seatunnel.shade.com.typesafe.config.ConfigList; -import org.apache.seatunnel.shade.com.typesafe.config.ConfigObject; import org.apache.seatunnel.shade.com.typesafe.config.ConfigUtil; -import org.apache.seatunnel.shade.com.typesafe.config.ConfigValue; import org.apache.seatunnel.shade.com.typesafe.config.ConfigValueFactory; -import org.apache.seatunnel.shade.com.typesafe.config.ConfigValueType; -import org.apache.seatunnel.api.metalake.MetalakeClient; -import org.apache.seatunnel.api.metalake.MetalakeClientFactory; +import org.apache.seatunnel.api.metalake.MetalakeConfigUtils; import org.apache.seatunnel.common.Constants; import org.apache.seatunnel.core.starter.command.Command; import org.apache.seatunnel.core.starter.exception.CommandExecuteException; @@ -38,11 +32,7 @@ import lombok.extern.slf4j.Slf4j; -import java.io.IOException; import java.nio.file.Path; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; import static org.apache.seatunnel.core.starter.utils.FileUtils.checkConfigExist; @@ -64,7 +54,7 @@ public void execute() throws CommandExecuteException { Boolean.parseBoolean(System.getenv().getOrDefault("METALAKE_ENABLED", "false")); if (metalakeEnabled) { config = - getMetalakeConfig( + MetalakeConfigUtils.getMetalakeConfig( ConfigBuilder.of(configFile, sparkCommandArgs.getVariables())); } else { config = ConfigBuilder.of(configFile, sparkCommandArgs.getVariables()); @@ -82,87 +72,4 @@ public void execute() throws CommandExecuteException { throw new CommandExecuteException("Run SeaTunnel on spark failed", e); } } - - private Config getMetalakeConfig(Config jobConfigTmp) { - Config update = jobConfigTmp; - String metalakeType = System.getenv("METALAKE_TYPE"); - String metalakeUrl = System.getenv("METALAKE_URL"); - - MetalakeClient metalakeClient = MetalakeClientFactory.create(metalakeType, metalakeUrl); - - try { - ConfigList sourceList = jobConfigTmp.getList("source"); - List newSourceList = new ArrayList<>(sourceList); - - for (int i = 0; i < sourceList.size(); i++) { - ConfigObject sourceObj = (ConfigObject) sourceList.get(i); - if (sourceObj.containsKey("sourceId")) { - ConfigObject tmp = sourceObj; - String sourceId = sourceObj.toConfig().getString("sourceId"); - JsonNode metalakeJson = metalakeClient.getMetaInfo(sourceId); - for (Map.Entry entry : sourceObj.entrySet()) { - String subKey = entry.getKey(); - ConfigValue value = entry.getValue(); - - if (value.valueType() == ConfigValueType.STRING) { - String strValue = (String) value.unwrapped(); - if (strValue.startsWith("${") && strValue.endsWith("}")) { - String placeholder = strValue.substring(2, strValue.length() - 1); - - if (metalakeJson.has(placeholder)) { - String replaced = metalakeJson.get(placeholder).asText(); - tmp = - tmp.withValue( - subKey, - ConfigValueFactory.fromAnyRef(replaced)); - } - } - } - } - newSourceList.set(i, tmp); - } - } - update = update.withValue("source", ConfigValueFactory.fromIterable(newSourceList)); - } catch (IOException e) { - log.error("Fail to get MetaInfo, metalakeUrl: {}", metalakeUrl, e); - } - - try { - ConfigList sinkList = jobConfigTmp.getList("sink"); - List newSinkList = new ArrayList<>(sinkList); - - for (int i = 0; i < sinkList.size(); i++) { - ConfigObject sinkObj = (ConfigObject) sinkList.get(i); - if (sinkObj.containsKey("sourceId")) { - ConfigObject tmp = sinkObj; - String sourceId = sinkObj.toConfig().getString("sourceId"); - JsonNode metalakeJson = metalakeClient.getMetaInfo(sourceId); - for (Map.Entry entry : sinkObj.entrySet()) { - String subKey = entry.getKey(); - ConfigValue value = entry.getValue(); - - if (value.valueType() == ConfigValueType.STRING) { - String strValue = (String) value.unwrapped(); - if (strValue.startsWith("${") && strValue.endsWith("}")) { - String placeholder = strValue.substring(2, strValue.length() - 1); - - if (metalakeJson.has(placeholder)) { - String replaced = metalakeJson.get(placeholder).asText(); - tmp = - tmp.withValue( - subKey, - ConfigValueFactory.fromAnyRef(replaced)); - } - } - } - } - newSinkList.set(i, tmp); - } - } - update = update.withValue("sink", ConfigValueFactory.fromIterable(newSinkList)); - } catch (IOException e) { - log.error("Fail to get MetaInfo, metalakeUrl: {}", metalakeUrl, e); - } - return update; - } } diff --git a/seatunnel-engine/seatunnel-engine-core/src/main/java/org/apache/seatunnel/engine/core/parse/MultipleTableJobConfigParser.java b/seatunnel-engine/seatunnel-engine-core/src/main/java/org/apache/seatunnel/engine/core/parse/MultipleTableJobConfigParser.java index 216bc7a8ffd..87b1a20e7b9 100644 --- a/seatunnel-engine/seatunnel-engine-core/src/main/java/org/apache/seatunnel/engine/core/parse/MultipleTableJobConfigParser.java +++ b/seatunnel-engine/seatunnel-engine-core/src/main/java/org/apache/seatunnel/engine/core/parse/MultipleTableJobConfigParser.java @@ -17,21 +17,14 @@ package org.apache.seatunnel.engine.core.parse; -import org.apache.seatunnel.shade.com.fasterxml.jackson.databind.JsonNode; import org.apache.seatunnel.shade.com.google.common.base.Preconditions; import org.apache.seatunnel.shade.com.google.common.collect.Lists; import org.apache.seatunnel.shade.com.typesafe.config.Config; -import org.apache.seatunnel.shade.com.typesafe.config.ConfigList; -import org.apache.seatunnel.shade.com.typesafe.config.ConfigObject; -import org.apache.seatunnel.shade.com.typesafe.config.ConfigValue; -import org.apache.seatunnel.shade.com.typesafe.config.ConfigValueFactory; -import org.apache.seatunnel.shade.com.typesafe.config.ConfigValueType; import org.apache.seatunnel.api.common.PluginIdentifier; import org.apache.seatunnel.api.configuration.ReadonlyConfig; import org.apache.seatunnel.api.configuration.util.ConfigValidator; -import org.apache.seatunnel.api.metalake.MetalakeClient; -import org.apache.seatunnel.api.metalake.MetalakeClientFactory; +import org.apache.seatunnel.api.metalake.MetalakeConfigUtils; import org.apache.seatunnel.api.options.ConnectorCommonOptions; import org.apache.seatunnel.api.options.EnvCommonOptions; import org.apache.seatunnel.api.options.EnvOptionRule; @@ -81,7 +74,6 @@ import lombok.extern.slf4j.Slf4j; import scala.Tuple2; -import java.io.IOException; import java.io.Serializable; import java.net.MalformedURLException; import java.net.URL; @@ -185,7 +177,8 @@ public MultipleTableJobConfigParser( Boolean.parseBoolean(System.getenv().getOrDefault("METALAKE_ENABLED", "false")); if (metalakeEnabled) { this.seaTunnelJobConfig = - getMetalakeConfig(ConfigBuilder.of(Paths.get(jobDefineFilePath), variables)); + MetalakeConfigUtils.getMetalakeConfig( + ConfigBuilder.of(Paths.get(jobDefineFilePath), variables)); } else { this.seaTunnelJobConfig = ConfigBuilder.of(Paths.get(jobDefineFilePath), variables); } @@ -847,87 +840,4 @@ private ChangeStreamTableSourceCheckpoint getSourceCheckpoint( .collect(Collectors.toList()); return new ChangeStreamTableSourceCheckpoint(coordinatorState, subtaskState); } - - private Config getMetalakeConfig(Config jobConfigTmp) { - Config update = jobConfigTmp; - String metalakeType = System.getenv("METALAKE_TYPE"); - String metalakeUrl = System.getenv("METALAKE_URL"); - - MetalakeClient metalakeClient = MetalakeClientFactory.create(metalakeType, metalakeUrl); - - try { - ConfigList sourceList = jobConfigTmp.getList("source"); - List newSourceList = new ArrayList<>(sourceList); - - for (int i = 0; i < sourceList.size(); i++) { - ConfigObject sourceObj = (ConfigObject) sourceList.get(i); - if (sourceObj.containsKey("sourceId")) { - ConfigObject tmp = sourceObj; - String sourceId = sourceObj.toConfig().getString("sourceId"); - JsonNode metalakeJson = metalakeClient.getMetaInfo(sourceId); - for (Map.Entry entry : sourceObj.entrySet()) { - String subKey = entry.getKey(); - ConfigValue value = entry.getValue(); - - if (value.valueType() == ConfigValueType.STRING) { - String strValue = (String) value.unwrapped(); - if (strValue.startsWith("${") && strValue.endsWith("}")) { - String placeholder = strValue.substring(2, strValue.length() - 1); - - if (metalakeJson.has(placeholder)) { - String replaced = metalakeJson.get(placeholder).asText(); - tmp = - tmp.withValue( - subKey, - ConfigValueFactory.fromAnyRef(replaced)); - } - } - } - } - newSourceList.set(i, tmp); - } - } - update = update.withValue("source", ConfigValueFactory.fromIterable(newSourceList)); - } catch (IOException e) { - log.error("Fail to get MetaInfo, metalakeUrl: {}", metalakeUrl, e); - } - - try { - ConfigList sinkList = jobConfigTmp.getList("sink"); - List newSinkList = new ArrayList<>(sinkList); - - for (int i = 0; i < sinkList.size(); i++) { - ConfigObject sinkObj = (ConfigObject) sinkList.get(i); - if (sinkObj.containsKey("sourceId")) { - ConfigObject tmp = sinkObj; - String sourceId = sinkObj.toConfig().getString("sourceId"); - JsonNode metalakeJson = metalakeClient.getMetaInfo(sourceId); - for (Map.Entry entry : sinkObj.entrySet()) { - String subKey = entry.getKey(); - ConfigValue value = entry.getValue(); - - if (value.valueType() == ConfigValueType.STRING) { - String strValue = (String) value.unwrapped(); - if (strValue.startsWith("${") && strValue.endsWith("}")) { - String placeholder = strValue.substring(2, strValue.length() - 1); - - if (metalakeJson.has(placeholder)) { - String replaced = metalakeJson.get(placeholder).asText(); - tmp = - tmp.withValue( - subKey, - ConfigValueFactory.fromAnyRef(replaced)); - } - } - } - } - newSinkList.set(i, tmp); - } - } - update = update.withValue("sink", ConfigValueFactory.fromIterable(newSinkList)); - } catch (IOException e) { - log.error("Fail to get MetaInfo, metalakeUrl: {}", metalakeUrl, e); - } - return update; - } } From 72a75387945b71d4785e42b5d8e1a8f15dd0b881 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=90=B4=E5=A4=A9=E5=AE=87?= <1599646055@qq.com> Date: Sat, 13 Sep 2025 14:19:02 +0800 Subject: [PATCH 30/39] fix: add license header --- .../api/metalake/MetalakeConfigUtils.java | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/seatunnel-api/src/main/java/org/apache/seatunnel/api/metalake/MetalakeConfigUtils.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/metalake/MetalakeConfigUtils.java index e837cfc6b12..62b44e5dd02 100644 --- a/seatunnel-api/src/main/java/org/apache/seatunnel/api/metalake/MetalakeConfigUtils.java +++ b/seatunnel-api/src/main/java/org/apache/seatunnel/api/metalake/MetalakeConfigUtils.java @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.seatunnel.api.metalake; import org.apache.seatunnel.shade.com.fasterxml.jackson.databind.JsonNode; From 46d19f7d620449b43e8e44eaae81a808721a6286 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=90=B4=E5=A4=A9=E5=AE=87?= <1599646055@qq.com> Date: Sat, 13 Sep 2025 23:20:39 +0800 Subject: [PATCH 31/39] fix: add capture group in pattern matcher --- .../org/apache/seatunnel/api/metalake/MetalakeConfigUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/seatunnel-api/src/main/java/org/apache/seatunnel/api/metalake/MetalakeConfigUtils.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/metalake/MetalakeConfigUtils.java index 62b44e5dd02..0e1d1c12019 100644 --- a/seatunnel-api/src/main/java/org/apache/seatunnel/api/metalake/MetalakeConfigUtils.java +++ b/seatunnel-api/src/main/java/org/apache/seatunnel/api/metalake/MetalakeConfigUtils.java @@ -39,7 +39,7 @@ @Slf4j public class MetalakeConfigUtils { - private static final Pattern pattern = Pattern.compile("\\$\\{[^}]*\\}"); + private static final Pattern pattern = Pattern.compile("\\$\\{([^}]*)\\}"); public static Config getMetalakeConfig(Config jobConfigTmp) { Config update = jobConfigTmp; From 8c96119ddee2e91b0f6f3130f715e3810aefbdd6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=90=B4=E5=A4=A9=E5=AE=87?= <1599646055@qq.com> Date: Mon, 15 Sep 2025 20:06:50 +0800 Subject: [PATCH 32/39] fix: refactor MetalakeConfigUtils and use PlaceholderUtils and JsonUtils --- .../api/metalake/GravitinoClient.java | 6 +- .../api/metalake/MetalakeConfigUtils.java | 113 ++++++------------ .../seatunnel/common/utils/JsonUtils.java | 5 + .../common/utils/PlaceholderUtils.java | 16 +++ 4 files changed, 60 insertions(+), 80 deletions(-) diff --git a/seatunnel-api/src/main/java/org/apache/seatunnel/api/metalake/GravitinoClient.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/metalake/GravitinoClient.java index de95bf3a875..f479974988a 100644 --- a/seatunnel-api/src/main/java/org/apache/seatunnel/api/metalake/GravitinoClient.java +++ b/seatunnel-api/src/main/java/org/apache/seatunnel/api/metalake/GravitinoClient.java @@ -18,7 +18,8 @@ package org.apache.seatunnel.api.metalake; import org.apache.seatunnel.shade.com.fasterxml.jackson.databind.JsonNode; -import org.apache.seatunnel.shade.com.fasterxml.jackson.databind.ObjectMapper; + +import org.apache.seatunnel.common.utils.JsonUtils; import org.apache.http.HttpEntity; import org.apache.http.client.methods.CloseableHttpResponse; @@ -51,8 +52,7 @@ public JsonNode getMetaInfo(String sourceId) throws IOException { if (entity == null) { throw new RuntimeException("No response entity"); } - ObjectMapper mapper = new ObjectMapper(); - JsonNode rootNode = mapper.readTree(entity.getContent()); + JsonNode rootNode = JsonUtils.readTree(entity.getContent()); EntityUtils.consume(entity); JsonNode catalogNode = rootNode.get("catalog"); if (catalogNode == null) { diff --git a/seatunnel-api/src/main/java/org/apache/seatunnel/api/metalake/MetalakeConfigUtils.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/metalake/MetalakeConfigUtils.java index 0e1d1c12019..bb869ad60e0 100644 --- a/seatunnel-api/src/main/java/org/apache/seatunnel/api/metalake/MetalakeConfigUtils.java +++ b/seatunnel-api/src/main/java/org/apache/seatunnel/api/metalake/MetalakeConfigUtils.java @@ -33,102 +33,61 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; -import java.util.regex.Matcher; -import java.util.regex.Pattern; @Slf4j public class MetalakeConfigUtils { - private static final Pattern pattern = Pattern.compile("\\$\\{([^}]*)\\}"); - public static Config getMetalakeConfig(Config jobConfigTmp) { Config update = jobConfigTmp; - String metalakeType = System.getenv("METALAKE_TYPE"); - String metalakeUrl = System.getenv("METALAKE_URL"); - - MetalakeClient metalakeClient = MetalakeClientFactory.create(metalakeType, metalakeUrl); try { ConfigList sourceList = jobConfigTmp.getList("source"); - List newSourceList = new ArrayList<>(sourceList); - - for (int i = 0; i < sourceList.size(); i++) { - ConfigObject sourceObj = (ConfigObject) sourceList.get(i); - if (sourceObj.containsKey("sourceId")) { - ConfigObject tmp = sourceObj; - String sourceId = sourceObj.toConfig().getString("sourceId"); - JsonNode metalakeJson = metalakeClient.getMetaInfo(sourceId); - for (Map.Entry entry : sourceObj.entrySet()) { - String subKey = entry.getKey(); - ConfigValue value = entry.getValue(); - - if (value.valueType() == ConfigValueType.STRING) { - String strValue = (String) value.unwrapped(); - Matcher matcher = pattern.matcher(strValue); - if (matcher.find()) { - String placeholder = matcher.group(1); - - if (metalakeJson.has(placeholder)) { - String replaced = metalakeJson.get(placeholder).asText(); - String newValue = - PlaceholderUtils.replacePlaceholders( - strValue, placeholder, replaced); - tmp = - tmp.withValue( - subKey, - ConfigValueFactory.fromAnyRef(newValue)); - } - } - } - } - newSourceList.set(i, tmp); - } - } - update = update.withValue("source", ConfigValueFactory.fromIterable(newSourceList)); + update = + update.withValue( + "source", + ConfigValueFactory.fromIterable(replaceConfigList(sourceList))); } catch (IOException e) { - log.error("Fail to get MetaInfo, metalakeUrl: {}", metalakeUrl, e); + log.error("Fail to get MetaInfo", e); } try { ConfigList sinkList = jobConfigTmp.getList("sink"); - List newSinkList = new ArrayList<>(sinkList); - - for (int i = 0; i < sinkList.size(); i++) { - ConfigObject sinkObj = (ConfigObject) sinkList.get(i); - if (sinkObj.containsKey("sourceId")) { - ConfigObject tmp = sinkObj; - String sourceId = sinkObj.toConfig().getString("sourceId"); - JsonNode metalakeJson = metalakeClient.getMetaInfo(sourceId); - for (Map.Entry entry : sinkObj.entrySet()) { - String subKey = entry.getKey(); - ConfigValue value = entry.getValue(); + update = + update.withValue( + "sink", ConfigValueFactory.fromIterable(replaceConfigList(sinkList))); + } catch (IOException e) { + log.error("Fail to get MetaInfo", e); + } + return update; + } - if (value.valueType() == ConfigValueType.STRING) { - String strValue = (String) value.unwrapped(); - Matcher matcher = pattern.matcher(strValue); - if (matcher.find()) { - String placeholder = matcher.group(1); + private static List replaceConfigList(ConfigList list) throws IOException { + String metalakeType = System.getenv("METALAKE_TYPE"); + String metalakeUrl = System.getenv("METALAKE_URL"); + MetalakeClient metalakeClient = MetalakeClientFactory.create(metalakeType, metalakeUrl); - if (metalakeJson.has(placeholder)) { - String replaced = metalakeJson.get(placeholder).asText(); - String newValue = - PlaceholderUtils.replacePlaceholders( - strValue, placeholder, replaced); - tmp = - tmp.withValue( - subKey, - ConfigValueFactory.fromAnyRef(newValue)); - } - } - } + List newConfigList = new ArrayList<>(list); + + for (int i = 0; i < list.size(); i++) { + ConfigObject Obj = (ConfigObject) list.get(i); + if (Obj.containsKey("sourceId")) { + ConfigObject tmp = Obj; + String sourceId = Obj.toConfig().getString("sourceId"); + JsonNode metalakeJson = metalakeClient.getMetaInfo(sourceId); + for (Map.Entry entry : Obj.entrySet()) { + String subKey = entry.getKey(); + ConfigValue value = entry.getValue(); + + if (value.valueType() == ConfigValueType.STRING) { + String strValue = (String) value.unwrapped(); + String newValue = + PlaceholderUtils.replacePlaceholders(strValue, metalakeJson); + tmp = tmp.withValue(subKey, ConfigValueFactory.fromAnyRef(newValue)); } - newSinkList.set(i, tmp); } + newConfigList.set(i, tmp); } - update = update.withValue("sink", ConfigValueFactory.fromIterable(newSinkList)); - } catch (IOException e) { - log.error("Fail to get MetaInfo, metalakeUrl: {}", metalakeUrl, e); } - return update; + return newConfigList; } } diff --git a/seatunnel-common/src/main/java/org/apache/seatunnel/common/utils/JsonUtils.java b/seatunnel-common/src/main/java/org/apache/seatunnel/common/utils/JsonUtils.java index 5d765583ba8..3d8e62684d1 100644 --- a/seatunnel-common/src/main/java/org/apache/seatunnel/common/utils/JsonUtils.java +++ b/seatunnel-common/src/main/java/org/apache/seatunnel/common/utils/JsonUtils.java @@ -39,6 +39,7 @@ import org.apache.commons.lang3.StringUtils; import java.io.IOException; +import java.io.InputStream; import java.util.ArrayList; import java.util.Collections; import java.util.LinkedHashMap; @@ -90,6 +91,10 @@ public static JsonNode readTree(byte[] obj) throws IOException { return OBJECT_MAPPER.readTree(obj); } + public static JsonNode readTree(InputStream obj) throws IOException { + return OBJECT_MAPPER.readTree(obj); + } + /** * json representation of object * diff --git a/seatunnel-common/src/main/java/org/apache/seatunnel/common/utils/PlaceholderUtils.java b/seatunnel-common/src/main/java/org/apache/seatunnel/common/utils/PlaceholderUtils.java index ab697e7357f..b707b895fe8 100644 --- a/seatunnel-common/src/main/java/org/apache/seatunnel/common/utils/PlaceholderUtils.java +++ b/seatunnel-common/src/main/java/org/apache/seatunnel/common/utils/PlaceholderUtils.java @@ -17,6 +17,8 @@ package org.apache.seatunnel.common.utils; +import org.apache.seatunnel.shade.com.fasterxml.jackson.databind.JsonNode; + import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -48,4 +50,18 @@ public static String replacePlaceholders( matcher.appendTail(result); return result.toString(); } + + public static String replacePlaceholders(String input, JsonNode json) { + Pattern pattern = Pattern.compile("\\$\\{([^}]*)\\}"); + Matcher matcher = pattern.matcher(input); + if (matcher.find()) { + String placeholder = matcher.group(1); + + if (json.has(placeholder)) { + String replaced = json.get(placeholder).asText(); + return replacePlaceholders(input, placeholder, replaced); + } + } + return input; + } } From 4df31f8dbbd318460244fd12a58f88169a2dd95f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=90=B4=E5=A4=A9=E5=AE=87?= <1599646055@qq.com> Date: Wed, 17 Sep 2025 20:09:12 +0800 Subject: [PATCH 33/39] fix: unify the version of httpcore --- seatunnel-api/pom.xml | 12 +++++++++++- seatunnel-connectors-v2/connector-doris/pom.xml | 2 +- .../connector-http/connector-http-base/pom.xml | 2 +- .../connector-selectdb-cloud/pom.xml | 2 +- seatunnel-connectors-v2/connector-starrocks/pom.xml | 2 +- 5 files changed, 15 insertions(+), 5 deletions(-) diff --git a/seatunnel-api/pom.xml b/seatunnel-api/pom.xml index 80c771f4309..f7f813ab50d 100644 --- a/seatunnel-api/pom.xml +++ b/seatunnel-api/pom.xml @@ -29,6 +29,11 @@ seatunnel-api SeaTunnel : Api + + 4.5.13 + 4.4.16 + + org.apache.seatunnel @@ -44,7 +49,12 @@ org.apache.httpcomponents httpclient - 4.5.13 + ${httpclient.version} + + + org.apache.httpcomponents + httpcore + ${httpcore.version} diff --git a/seatunnel-connectors-v2/connector-doris/pom.xml b/seatunnel-connectors-v2/connector-doris/pom.xml index 06cd650f4a9..31abc163182 100644 --- a/seatunnel-connectors-v2/connector-doris/pom.xml +++ b/seatunnel-connectors-v2/connector-doris/pom.xml @@ -31,7 +31,7 @@ 4.5.13 - 4.4.4 + 4.4.16 diff --git a/seatunnel-connectors-v2/connector-http/connector-http-base/pom.xml b/seatunnel-connectors-v2/connector-http/connector-http-base/pom.xml index e19e9ab9848..1e3882017cc 100644 --- a/seatunnel-connectors-v2/connector-http/connector-http-base/pom.xml +++ b/seatunnel-connectors-v2/connector-http/connector-http-base/pom.xml @@ -31,7 +31,7 @@ 4.5.13 - 4.4.4 + 4.4.16 2.0.0 3.12.4 diff --git a/seatunnel-connectors-v2/connector-selectdb-cloud/pom.xml b/seatunnel-connectors-v2/connector-selectdb-cloud/pom.xml index 2750ddaa333..908f365d50f 100644 --- a/seatunnel-connectors-v2/connector-selectdb-cloud/pom.xml +++ b/seatunnel-connectors-v2/connector-selectdb-cloud/pom.xml @@ -31,7 +31,7 @@ 4.5.13 - 4.4.4 + 4.4.16 diff --git a/seatunnel-connectors-v2/connector-starrocks/pom.xml b/seatunnel-connectors-v2/connector-starrocks/pom.xml index 560c26ca13c..52dd6509698 100644 --- a/seatunnel-connectors-v2/connector-starrocks/pom.xml +++ b/seatunnel-connectors-v2/connector-starrocks/pom.xml @@ -32,7 +32,7 @@ connector.starrocks 4.5.13 - 4.4.4 + 4.4.16 8.0.16 1.0.1 5.0.0 From 0f3363a78ffce5cee684950a7182e60442d14da1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=90=B4=E5=A4=A9=E5=AE=87?= <1599646055@qq.com> Date: Wed, 17 Sep 2025 20:36:07 +0800 Subject: [PATCH 34/39] fix: remove extra version of httpcore in kwown-dependencies.txt --- tools/dependencies/known-dependencies.txt | 2 -- 1 file changed, 2 deletions(-) diff --git a/tools/dependencies/known-dependencies.txt b/tools/dependencies/known-dependencies.txt index 0bd82cb2783..7620152ce7f 100755 --- a/tools/dependencies/known-dependencies.txt +++ b/tools/dependencies/known-dependencies.txt @@ -9,9 +9,7 @@ disruptor-3.4.4.jar guava-27.0-jre.jar hazelcast-5.1.jar httpclient-4.5.13.jar -httpcore-4.4.13.jar httpcore-4.4.16.jar -httpcore-4.4.4.jar jackson-annotations-2.13.3.jar jackson-core-2.13.3.jar jackson-databind-2.13.3.jar From dffb3b61b52725707a5034d9cfa5b5d54ec56fd4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=90=B4=E5=A4=A9=E5=AE=87?= <1599646055@qq.com> Date: Thu, 18 Sep 2025 20:05:24 +0800 Subject: [PATCH 35/39] feat: get variables from env before from system --- .../api/metalake/MetalakeConfigUtils.java | 21 ++++++++++++++----- .../command/FlinkTaskExecuteCommand.java | 7 ++++++- .../command/SparkTaskExecuteCommand.java | 7 ++++++- .../parse/MultipleTableJobConfigParser.java | 7 ++++++- 4 files changed, 34 insertions(+), 8 deletions(-) diff --git a/seatunnel-api/src/main/java/org/apache/seatunnel/api/metalake/MetalakeConfigUtils.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/metalake/MetalakeConfigUtils.java index bb869ad60e0..34beea193fc 100644 --- a/seatunnel-api/src/main/java/org/apache/seatunnel/api/metalake/MetalakeConfigUtils.java +++ b/seatunnel-api/src/main/java/org/apache/seatunnel/api/metalake/MetalakeConfigUtils.java @@ -39,13 +39,15 @@ public class MetalakeConfigUtils { public static Config getMetalakeConfig(Config jobConfigTmp) { Config update = jobConfigTmp; + Config envConfig = jobConfigTmp.getConfig("env"); try { ConfigList sourceList = jobConfigTmp.getList("source"); update = update.withValue( "source", - ConfigValueFactory.fromIterable(replaceConfigList(sourceList))); + ConfigValueFactory.fromIterable( + replaceConfigList(sourceList, envConfig))); } catch (IOException e) { log.error("Fail to get MetaInfo", e); } @@ -54,16 +56,25 @@ public static Config getMetalakeConfig(Config jobConfigTmp) { ConfigList sinkList = jobConfigTmp.getList("sink"); update = update.withValue( - "sink", ConfigValueFactory.fromIterable(replaceConfigList(sinkList))); + "sink", + ConfigValueFactory.fromIterable( + replaceConfigList(sinkList, envConfig))); } catch (IOException e) { log.error("Fail to get MetaInfo", e); } return update; } - private static List replaceConfigList(ConfigList list) throws IOException { - String metalakeType = System.getenv("METALAKE_TYPE"); - String metalakeUrl = System.getenv("METALAKE_URL"); + private static List replaceConfigList(ConfigList list, Config envConfig) + throws IOException { + String metalakeType = + envConfig.hasPath("metalake_type") + ? envConfig.getString("metalake_type") + : System.getenv("METALAKE_TYPE"); + String metalakeUrl = + envConfig.hasPath("metalake_url") + ? envConfig.getString("metalake_url") + : System.getenv("METALAKE_URL"); MetalakeClient metalakeClient = MetalakeClientFactory.create(metalakeType, metalakeUrl); List newConfigList = new ArrayList<>(list); diff --git a/seatunnel-core/seatunnel-flink-starter/seatunnel-flink-starter-common/src/main/java/org/apache/seatunnel/core/starter/flink/command/FlinkTaskExecuteCommand.java b/seatunnel-core/seatunnel-flink-starter/seatunnel-flink-starter-common/src/main/java/org/apache/seatunnel/core/starter/flink/command/FlinkTaskExecuteCommand.java index 015695a12e5..00170e5dfcd 100644 --- a/seatunnel-core/seatunnel-flink-starter/seatunnel-flink-starter-common/src/main/java/org/apache/seatunnel/core/starter/flink/command/FlinkTaskExecuteCommand.java +++ b/seatunnel-core/seatunnel-flink-starter/seatunnel-flink-starter-common/src/main/java/org/apache/seatunnel/core/starter/flink/command/FlinkTaskExecuteCommand.java @@ -50,8 +50,13 @@ public void execute() throws CommandExecuteException { Path configFile = FileUtils.getConfigPath(flinkCommandArgs); checkConfigExist(configFile); Config config = null; + Config envConfig = + ConfigBuilder.of(configFile, flinkCommandArgs.getVariables()).getConfig("env"); boolean metalakeEnabled = - Boolean.parseBoolean(System.getenv().getOrDefault("METALAKE_ENABLED", "false")); + envConfig.hasPath("metalake_enabled") + ? envConfig.getBoolean("metalake_enabled") + : Boolean.parseBoolean( + System.getenv().getOrDefault("METALAKE_ENABLED", "false")); if (metalakeEnabled) { config = MetalakeConfigUtils.getMetalakeConfig( diff --git a/seatunnel-core/seatunnel-spark-starter/seatunnel-spark-starter-common/src/main/java/org/apache/seatunnel/core/starter/spark/command/SparkTaskExecuteCommand.java b/seatunnel-core/seatunnel-spark-starter/seatunnel-spark-starter-common/src/main/java/org/apache/seatunnel/core/starter/spark/command/SparkTaskExecuteCommand.java index c5ad4243a91..4484d018bec 100644 --- a/seatunnel-core/seatunnel-spark-starter/seatunnel-spark-starter-common/src/main/java/org/apache/seatunnel/core/starter/spark/command/SparkTaskExecuteCommand.java +++ b/seatunnel-core/seatunnel-spark-starter/seatunnel-spark-starter-common/src/main/java/org/apache/seatunnel/core/starter/spark/command/SparkTaskExecuteCommand.java @@ -50,8 +50,13 @@ public void execute() throws CommandExecuteException { Path configFile = FileUtils.getConfigPath(sparkCommandArgs); checkConfigExist(configFile); Config config = null; + Config envConfig = + ConfigBuilder.of(configFile, sparkCommandArgs.getVariables()).getConfig("env"); boolean metalakeEnabled = - Boolean.parseBoolean(System.getenv().getOrDefault("METALAKE_ENABLED", "false")); + envConfig.hasPath("metalake_enabled") + ? envConfig.getBoolean("metalake_enabled") + : Boolean.parseBoolean( + System.getenv().getOrDefault("METALAKE_ENABLED", "false")); if (metalakeEnabled) { config = MetalakeConfigUtils.getMetalakeConfig( diff --git a/seatunnel-engine/seatunnel-engine-core/src/main/java/org/apache/seatunnel/engine/core/parse/MultipleTableJobConfigParser.java b/seatunnel-engine/seatunnel-engine-core/src/main/java/org/apache/seatunnel/engine/core/parse/MultipleTableJobConfigParser.java index 87b1a20e7b9..e232a0044e4 100644 --- a/seatunnel-engine/seatunnel-engine-core/src/main/java/org/apache/seatunnel/engine/core/parse/MultipleTableJobConfigParser.java +++ b/seatunnel-engine/seatunnel-engine-core/src/main/java/org/apache/seatunnel/engine/core/parse/MultipleTableJobConfigParser.java @@ -173,8 +173,13 @@ public MultipleTableJobConfigParser( this.jobConfig = jobConfig; this.commonPluginJars = commonPluginJars; this.isStartWithSavePoint = isStartWithSavePoint; + Config envConfig = + ConfigBuilder.of(Paths.get(jobDefineFilePath), variables).getConfig("env"); boolean metalakeEnabled = - Boolean.parseBoolean(System.getenv().getOrDefault("METALAKE_ENABLED", "false")); + envConfig.hasPath("metalake_enabled") + ? envConfig.getBoolean("metalake_enabled") + : Boolean.parseBoolean( + System.getenv().getOrDefault("METALAKE_ENABLED", "false")); if (metalakeEnabled) { this.seaTunnelJobConfig = MetalakeConfigUtils.getMetalakeConfig( From 5bb154f63874b3e44f912c25a2217624d832dc83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=90=B4=E5=A4=A9=E5=AE=87?= <1599646055@qq.com> Date: Fri, 19 Sep 2025 15:30:13 +0800 Subject: [PATCH 36/39] feat: refactor getMetalakeConfig method and support transform --- .../api/metalake/MetalakeConfigUtils.java | 19 ++++++++++++++++++- .../command/FlinkTaskExecuteCommand.java | 18 +++--------------- .../command/SparkTaskExecuteCommand.java | 18 +++--------------- .../parse/MultipleTableJobConfigParser.java | 17 +++-------------- 4 files changed, 27 insertions(+), 45 deletions(-) diff --git a/seatunnel-api/src/main/java/org/apache/seatunnel/api/metalake/MetalakeConfigUtils.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/metalake/MetalakeConfigUtils.java index 34beea193fc..389aff4390a 100644 --- a/seatunnel-api/src/main/java/org/apache/seatunnel/api/metalake/MetalakeConfigUtils.java +++ b/seatunnel-api/src/main/java/org/apache/seatunnel/api/metalake/MetalakeConfigUtils.java @@ -38,9 +38,15 @@ public class MetalakeConfigUtils { public static Config getMetalakeConfig(Config jobConfigTmp) { - Config update = jobConfigTmp; Config envConfig = jobConfigTmp.getConfig("env"); + boolean metalakeEnabled = + envConfig.hasPath("metalake_enabled") + ? envConfig.getBoolean("metalake_enabled") + : Boolean.parseBoolean( + System.getenv().getOrDefault("METALAKE_ENABLED", "false")); + if (!metalakeEnabled) return jobConfigTmp; + Config update = jobConfigTmp; try { ConfigList sourceList = jobConfigTmp.getList("source"); update = @@ -62,6 +68,17 @@ public static Config getMetalakeConfig(Config jobConfigTmp) { } catch (IOException e) { log.error("Fail to get MetaInfo", e); } + + try { + ConfigList sinkList = jobConfigTmp.getList("transform"); + update = + update.withValue( + "transform", + ConfigValueFactory.fromIterable( + replaceConfigList(sinkList, envConfig))); + } catch (IOException e) { + log.error("Fail to get MetaInfo", e); + } return update; } diff --git a/seatunnel-core/seatunnel-flink-starter/seatunnel-flink-starter-common/src/main/java/org/apache/seatunnel/core/starter/flink/command/FlinkTaskExecuteCommand.java b/seatunnel-core/seatunnel-flink-starter/seatunnel-flink-starter-common/src/main/java/org/apache/seatunnel/core/starter/flink/command/FlinkTaskExecuteCommand.java index 00170e5dfcd..b251e381aba 100644 --- a/seatunnel-core/seatunnel-flink-starter/seatunnel-flink-starter-common/src/main/java/org/apache/seatunnel/core/starter/flink/command/FlinkTaskExecuteCommand.java +++ b/seatunnel-core/seatunnel-flink-starter/seatunnel-flink-starter-common/src/main/java/org/apache/seatunnel/core/starter/flink/command/FlinkTaskExecuteCommand.java @@ -49,21 +49,9 @@ public FlinkTaskExecuteCommand(FlinkCommandArgs flinkCommandArgs) { public void execute() throws CommandExecuteException { Path configFile = FileUtils.getConfigPath(flinkCommandArgs); checkConfigExist(configFile); - Config config = null; - Config envConfig = - ConfigBuilder.of(configFile, flinkCommandArgs.getVariables()).getConfig("env"); - boolean metalakeEnabled = - envConfig.hasPath("metalake_enabled") - ? envConfig.getBoolean("metalake_enabled") - : Boolean.parseBoolean( - System.getenv().getOrDefault("METALAKE_ENABLED", "false")); - if (metalakeEnabled) { - config = - MetalakeConfigUtils.getMetalakeConfig( - ConfigBuilder.of(configFile, flinkCommandArgs.getVariables())); - } else { - config = ConfigBuilder.of(configFile, flinkCommandArgs.getVariables()); - } + Config config = + MetalakeConfigUtils.getMetalakeConfig( + ConfigBuilder.of(configFile, flinkCommandArgs.getVariables())); // if user specified job name using command line arguments, override config option if (!flinkCommandArgs.getJobName().equals(Constants.LOGO)) { config = diff --git a/seatunnel-core/seatunnel-spark-starter/seatunnel-spark-starter-common/src/main/java/org/apache/seatunnel/core/starter/spark/command/SparkTaskExecuteCommand.java b/seatunnel-core/seatunnel-spark-starter/seatunnel-spark-starter-common/src/main/java/org/apache/seatunnel/core/starter/spark/command/SparkTaskExecuteCommand.java index 4484d018bec..67fef2ac4c9 100644 --- a/seatunnel-core/seatunnel-spark-starter/seatunnel-spark-starter-common/src/main/java/org/apache/seatunnel/core/starter/spark/command/SparkTaskExecuteCommand.java +++ b/seatunnel-core/seatunnel-spark-starter/seatunnel-spark-starter-common/src/main/java/org/apache/seatunnel/core/starter/spark/command/SparkTaskExecuteCommand.java @@ -49,21 +49,9 @@ public SparkTaskExecuteCommand(SparkCommandArgs sparkCommandArgs) { public void execute() throws CommandExecuteException { Path configFile = FileUtils.getConfigPath(sparkCommandArgs); checkConfigExist(configFile); - Config config = null; - Config envConfig = - ConfigBuilder.of(configFile, sparkCommandArgs.getVariables()).getConfig("env"); - boolean metalakeEnabled = - envConfig.hasPath("metalake_enabled") - ? envConfig.getBoolean("metalake_enabled") - : Boolean.parseBoolean( - System.getenv().getOrDefault("METALAKE_ENABLED", "false")); - if (metalakeEnabled) { - config = - MetalakeConfigUtils.getMetalakeConfig( - ConfigBuilder.of(configFile, sparkCommandArgs.getVariables())); - } else { - config = ConfigBuilder.of(configFile, sparkCommandArgs.getVariables()); - } + Config config = + MetalakeConfigUtils.getMetalakeConfig( + ConfigBuilder.of(configFile, sparkCommandArgs.getVariables())); if (!sparkCommandArgs.getJobName().equals(Constants.LOGO)) { config = config.withValue( diff --git a/seatunnel-engine/seatunnel-engine-core/src/main/java/org/apache/seatunnel/engine/core/parse/MultipleTableJobConfigParser.java b/seatunnel-engine/seatunnel-engine-core/src/main/java/org/apache/seatunnel/engine/core/parse/MultipleTableJobConfigParser.java index e232a0044e4..c815a031768 100644 --- a/seatunnel-engine/seatunnel-engine-core/src/main/java/org/apache/seatunnel/engine/core/parse/MultipleTableJobConfigParser.java +++ b/seatunnel-engine/seatunnel-engine-core/src/main/java/org/apache/seatunnel/engine/core/parse/MultipleTableJobConfigParser.java @@ -173,20 +173,9 @@ public MultipleTableJobConfigParser( this.jobConfig = jobConfig; this.commonPluginJars = commonPluginJars; this.isStartWithSavePoint = isStartWithSavePoint; - Config envConfig = - ConfigBuilder.of(Paths.get(jobDefineFilePath), variables).getConfig("env"); - boolean metalakeEnabled = - envConfig.hasPath("metalake_enabled") - ? envConfig.getBoolean("metalake_enabled") - : Boolean.parseBoolean( - System.getenv().getOrDefault("METALAKE_ENABLED", "false")); - if (metalakeEnabled) { - this.seaTunnelJobConfig = - MetalakeConfigUtils.getMetalakeConfig( - ConfigBuilder.of(Paths.get(jobDefineFilePath), variables)); - } else { - this.seaTunnelJobConfig = ConfigBuilder.of(Paths.get(jobDefineFilePath), variables); - } + this.seaTunnelJobConfig = + MetalakeConfigUtils.getMetalakeConfig( + ConfigBuilder.of(Paths.get(jobDefineFilePath), variables)); this.envOptions = ReadonlyConfig.fromConfig(seaTunnelJobConfig.getConfig("env")); this.pipelineCheckpoints = pipelineCheckpoints; ConfigValidator.of(this.envOptions).validate(new EnvOptionRule().optionRule()); From dde368c516a3ddae9de7b904029e7ff03e60e45d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=90=B4=E5=A4=A9=E5=AE=87?= <1599646055@qq.com> Date: Sat, 20 Sep 2025 23:36:43 +0800 Subject: [PATCH 37/39] fix: modify the arg name of replacePlaceholders method and refactor getMetalakeConfig method --- .../api/metalake/MetalakeConfigUtils.java | 76 +++++++------------ .../common/utils/PlaceholderUtils.java | 6 +- 2 files changed, 29 insertions(+), 53 deletions(-) diff --git a/seatunnel-api/src/main/java/org/apache/seatunnel/api/metalake/MetalakeConfigUtils.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/metalake/MetalakeConfigUtils.java index 389aff4390a..e6c0cb3e405 100644 --- a/seatunnel-api/src/main/java/org/apache/seatunnel/api/metalake/MetalakeConfigUtils.java +++ b/seatunnel-api/src/main/java/org/apache/seatunnel/api/metalake/MetalakeConfigUtils.java @@ -47,43 +47,14 @@ public static Config getMetalakeConfig(Config jobConfigTmp) { if (!metalakeEnabled) return jobConfigTmp; Config update = jobConfigTmp; - try { - ConfigList sourceList = jobConfigTmp.getList("source"); - update = - update.withValue( - "source", - ConfigValueFactory.fromIterable( - replaceConfigList(sourceList, envConfig))); - } catch (IOException e) { - log.error("Fail to get MetaInfo", e); - } - - try { - ConfigList sinkList = jobConfigTmp.getList("sink"); - update = - update.withValue( - "sink", - ConfigValueFactory.fromIterable( - replaceConfigList(sinkList, envConfig))); - } catch (IOException e) { - log.error("Fail to get MetaInfo", e); - } - - try { - ConfigList sinkList = jobConfigTmp.getList("transform"); - update = - update.withValue( - "transform", - ConfigValueFactory.fromIterable( - replaceConfigList(sinkList, envConfig))); - } catch (IOException e) { - log.error("Fail to get MetaInfo", e); - } + update = replaceConfigList(update, "source"); + update = replaceConfigList(update, "sink"); + update = replaceConfigList(update, "transform"); return update; } - private static List replaceConfigList(ConfigList list, Config envConfig) - throws IOException { + private static Config replaceConfigList(Config updateConfig, String key) { + Config envConfig = updateConfig.getConfig("env"); String metalakeType = envConfig.hasPath("metalake_type") ? envConfig.getString("metalake_type") @@ -94,28 +65,33 @@ private static List replaceConfigList(ConfigList list, Config envCo : System.getenv("METALAKE_URL"); MetalakeClient metalakeClient = MetalakeClientFactory.create(metalakeType, metalakeUrl); + ConfigList list = updateConfig.getList(key); List newConfigList = new ArrayList<>(list); - for (int i = 0; i < list.size(); i++) { - ConfigObject Obj = (ConfigObject) list.get(i); - if (Obj.containsKey("sourceId")) { - ConfigObject tmp = Obj; - String sourceId = Obj.toConfig().getString("sourceId"); - JsonNode metalakeJson = metalakeClient.getMetaInfo(sourceId); - for (Map.Entry entry : Obj.entrySet()) { - String subKey = entry.getKey(); - ConfigValue value = entry.getValue(); + try { + for (int i = 0; i < list.size(); i++) { + ConfigObject Obj = (ConfigObject) list.get(i); + if (Obj.containsKey("sourceId")) { + ConfigObject tmp = Obj; + String sourceId = Obj.toConfig().getString("sourceId"); + JsonNode metalakeJson = metalakeClient.getMetaInfo(sourceId); + for (Map.Entry entry : Obj.entrySet()) { + String subKey = entry.getKey(); + ConfigValue value = entry.getValue(); - if (value.valueType() == ConfigValueType.STRING) { - String strValue = (String) value.unwrapped(); - String newValue = - PlaceholderUtils.replacePlaceholders(strValue, metalakeJson); - tmp = tmp.withValue(subKey, ConfigValueFactory.fromAnyRef(newValue)); + if (value.valueType() == ConfigValueType.STRING) { + String strValue = (String) value.unwrapped(); + String newValue = + PlaceholderUtils.replacePlaceholders(strValue, metalakeJson); + tmp = tmp.withValue(subKey, ConfigValueFactory.fromAnyRef(newValue)); + } } + newConfigList.set(i, tmp); } - newConfigList.set(i, tmp); } + } catch (IOException e) { + log.error("Fail to get MetaInfo", e); } - return newConfigList; + return updateConfig.withValue(key, ConfigValueFactory.fromIterable(newConfigList)); } } diff --git a/seatunnel-common/src/main/java/org/apache/seatunnel/common/utils/PlaceholderUtils.java b/seatunnel-common/src/main/java/org/apache/seatunnel/common/utils/PlaceholderUtils.java index b707b895fe8..c14d1c3b473 100644 --- a/seatunnel-common/src/main/java/org/apache/seatunnel/common/utils/PlaceholderUtils.java +++ b/seatunnel-common/src/main/java/org/apache/seatunnel/common/utils/PlaceholderUtils.java @@ -51,14 +51,14 @@ public static String replacePlaceholders( return result.toString(); } - public static String replacePlaceholders(String input, JsonNode json) { + public static String replacePlaceholders(String input, JsonNode supportedValues) { Pattern pattern = Pattern.compile("\\$\\{([^}]*)\\}"); Matcher matcher = pattern.matcher(input); if (matcher.find()) { String placeholder = matcher.group(1); - if (json.has(placeholder)) { - String replaced = json.get(placeholder).asText(); + if (supportedValues.has(placeholder)) { + String replaced = supportedValues.get(placeholder).asText(); return replacePlaceholders(input, placeholder, replaced); } } From 236240248c76295f3d03fc1773567e574921fea1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=90=B4=E5=A4=A9=E5=AE=87?= <1599646055@qq.com> Date: Sun, 21 Sep 2025 17:16:53 +0800 Subject: [PATCH 38/39] fix: create metalakeClient only once in getMetalakeConfig method --- .../api/metalake/MetalakeConfigUtils.java | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/seatunnel-api/src/main/java/org/apache/seatunnel/api/metalake/MetalakeConfigUtils.java b/seatunnel-api/src/main/java/org/apache/seatunnel/api/metalake/MetalakeConfigUtils.java index e6c0cb3e405..12f683cb89c 100644 --- a/seatunnel-api/src/main/java/org/apache/seatunnel/api/metalake/MetalakeConfigUtils.java +++ b/seatunnel-api/src/main/java/org/apache/seatunnel/api/metalake/MetalakeConfigUtils.java @@ -47,14 +47,6 @@ public static Config getMetalakeConfig(Config jobConfigTmp) { if (!metalakeEnabled) return jobConfigTmp; Config update = jobConfigTmp; - update = replaceConfigList(update, "source"); - update = replaceConfigList(update, "sink"); - update = replaceConfigList(update, "transform"); - return update; - } - - private static Config replaceConfigList(Config updateConfig, String key) { - Config envConfig = updateConfig.getConfig("env"); String metalakeType = envConfig.hasPath("metalake_type") ? envConfig.getString("metalake_type") @@ -64,7 +56,14 @@ private static Config replaceConfigList(Config updateConfig, String key) { ? envConfig.getString("metalake_url") : System.getenv("METALAKE_URL"); MetalakeClient metalakeClient = MetalakeClientFactory.create(metalakeType, metalakeUrl); + update = replaceConfigList(update, "source", metalakeClient); + update = replaceConfigList(update, "sink", metalakeClient); + update = replaceConfigList(update, "transform", metalakeClient); + return update; + } + private static Config replaceConfigList( + Config updateConfig, String key, MetalakeClient metalakeClient) { ConfigList list = updateConfig.getList(key); List newConfigList = new ArrayList<>(list); From 788bfda16a1e78d9c53510bae539bf43361a12a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=90=B4=E5=A4=A9=E5=AE=87?= <1599646055@qq.com> Date: Mon, 22 Sep 2025 22:19:40 +0800 Subject: [PATCH 39/39] fix: add metalake in sidebar.js of docs --- docs/sidebars.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/sidebars.js b/docs/sidebars.js index cfd589ec112..c0640c044d2 100644 --- a/docs/sidebars.js +++ b/docs/sidebars.js @@ -94,7 +94,8 @@ const sidebars = { 'concept/sql-config', 'concept/speed-limit', 'concept/event-listener', - 'concept/schema-evolution' + 'concept/schema-evolution', + 'concept/metalake' ] }, {