From 77e4f5a4718036703ec1fdd58c017e9f81bf6fb8 Mon Sep 17 00:00:00 2001 From: Shreya Khajanchi Date: Tue, 29 Oct 2024 13:31:49 +0530 Subject: [PATCH 01/13] minor changes --- .../teleport/v2/templates/mysql/MySqlDao.java | 52 ++++++++++++++ .../templates/transforms/SourceWriterFn.java | 33 ++++----- .../v2/templates/utils/ISourceDao.java | 29 ++++++++ .../utils/ISourceProcessorFactory.java | 5 ++ .../templates/utils/InputRecordProcessor.java | 6 +- .../teleport/v2/templates/utils/MySqlDao.java | 70 ------------------- .../transforms/SourceWriterFnTest.java | 2 +- .../v2/templates/utils/MySqlDaoTest.java | 1 + 8 files changed, 106 insertions(+), 92 deletions(-) create mode 100644 v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/mysql/MySqlDao.java create mode 100644 v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/ISourceDao.java create mode 100644 v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/ISourceProcessorFactory.java delete mode 100644 v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/MySqlDao.java diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/mysql/MySqlDao.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/mysql/MySqlDao.java new file mode 100644 index 0000000000..f42ed3554c --- /dev/null +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/mysql/MySqlDao.java @@ -0,0 +1,52 @@ +/* + * Copyright (C) 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.templates.mysql; + +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; +import com.google.cloud.teleport.v2.templates.utils.ConnectionHelper; +import com.google.cloud.teleport.v2.templates.utils.ISourceDao; + +public class MySqlDao implements ISourceDao { + private String sqlUrl; + private String sqlUser; + private String sqlPasswd; + private Connection connObj; + + @Override + public void initialize(String url, String user, String password) throws Exception { + this.sqlUrl = url; + this.sqlUser = user; + this.sqlPasswd = password; + Class.forName("com.mysql.cj.jdbc.Driver"); + this.connObj = ConnectionHelper.getConnection(this.sqlUrl, this.sqlUser, this.sqlPasswd); + } + + @Override + public void write(String sqlStatement) throws SQLException { + try (Statement statement = connObj.createStatement()) { + statement.executeUpdate(sqlStatement); + } + } + + @Override + public void close() throws SQLException { + if (connObj != null && !connObj.isClosed()) { + connObj.close(); + } + } +} diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/transforms/SourceWriterFn.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/transforms/SourceWriterFn.java index 533155152e..fb8a857ab1 100644 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/transforms/SourceWriterFn.java +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/transforms/SourceWriterFn.java @@ -30,12 +30,8 @@ import com.google.cloud.teleport.v2.templates.changestream.ChangeStreamErrorRecord; import com.google.cloud.teleport.v2.templates.changestream.TrimmedShardedDataChangeRecord; import com.google.cloud.teleport.v2.templates.constants.Constants; -import com.google.cloud.teleport.v2.templates.utils.ConnectionException; -import com.google.cloud.teleport.v2.templates.utils.ConnectionHelper; -import com.google.cloud.teleport.v2.templates.utils.InputRecordProcessor; -import com.google.cloud.teleport.v2.templates.utils.MySqlDao; -import com.google.cloud.teleport.v2.templates.utils.ShadowTableRecord; -import com.google.cloud.teleport.v2.templates.utils.SpannerDao; +import com.google.cloud.teleport.v2.templates.utils.*; +import com.google.cloud.teleport.v2.templates.mysql.MySqlDao; import com.google.common.collect.ImmutableList; import com.google.gson.Gson; import java.io.Serializable; @@ -50,8 +46,6 @@ import org.apache.beam.sdk.metrics.Distribution; import org.apache.beam.sdk.metrics.Metrics; import org.apache.beam.sdk.transforms.DoFn; -import org.apache.beam.sdk.transforms.DoFn.ProcessContext; -import org.apache.beam.sdk.transforms.DoFn.ProcessElement; import org.apache.beam.sdk.values.KV; import org.apache.beam.sdk.values.TupleTag; import org.slf4j.Logger; @@ -76,7 +70,7 @@ public class SourceWriterFn extends DoFn mySqlDaoMap = new HashMap<>(); + private transient Map sourceDaoMap = new HashMap<>(); private final Schema schema; private final String sourceDbTimezoneOffset; @@ -114,8 +108,8 @@ public void setSpannerDao(SpannerDao spannerDao) { } // for unit testing purposes - public void setMySqlDaoMap(Map mySqlDaoMap) { - this.mySqlDaoMap = mySqlDaoMap; + public void setSourceDaoMap(Map sourceDaoMap) { + this.sourceDaoMap = sourceDaoMap; } // for unit testing purposes @@ -125,17 +119,17 @@ public void setObjectMapper(ObjectMapper mapper) { /** Setup function connects to Cloud Spanner. */ @Setup - public void setup() { + public void setup() throws Exception { mapper = new ObjectMapper(); mapper.enable(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS); ConnectionHelper.init(shards, null, maxThreadPerDataflowWorker); - mySqlDaoMap = new HashMap<>(); + sourceDaoMap = new HashMap<>(); for (Shard shard : shards) { String sourceConnectionUrl = "jdbc:mysql://" + shard.getHost() + ":" + shard.getPort() + "/" + shard.getDbName(); - mySqlDaoMap.put( - shard.getLogicalShardId(), - new MySqlDao(sourceConnectionUrl, shard.getUserName(), shard.getPassword())); + ISourceDao mySqlDao = new MySqlDao(); + mySqlDao.initialize(sourceConnectionUrl, shard.getUserName(), shard.getPassword()); + sourceDaoMap.put(shard.getLogicalShardId(), mySqlDao); } spannerDao = new SpannerDao(spannerConfig); } @@ -144,7 +138,7 @@ public void setup() { @Teardown public void teardown() throws Exception { spannerDao.close(); - mySqlDaoMap.clear(); + sourceDaoMap.clear(); } @ProcessElement @@ -188,9 +182,10 @@ public void processElement(ProcessContext c) { > Long.parseLong(spannerRec.getRecordSequence()))); if (!isSourceAhead) { - MySqlDao mySqlDao = mySqlDaoMap.get(shardId); + ISourceDao mySqlDao = sourceDaoMap.get(shardId); - InputRecordProcessor.processRecord( + InputRecordProcessor inputRecordProcessor = new InputRecordProcessor(); + inputRecordProcessor.processRecord( spannerRec, schema, mySqlDao, shardId, sourceDbTimezoneOffset); spannerDao.updateShadowTable( diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/ISourceDao.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/ISourceDao.java new file mode 100644 index 0000000000..08974b961f --- /dev/null +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/ISourceDao.java @@ -0,0 +1,29 @@ +package com.google.cloud.teleport.v2.templates.utils; + +import java.sql.SQLException; + +public interface ISourceDao { + + /** + * Initializes the DAO with the necessary connection parameters. + * + * @param url Connection URL. + * @param user Database user. + * @param password User password. + */ + void initialize(String url, String user, String password) throws Exception; + + /** + * Executes a given write statement. + * + * @param statement The SQL or query statement. + * @throws SQLException If there is an error executing the statement. + */ + void write(String statement) throws SQLException; + + /** + * Closes any open connections. + */ + void close() throws SQLException; +} + diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/ISourceProcessorFactory.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/ISourceProcessorFactory.java new file mode 100644 index 0000000000..daab702cfc --- /dev/null +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/ISourceProcessorFactory.java @@ -0,0 +1,5 @@ +package com.google.cloud.teleport.v2.templates.utils; + +public interface ISourceProcessorFactory { + +} diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/InputRecordProcessor.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/InputRecordProcessor.java index bf6040ed2f..43cd9d7332 100644 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/InputRecordProcessor.java +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/InputRecordProcessor.java @@ -19,6 +19,8 @@ import com.google.cloud.teleport.v2.templates.changestream.TrimmedShardedDataChangeRecord; import java.time.Instant; import java.time.temporal.ChronoUnit; + +import com.google.cloud.teleport.v2.templates.mysql.MySqlDao; import org.apache.beam.sdk.metrics.Counter; import org.apache.beam.sdk.metrics.Distribution; import org.apache.beam.sdk.metrics.Metrics; @@ -32,10 +34,10 @@ public class InputRecordProcessor { private static final Logger LOG = LoggerFactory.getLogger(InputRecordProcessor.class); - public static void processRecord( + public void processRecord( TrimmedShardedDataChangeRecord spannerRecord, Schema schema, - MySqlDao dao, + ISourceDao dao, String shardId, String sourceDbTimezoneOffset) throws java.sql.SQLException, ConnectionException { diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/MySqlDao.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/MySqlDao.java deleted file mode 100644 index 31abd4e040..0000000000 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/MySqlDao.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright (C) 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ -package com.google.cloud.teleport.v2.templates.utils; - -import java.io.Serializable; -import java.sql.Connection; -import java.sql.SQLException; -import java.sql.Statement; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** Writes data to MySQL. */ -public class MySqlDao implements Serializable { - private static final Logger LOG = LoggerFactory.getLogger(MySqlDao.class); - - static final String JDBC_DRIVER = "com.mysql.jdbc.Driver"; - - String sqlUrl; - String sqlUser; - String sqlPasswd; - - public MySqlDao(String sqlUrl, String sqlUser, String sqlPasswd) { - try { - Class dirverClass = Class.forName(JDBC_DRIVER); - } catch (ClassNotFoundException e) { - LOG.error("There was not able to find the driver class"); - } - this.sqlUrl = sqlUrl; - this.sqlUser = sqlUser; - this.sqlPasswd = sqlPasswd; - } - - // writes to database - public void write(String sqlStatement) throws SQLException, ConnectionException { - Connection connObj = null; - Statement statement = null; - - try { - - connObj = ConnectionHelper.getConnection(this.sqlUrl, this.sqlUser, this.sqlPasswd); - if (connObj == null) { - throw new ConnectionException("Connection is null"); - } - statement = connObj.createStatement(); - statement.executeUpdate(sqlStatement); - - } finally { - - if (statement != null) { - statement.close(); - } - if (connObj != null) { - connObj.close(); - } - } - } -} diff --git a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/transforms/SourceWriterFnTest.java b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/transforms/SourceWriterFnTest.java index f695d67370..990e190a04 100644 --- a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/transforms/SourceWriterFnTest.java +++ b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/transforms/SourceWriterFnTest.java @@ -42,7 +42,7 @@ import com.google.cloud.teleport.v2.templates.changestream.ChangeStreamErrorRecord; import com.google.cloud.teleport.v2.templates.changestream.TrimmedShardedDataChangeRecord; import com.google.cloud.teleport.v2.templates.constants.Constants; -import com.google.cloud.teleport.v2.templates.utils.MySqlDao; +import com.google.cloud.teleport.v2.templates.mysql.MySqlDao; import com.google.cloud.teleport.v2.templates.utils.ShadowTableRecord; import com.google.cloud.teleport.v2.templates.utils.SpannerDao; import com.google.common.collect.ImmutableList; diff --git a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/utils/MySqlDaoTest.java b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/utils/MySqlDaoTest.java index 0d6754e005..e6c4022218 100644 --- a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/utils/MySqlDaoTest.java +++ b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/utils/MySqlDaoTest.java @@ -21,6 +21,7 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import com.google.cloud.teleport.v2.templates.mysql.MySqlDao; import com.zaxxer.hikari.HikariDataSource; import java.sql.Connection; import java.sql.Statement; From 6d88d89f3a63e715ffe472f0ededeb402656e064 Mon Sep 17 00:00:00 2001 From: Shreya Khajanchi Date: Mon, 4 Nov 2024 10:25:24 +0530 Subject: [PATCH 02/13] minor changes --- .../v2/templates/transforms/SourceWriterFn.java | 4 ++-- .../teleport/v2/templates/utils/ISourceDao.java | 12 +++++------- .../v2/templates/utils/InputRecordProcessor.java | 2 +- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/transforms/SourceWriterFn.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/transforms/SourceWriterFn.java index fb8a857ab1..7f8720dc4c 100644 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/transforms/SourceWriterFn.java +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/transforms/SourceWriterFn.java @@ -182,11 +182,11 @@ public void processElement(ProcessContext c) { > Long.parseLong(spannerRec.getRecordSequence()))); if (!isSourceAhead) { - ISourceDao mySqlDao = sourceDaoMap.get(shardId); + ISourceDao sourceDao = sourceDaoMap.get(shardId); InputRecordProcessor inputRecordProcessor = new InputRecordProcessor(); inputRecordProcessor.processRecord( - spannerRec, schema, mySqlDao, shardId, sourceDbTimezoneOffset); + spannerRec, schema, sourceDao, shardId, sourceDbTimezoneOffset); spannerDao.updateShadowTable( getShadowTableMutation( diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/ISourceDao.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/ISourceDao.java index 08974b961f..445210fff3 100644 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/ISourceDao.java +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/ISourceDao.java @@ -1,14 +1,12 @@ package com.google.cloud.teleport.v2.templates.utils; -import java.sql.SQLException; - public interface ISourceDao { /** * Initializes the DAO with the necessary connection parameters. * - * @param url Connection URL. - * @param user Database user. + * @param url Connection URL. + * @param user Database user. * @param password User password. */ void initialize(String url, String user, String password) throws Exception; @@ -17,13 +15,13 @@ public interface ISourceDao { * Executes a given write statement. * * @param statement The SQL or query statement. - * @throws SQLException If there is an error executing the statement. + * @throws Exception If there is an error executing the statement. */ - void write(String statement) throws SQLException; + void write(String statement) throws Exception; /** * Closes any open connections. */ - void close() throws SQLException; + void close() throws Exception; } diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/InputRecordProcessor.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/InputRecordProcessor.java index 43cd9d7332..d93a3c5508 100644 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/InputRecordProcessor.java +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/InputRecordProcessor.java @@ -40,7 +40,7 @@ public void processRecord( ISourceDao dao, String shardId, String sourceDbTimezoneOffset) - throws java.sql.SQLException, ConnectionException { + throws Exception { try { From 171d33c8f29ccee8968680a4e3f921a0ddc5ec9a Mon Sep 17 00:00:00 2001 From: Shreya Khajanchi Date: Sun, 10 Nov 2024 16:41:11 +0530 Subject: [PATCH 03/13] refactoring code --- .../v2/templates/SpannerToSourceDb.java | 25 +- .../templates/common/DMLGeneratorRequest.java | 67 ++++++ .../v2/templates/common/IDMLGenerator.java | 20 ++ .../v2/templates/common/ISourceDao.java | 27 +++ .../common/SourceProcessorFactory.java | 51 ++++ .../v2/templates/common/package-info.java | 18 ++ .../v2/templates/constants/Constants.java | 5 + .../MySQLConnectionHelper.java} | 6 +- .../MySQLDMLGenerator.java} | 65 +++-- .../teleport/v2/templates/mysql/MySqlDao.java | 50 ++-- .../v2/templates/mysql/package-info.java | 18 ++ .../templates/transforms/SourceWriterFn.java | 31 +-- .../transforms/SourceWriterTransform.java | 12 +- .../v2/templates/utils/ISourceDao.java | 27 --- .../utils/ISourceProcessorFactory.java | 5 - .../templates/utils/InputRecordProcessor.java | 19 +- .../common/DMLGeneratorRequestTest.java | 65 +++++ .../common/SourceProcessorFactoryTest.java | 103 ++++++++ .../MySQLDMLGeneratorTest.java} | 225 ++++++++++++------ .../{utils => mysql}/MySqlDaoTest.java | 14 +- .../transforms/SourceWriterFnTest.java | 77 ++++-- 21 files changed, 713 insertions(+), 217 deletions(-) create mode 100644 v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/common/DMLGeneratorRequest.java create mode 100644 v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/common/IDMLGenerator.java create mode 100644 v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/common/ISourceDao.java create mode 100644 v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/common/SourceProcessorFactory.java create mode 100644 v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/common/package-info.java rename v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/{utils/ConnectionHelper.java => mysql/MySQLConnectionHelper.java} (94%) rename v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/{utils/DMLGenerator.java => mysql/MySQLDMLGenerator.java} (86%) create mode 100644 v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/mysql/package-info.java delete mode 100644 v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/ISourceDao.java delete mode 100644 v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/ISourceProcessorFactory.java create mode 100644 v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/common/DMLGeneratorRequestTest.java create mode 100644 v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/common/SourceProcessorFactoryTest.java rename v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/{utils/DMLGeneratorTest.java => mysql/MySQLDMLGeneratorTest.java} (80%) rename v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/{utils => mysql}/MySqlDaoTest.java (86%) diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/SpannerToSourceDb.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/SpannerToSourceDb.java index f02e9321a5..8d65463452 100644 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/SpannerToSourceDb.java +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/SpannerToSourceDb.java @@ -356,6 +356,16 @@ public interface Options extends PipelineOptions, StreamingOptions { Integer getDlqRetryMinutes(); void setDlqRetryMinutes(Integer value); + + @TemplateParameter.Text( + order = 24, + optional = true, + description = "Source database type, ex: mysql", + helpText = "The type of source database to reverse replicate to.") + @Default.String("mysql") + String getSourceType(); + + void setSourceType(String value); } /** @@ -454,9 +464,14 @@ public static PipelineResult run(Options options) { Ddl ddl = SpannerSchema.getInformationSchemaAsDdl(spannerConfig); ShardFileReader shardFileReader = new ShardFileReader(new SecretManagerAccessorImpl()); List shards = shardFileReader.getOrderedShardDetails(options.getSourceShardsFilePath()); - String shardingMode = Constants.SHARDING_MODE_SINGLE_SHARD; - if (shards.size() > 1) { - shardingMode = Constants.SHARDING_MODE_MULTI_SHARD; + String shardingMode = Constants.SHARDING_MODE_MULTI_SHARD; + if (shards.size() == 1) { + shardingMode = Constants.SHARDING_MODE_SINGLE_SHARD; + + Shard singleShard = shards.get(0); + if (singleShard.getLogicalShardId() == null) { + singleShard.setLogicalShardId(Constants.DEFAULT_SHARD_ID); + } } boolean isRegularMode = "regular".equals(options.getRunMode()); PCollectionTuple reconsumedElements = null; @@ -559,7 +574,9 @@ public static PipelineResult run(Options options) { ddl, options.getShadowTablePrefix(), options.getSkipDirectoryName(), - connectionPoolSizePerWorker)); + connectionPoolSizePerWorker, + options.getSourceType(), + shardingMode)); PCollection> dlqPermErrorRecords = reconsumedElements diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/common/DMLGeneratorRequest.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/common/DMLGeneratorRequest.java new file mode 100644 index 0000000000..73716a1c5c --- /dev/null +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/common/DMLGeneratorRequest.java @@ -0,0 +1,67 @@ +/* + * Copyright (C) 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.templates.common; + +import com.google.cloud.teleport.v2.spanner.migrations.schema.Schema; +import org.json.JSONObject; + +public class DMLGeneratorRequest { + private String modType; + private String spannerTableName; + private Schema schema; + private JSONObject newValuesJson; + private JSONObject keyValuesJson; + private String sourceDbTimezoneOffset; + + public String getModType() { + return modType; + } + + public String getSpannerTableName() { + return spannerTableName; + } + + public Schema getSchema() { + return schema; + } + + public JSONObject getNewValuesJson() { + return newValuesJson; + } + + public JSONObject getKeyValuesJson() { + return keyValuesJson; + } + + public String getSourceDbTimezoneOffset() { + return sourceDbTimezoneOffset; + } + + public DMLGeneratorRequest( + String modType, + String spannerTableName, + Schema schema, + JSONObject newValuesJson, + JSONObject keyValuesJson, + String sourceDbTimezoneOffset) { + this.modType = modType; + this.spannerTableName = spannerTableName; + this.schema = schema; + this.newValuesJson = newValuesJson; + this.keyValuesJson = keyValuesJson; + this.sourceDbTimezoneOffset = sourceDbTimezoneOffset; + } +} diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/common/IDMLGenerator.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/common/IDMLGenerator.java new file mode 100644 index 0000000000..10f32ebc2c --- /dev/null +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/common/IDMLGenerator.java @@ -0,0 +1,20 @@ +/* + * Copyright (C) 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.templates.common; + +public interface IDMLGenerator { + String getDMLStatement(DMLGeneratorRequest dmlGeneratorRequest); +} diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/common/ISourceDao.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/common/ISourceDao.java new file mode 100644 index 0000000000..864cef9fca --- /dev/null +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/common/ISourceDao.java @@ -0,0 +1,27 @@ +/* + * Copyright (C) 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.templates.common; + +public interface ISourceDao { + + /** + * Executes a given write statement. + * + * @param statement The SQL or query statement. + * @throws Exception If there is an error executing the statement. + */ + void write(String statement) throws Exception; +} diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/common/SourceProcessorFactory.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/common/SourceProcessorFactory.java new file mode 100644 index 0000000000..f5c306615f --- /dev/null +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/common/SourceProcessorFactory.java @@ -0,0 +1,51 @@ +/* + * Copyright (C) 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.templates.common; + +import com.google.cloud.teleport.v2.spanner.migrations.shard.Shard; +import com.google.cloud.teleport.v2.templates.constants.Constants; +import com.google.cloud.teleport.v2.templates.mysql.MySQLConnectionHelper; +import com.google.cloud.teleport.v2.templates.mysql.MySQLDMLGenerator; +import com.google.cloud.teleport.v2.templates.mysql.MySqlDao; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class SourceProcessorFactory { + public static IDMLGenerator getDMLGenerator(String source) throws Exception { + if (source.equalsIgnoreCase(Constants.SOURCE_MYSQL)) { + return new MySQLDMLGenerator(); + } + throw new Exception("Invalid source type: " + source); + } + + public static Map getSourceDaoMap( + String source, List shards, int maxConnections) throws Exception { + if (source.equalsIgnoreCase(Constants.SOURCE_MYSQL)) { + Map sourceDaoMap = new HashMap<>(); + MySQLConnectionHelper.init(shards, null, maxConnections); + for (Shard shard : shards) { + String sourceConnectionUrl = + "jdbc:mysql://" + shard.getHost() + ":" + shard.getPort() + "/" + shard.getDbName(); + ISourceDao mySqlDao = + new MySqlDao(sourceConnectionUrl, shard.getUserName(), shard.getPassword()); + sourceDaoMap.put(shard.getLogicalShardId(), mySqlDao); + } + return sourceDaoMap; + } + throw new Exception("Invalid source type: " + source); + } +} diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/common/package-info.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/common/package-info.java new file mode 100644 index 0000000000..14cd579bfb --- /dev/null +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/common/package-info.java @@ -0,0 +1,18 @@ +/* + * Copyright (C) 2023 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +/** Package info for common module. */ +package com.google.cloud.teleport.v2.templates.common; diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/constants/Constants.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/constants/Constants.java index 8951aa8e03..31ed2b8428 100644 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/constants/Constants.java +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/constants/Constants.java @@ -67,4 +67,9 @@ public class Constants { // Default parallelism for the Dataflow workers public static final int DEFAULT_WORKER_HARNESS_THREAD_COUNT = 500; + + // Default shard id in case of single shard migration with logical shard id unspecified + public static final String DEFAULT_SHARD_ID = "single_shard"; + + public static final String SOURCE_MYSQL = "mysql"; } diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/ConnectionHelper.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/mysql/MySQLConnectionHelper.java similarity index 94% rename from v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/ConnectionHelper.java rename to v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/mysql/MySQLConnectionHelper.java index 188a4325d1..61c2c9094a 100644 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/ConnectionHelper.java +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/mysql/MySQLConnectionHelper.java @@ -13,7 +13,7 @@ * License for the specific language governing permissions and limitations under * the License. */ -package com.google.cloud.teleport.v2.templates.utils; +package com.google.cloud.teleport.v2.templates.mysql; import com.google.cloud.teleport.v2.spanner.migrations.shard.Shard; import com.zaxxer.hikari.HikariConfig; @@ -29,9 +29,9 @@ import org.slf4j.LoggerFactory; /** This is a per Dataflow worker singleton that holds connection pool. */ -public class ConnectionHelper { +public class MySQLConnectionHelper { - private static final Logger LOG = LoggerFactory.getLogger(ConnectionHelper.class); + private static final Logger LOG = LoggerFactory.getLogger(MySQLConnectionHelper.class); private static final String JDBC_DRIVER = "com.mysql.cj.jdbc.Driver"; private static Map connectionPoolMap = null; diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/DMLGenerator.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/mysql/MySQLDMLGenerator.java similarity index 86% rename from v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/DMLGenerator.java rename to v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/mysql/MySQLDMLGenerator.java index 44335f8733..90d62dedc8 100644 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/DMLGenerator.java +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/mysql/MySQLDMLGenerator.java @@ -13,14 +13,15 @@ * License for the specific language governing permissions and limitations under * the License. */ -package com.google.cloud.teleport.v2.templates.utils; +package com.google.cloud.teleport.v2.templates.mysql; import com.google.cloud.teleport.v2.spanner.migrations.schema.ColumnPK; -import com.google.cloud.teleport.v2.spanner.migrations.schema.Schema; import com.google.cloud.teleport.v2.spanner.migrations.schema.SourceColumnDefinition; import com.google.cloud.teleport.v2.spanner.migrations.schema.SourceTable; import com.google.cloud.teleport.v2.spanner.migrations.schema.SpannerColumnDefinition; import com.google.cloud.teleport.v2.spanner.migrations.schema.SpannerTable; +import com.google.cloud.teleport.v2.templates.common.DMLGeneratorRequest; +import com.google.cloud.teleport.v2.templates.common.IDMLGenerator; import java.util.HashMap; import java.util.Map; import java.util.Set; @@ -31,37 +32,40 @@ import org.slf4j.LoggerFactory; /** Creates DML statements. */ -public class DMLGenerator { - private static final Logger LOG = LoggerFactory.getLogger(DMLGenerator.class); +public class MySQLDMLGenerator implements IDMLGenerator { + private static final Logger LOG = LoggerFactory.getLogger(MySQLDMLGenerator.class); - public static String getDMLStatement( - String modType, - String spannerTableName, - Schema schema, - JSONObject newValuesJson, - JSONObject keyValuesJson, - String sourceDbTimezoneOffset) { + public String getDMLStatement(DMLGeneratorRequest dmlGeneratorRequest) { - if (schema.getSpannerToID().get(spannerTableName) == null) { + if (dmlGeneratorRequest + .getSchema() + .getSpannerToID() + .get(dmlGeneratorRequest.getSpannerTableName()) + == null) { LOG.warn( "The spanner table {} was not found in session file, dropping the record", - spannerTableName); + dmlGeneratorRequest.getSpannerTableName()); return ""; } - String spannerTableId = schema.getSpannerToID().get(spannerTableName).getName(); - SpannerTable spannerTable = schema.getSpSchema().get(spannerTableId); + String spannerTableId = + dmlGeneratorRequest + .getSchema() + .getSpannerToID() + .get(dmlGeneratorRequest.getSpannerTableName()) + .getName(); + SpannerTable spannerTable = dmlGeneratorRequest.getSchema().getSpSchema().get(spannerTableId); if (spannerTable == null) { LOG.warn( "The spanner table {} was not found in session file, dropping the record", - spannerTableName); + dmlGeneratorRequest.getSpannerTableName()); return ""; } - SourceTable sourceTable = schema.getSrcSchema().get(spannerTableId); + SourceTable sourceTable = dmlGeneratorRequest.getSchema().getSrcSchema().get(spannerTableId); if (sourceTable == null) { - LOG.warn("The table {} was not found in source", spannerTableName); + LOG.warn("The table {} was not found in source", dmlGeneratorRequest.getSpannerTableName()); return ""; } @@ -72,10 +76,15 @@ public static String getDMLStatement( return ""; } - if ("INSERT".equals(modType) || "UPDATE".equals(modType)) { + if ("INSERT".equals(dmlGeneratorRequest.getModType()) + || "UPDATE".equals(dmlGeneratorRequest.getModType())) { Map pkcolumnNameValues = getPkColumnValues( - spannerTable, sourceTable, newValuesJson, keyValuesJson, sourceDbTimezoneOffset); + spannerTable, + sourceTable, + dmlGeneratorRequest.getNewValuesJson(), + dmlGeneratorRequest.getKeyValuesJson(), + dmlGeneratorRequest.getSourceDbTimezoneOffset()); if (pkcolumnNameValues == null) { LOG.warn( "Cannot reverse replicate for table {} without primary key, skipping the record", @@ -84,17 +93,25 @@ public static String getDMLStatement( } Map columnNameValues = getColumnValues( - spannerTable, sourceTable, newValuesJson, keyValuesJson, sourceDbTimezoneOffset); + spannerTable, + sourceTable, + dmlGeneratorRequest.getNewValuesJson(), + dmlGeneratorRequest.getKeyValuesJson(), + dmlGeneratorRequest.getSourceDbTimezoneOffset()); return getUpsertStatement( sourceTable.getName(), sourceTable.getPrimaryKeySet(), columnNameValues, pkcolumnNameValues); - } else if ("DELETE".equals(modType)) { + } else if ("DELETE".equals(dmlGeneratorRequest.getModType())) { Map pkcolumnNameValues = getPkColumnValues( - spannerTable, sourceTable, newValuesJson, keyValuesJson, sourceDbTimezoneOffset); + spannerTable, + sourceTable, + dmlGeneratorRequest.getNewValuesJson(), + dmlGeneratorRequest.getKeyValuesJson(), + dmlGeneratorRequest.getSourceDbTimezoneOffset()); if (pkcolumnNameValues == null) { LOG.warn( "Cannot reverse replicate for table {} without primary key, skipping the record", @@ -103,7 +120,7 @@ public static String getDMLStatement( } return getDeleteStatement(sourceTable.getName(), pkcolumnNameValues); } else { - LOG.warn("Unsupported modType: " + modType); + LOG.warn("Unsupported modType: " + dmlGeneratorRequest.getModType()); return ""; } } diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/mysql/MySqlDao.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/mysql/MySqlDao.java index f42ed3554c..ae2b9edbe1 100644 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/mysql/MySqlDao.java +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/mysql/MySqlDao.java @@ -15,38 +15,48 @@ */ package com.google.cloud.teleport.v2.templates.mysql; +import com.google.cloud.teleport.v2.templates.common.ISourceDao; +import com.google.cloud.teleport.v2.templates.utils.ConnectionException; import java.sql.Connection; import java.sql.SQLException; import java.sql.Statement; -import com.google.cloud.teleport.v2.templates.utils.ConnectionHelper; -import com.google.cloud.teleport.v2.templates.utils.ISourceDao; public class MySqlDao implements ISourceDao { private String sqlUrl; private String sqlUser; private String sqlPasswd; - private Connection connObj; - - @Override - public void initialize(String url, String user, String password) throws Exception { - this.sqlUrl = url; - this.sqlUser = user; - this.sqlPasswd = password; - Class.forName("com.mysql.cj.jdbc.Driver"); - this.connObj = ConnectionHelper.getConnection(this.sqlUrl, this.sqlUser, this.sqlPasswd); + + public MySqlDao(String sqlUrl, String sqlUser, String sqlPasswd) { + this.sqlUrl = sqlUrl; + this.sqlUser = sqlUser; + this.sqlPasswd = sqlPasswd; } - @Override - public void write(String sqlStatement) throws SQLException { - try (Statement statement = connObj.createStatement()) { - statement.executeUpdate(sqlStatement); - } + public String getSourceConnectionUrl() { + return sqlUrl; } - @Override - public void close() throws SQLException { - if (connObj != null && !connObj.isClosed()) { - connObj.close(); + public void write(String sqlStatement) throws SQLException, ConnectionException { + Connection connObj = null; + Statement statement = null; + + try { + + connObj = MySQLConnectionHelper.getConnection(this.sqlUrl, this.sqlUser, this.sqlPasswd); + if (connObj == null) { + throw new ConnectionException("Connection is null"); + } + statement = connObj.createStatement(); + statement.executeUpdate(sqlStatement); + + } finally { + + if (statement != null) { + statement.close(); + } + if (connObj != null) { + connObj.close(); + } } } } diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/mysql/package-info.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/mysql/package-info.java new file mode 100644 index 0000000000..6b4f0082dd --- /dev/null +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/mysql/package-info.java @@ -0,0 +1,18 @@ +/* + * Copyright (C) 2023 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +/** Package info for mysql source module. */ +package com.google.cloud.teleport.v2.templates.mysql; diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/transforms/SourceWriterFn.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/transforms/SourceWriterFn.java index 7f8720dc4c..db74e38955 100644 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/transforms/SourceWriterFn.java +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/transforms/SourceWriterFn.java @@ -29,9 +29,13 @@ import com.google.cloud.teleport.v2.spanner.migrations.shard.Shard; import com.google.cloud.teleport.v2.templates.changestream.ChangeStreamErrorRecord; import com.google.cloud.teleport.v2.templates.changestream.TrimmedShardedDataChangeRecord; +import com.google.cloud.teleport.v2.templates.common.ISourceDao; +import com.google.cloud.teleport.v2.templates.common.SourceProcessorFactory; import com.google.cloud.teleport.v2.templates.constants.Constants; -import com.google.cloud.teleport.v2.templates.utils.*; -import com.google.cloud.teleport.v2.templates.mysql.MySqlDao; +import com.google.cloud.teleport.v2.templates.utils.ConnectionException; +import com.google.cloud.teleport.v2.templates.utils.InputRecordProcessor; +import com.google.cloud.teleport.v2.templates.utils.ShadowTableRecord; +import com.google.cloud.teleport.v2.templates.utils.SpannerDao; import com.google.common.collect.ImmutableList; import com.google.gson.Gson; import java.io.Serializable; @@ -81,6 +85,8 @@ public class SourceWriterFn extends DoFn shards, @@ -90,7 +96,9 @@ public SourceWriterFn( Ddl ddl, String shadowTablePrefix, String skipDirName, - int maxThreadPerDataflowWorker) { + int maxThreadPerDataflowWorker, + String source, + String shardingMode) { this.schema = schema; this.sourceDbTimezoneOffset = sourceDbTimezoneOffset; @@ -100,6 +108,8 @@ public SourceWriterFn( this.shadowTablePrefix = shadowTablePrefix; this.skipDirName = skipDirName; this.maxThreadPerDataflowWorker = maxThreadPerDataflowWorker; + this.source = source; + this.shardingMode = shardingMode; } // for unit testing purposes @@ -122,15 +132,8 @@ public void setObjectMapper(ObjectMapper mapper) { public void setup() throws Exception { mapper = new ObjectMapper(); mapper.enable(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS); - ConnectionHelper.init(shards, null, maxThreadPerDataflowWorker); - sourceDaoMap = new HashMap<>(); - for (Shard shard : shards) { - String sourceConnectionUrl = - "jdbc:mysql://" + shard.getHost() + ":" + shard.getPort() + "/" + shard.getDbName(); - ISourceDao mySqlDao = new MySqlDao(); - mySqlDao.initialize(sourceConnectionUrl, shard.getUserName(), shard.getPassword()); - sourceDaoMap.put(shard.getLogicalShardId(), mySqlDao); - } + sourceDaoMap = + SourceProcessorFactory.getSourceDaoMap(source, shards, maxThreadPerDataflowWorker); spannerDao = new SpannerDao(spannerConfig); } @@ -150,7 +153,7 @@ public void processElement(ProcessContext c) { // no shard found, move to permanent error outputWithTag( c, Constants.PERMANENT_ERROR_TAG, Constants.SHARD_NOT_PRESENT_ERROR_MESSAGE, spannerRec); - } else if (shardId.equals(skipDirName)) { + } else if (shardId != null && shardId.equals(skipDirName)) { // the record is skipped skippedRecordCountMetric.inc(); outputWithTag(c, Constants.SKIPPED_TAG, Constants.SKIPPED_TAG_MESSAGE, spannerRec); @@ -186,7 +189,7 @@ public void processElement(ProcessContext c) { InputRecordProcessor inputRecordProcessor = new InputRecordProcessor(); inputRecordProcessor.processRecord( - spannerRec, schema, sourceDao, shardId, sourceDbTimezoneOffset); + spannerRec, schema, sourceDao, shardId, sourceDbTimezoneOffset, source); spannerDao.updateShadowTable( getShadowTableMutation( diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/transforms/SourceWriterTransform.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/transforms/SourceWriterTransform.java index 2837fe62cc..56429a46b6 100644 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/transforms/SourceWriterTransform.java +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/transforms/SourceWriterTransform.java @@ -51,6 +51,8 @@ public class SourceWriterTransform private final String shadowTablePrefix; private final String skipDirName; private final int maxThreadPerDataflowWorker; + private final String source; + private final String shardingMode; public SourceWriterTransform( List shards, @@ -60,7 +62,9 @@ public SourceWriterTransform( Ddl ddl, String shadowTablePrefix, String skipDirName, - int maxThreadPerDataflowWorker) { + int maxThreadPerDataflowWorker, + String source, + String shardingMode) { this.schema = schema; this.sourceDbTimezoneOffset = sourceDbTimezoneOffset; @@ -70,6 +74,8 @@ public SourceWriterTransform( this.shadowTablePrefix = shadowTablePrefix; this.skipDirName = skipDirName; this.maxThreadPerDataflowWorker = maxThreadPerDataflowWorker; + this.source = source; + this.shardingMode = shardingMode; } @Override @@ -87,7 +93,9 @@ public SourceWriterTransform.Result expand( this.ddl, this.shadowTablePrefix, this.skipDirName, - this.maxThreadPerDataflowWorker)) + this.maxThreadPerDataflowWorker, + this.source, + this.shardingMode)) .withOutputTags( Constants.SUCCESS_TAG, TupleTagList.of(Constants.PERMANENT_ERROR_TAG) diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/ISourceDao.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/ISourceDao.java deleted file mode 100644 index 445210fff3..0000000000 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/ISourceDao.java +++ /dev/null @@ -1,27 +0,0 @@ -package com.google.cloud.teleport.v2.templates.utils; - -public interface ISourceDao { - - /** - * Initializes the DAO with the necessary connection parameters. - * - * @param url Connection URL. - * @param user Database user. - * @param password User password. - */ - void initialize(String url, String user, String password) throws Exception; - - /** - * Executes a given write statement. - * - * @param statement The SQL or query statement. - * @throws Exception If there is an error executing the statement. - */ - void write(String statement) throws Exception; - - /** - * Closes any open connections. - */ - void close() throws Exception; -} - diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/ISourceProcessorFactory.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/ISourceProcessorFactory.java deleted file mode 100644 index daab702cfc..0000000000 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/ISourceProcessorFactory.java +++ /dev/null @@ -1,5 +0,0 @@ -package com.google.cloud.teleport.v2.templates.utils; - -public interface ISourceProcessorFactory { - -} diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/InputRecordProcessor.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/InputRecordProcessor.java index d93a3c5508..2f82023689 100644 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/InputRecordProcessor.java +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/InputRecordProcessor.java @@ -17,10 +17,12 @@ import com.google.cloud.teleport.v2.spanner.migrations.schema.Schema; import com.google.cloud.teleport.v2.templates.changestream.TrimmedShardedDataChangeRecord; +import com.google.cloud.teleport.v2.templates.common.DMLGeneratorRequest; +import com.google.cloud.teleport.v2.templates.common.IDMLGenerator; +import com.google.cloud.teleport.v2.templates.common.ISourceDao; +import com.google.cloud.teleport.v2.templates.common.SourceProcessorFactory; import java.time.Instant; import java.time.temporal.ChronoUnit; - -import com.google.cloud.teleport.v2.templates.mysql.MySqlDao; import org.apache.beam.sdk.metrics.Counter; import org.apache.beam.sdk.metrics.Distribution; import org.apache.beam.sdk.metrics.Metrics; @@ -39,8 +41,9 @@ public void processRecord( Schema schema, ISourceDao dao, String shardId, - String sourceDbTimezoneOffset) - throws Exception { + String sourceDbTimezoneOffset, + String source) + throws Exception { try { @@ -51,14 +54,16 @@ public void processRecord( JSONObject newValuesJson = new JSONObject(newValueJsonStr); JSONObject keysJson = new JSONObject(keysJsonStr); - String dmlStatement = - DMLGenerator.getDMLStatement( + DMLGeneratorRequest dmlGeneratorRequest = + new DMLGeneratorRequest( modType, tableName, schema, newValuesJson, keysJson, sourceDbTimezoneOffset); + + IDMLGenerator dmlGenerator = SourceProcessorFactory.getDMLGenerator(source); + String dmlStatement = dmlGenerator.getDMLStatement(dmlGeneratorRequest); if (dmlStatement.isEmpty()) { LOG.warn("DML statement is empty for table: " + tableName); return; } - dao.write(dmlStatement); Counter numRecProcessedMetric = diff --git a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/common/DMLGeneratorRequestTest.java b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/common/DMLGeneratorRequestTest.java new file mode 100644 index 0000000000..14e7b01255 --- /dev/null +++ b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/common/DMLGeneratorRequestTest.java @@ -0,0 +1,65 @@ +/* + * Copyright (C) 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.templates.common; + +import static org.junit.Assert.assertEquals; +import static org.mockito.Mockito.mock; + +import com.google.cloud.teleport.v2.spanner.migrations.schema.Schema; +import org.json.JSONObject; +import org.junit.Before; +import org.junit.Test; + +public class DMLGeneratorRequestTest { + private String modType; + private String spannerTableName; + private Schema schema; + private JSONObject newValuesJson; + private JSONObject keyValuesJson; + private String sourceDbTimezoneOffset; + + @Before + public void setUp() { + // Initialize mock data + modType = "INSERT"; + spannerTableName = "my_table"; + schema = mock(Schema.class); + newValuesJson = new JSONObject(); + newValuesJson.put("column1", "value1"); + keyValuesJson = new JSONObject(); + keyValuesJson.put("column1", "key1"); + sourceDbTimezoneOffset = "+00:00"; + } + + @Test + public void testConstructorAndGetters() { + DMLGeneratorRequest request = + new DMLGeneratorRequest( + modType, + spannerTableName, + schema, + newValuesJson, + keyValuesJson, + sourceDbTimezoneOffset); + + assertEquals(modType, request.getModType()); + assertEquals(spannerTableName, request.getSpannerTableName()); + assertEquals(schema, request.getSchema()); + assertEquals(newValuesJson.toString(), request.getNewValuesJson().toString()); + assertEquals(keyValuesJson.toString(), request.getKeyValuesJson().toString()); + assertEquals(sourceDbTimezoneOffset, request.getSourceDbTimezoneOffset()); + } +} diff --git a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/common/SourceProcessorFactoryTest.java b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/common/SourceProcessorFactoryTest.java new file mode 100644 index 0000000000..19f752a1bd --- /dev/null +++ b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/common/SourceProcessorFactoryTest.java @@ -0,0 +1,103 @@ +/* + * Copyright (C) 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.templates.common; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import com.google.cloud.teleport.v2.spanner.migrations.shard.Shard; +import com.google.cloud.teleport.v2.templates.constants.Constants; +import com.google.cloud.teleport.v2.templates.mysql.MySQLDMLGenerator; +import com.google.cloud.teleport.v2.templates.mysql.MySqlDao; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +public class SourceProcessorFactoryTest { + + @Rule public ExpectedException expectedEx = ExpectedException.none(); + + @Test + public void testGetDMLGenerator_MySQLSource() throws Exception { + IDMLGenerator dmlGenerator = SourceProcessorFactory.getDMLGenerator(Constants.SOURCE_MYSQL); + assertNotNull(dmlGenerator); + assertTrue(dmlGenerator instanceof MySQLDMLGenerator); + } + + @Test + public void testGetDMLGenerator_InvalidSource() throws Exception { + expectedEx.expect(Exception.class); + expectedEx.expectMessage("Invalid source type: invalidSource"); + SourceProcessorFactory.getDMLGenerator("invalidSource"); + } + + @Test + public void testGetSourceDaoMap_MySQLSource() throws Exception { + Shard shard1 = mock(Shard.class); + when(shard1.getHost()).thenReturn("localhost"); + when(shard1.getPort()).thenReturn("3306"); + when(shard1.getDbName()).thenReturn("test_db"); + when(shard1.getLogicalShardId()).thenReturn("shard1"); + when(shard1.getUserName()).thenReturn("root"); + when(shard1.getPassword()).thenReturn("password"); + + Shard shard2 = mock(Shard.class); + when(shard2.getHost()).thenReturn("localhost"); + when(shard2.getPort()).thenReturn("3307"); + when(shard2.getDbName()).thenReturn("test_db_2"); + when(shard2.getLogicalShardId()).thenReturn("shard2"); + when(shard2.getUserName()).thenReturn("root"); + when(shard2.getPassword()).thenReturn("password"); + + List shards = Arrays.asList(shard1, shard2); + + Map sourceDaoMap = + SourceProcessorFactory.getSourceDaoMap(Constants.SOURCE_MYSQL, shards, 10); + + assertNotNull(sourceDaoMap); + assertEquals(2, sourceDaoMap.size()); + + ISourceDao mySqlDao1 = sourceDaoMap.get("shard1"); + assertTrue(mySqlDao1 instanceof MySqlDao); + assertEquals( + "jdbc:mysql://localhost:3306/test_db", ((MySqlDao) mySqlDao1).getSourceConnectionUrl()); + + ISourceDao mySqlDao2 = sourceDaoMap.get("shard2"); + assertTrue(mySqlDao2 instanceof MySqlDao); + assertEquals( + "jdbc:mysql://localhost:3307/test_db_2", ((MySqlDao) mySqlDao2).getSourceConnectionUrl()); + } + + @Test + public void testGetSourceDaoMap_InvalidSource() throws Exception { + expectedEx.expect(Exception.class); + expectedEx.expectMessage("Invalid source type: invalidSource"); + List shards = Arrays.asList(mock(Shard.class), mock(Shard.class)); + SourceProcessorFactory.getSourceDaoMap("invalidSource", shards, 10); + } + + @Test + public void testGetSourceDaoMap_NullShards() throws Exception { + expectedEx.expect(NullPointerException.class); + SourceProcessorFactory.getSourceDaoMap(Constants.SOURCE_MYSQL, null, 10); + } +} diff --git a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/utils/DMLGeneratorTest.java b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/mysql/MySQLDMLGeneratorTest.java similarity index 80% rename from v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/utils/DMLGeneratorTest.java rename to v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/mysql/MySQLDMLGeneratorTest.java index 1b0f415b75..100d4df37d 100644 --- a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/utils/DMLGeneratorTest.java +++ b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/mysql/MySQLDMLGeneratorTest.java @@ -13,7 +13,7 @@ * License for the specific language governing permissions and limitations under * the License. */ -package com.google.cloud.teleport.v2.templates.utils; +package com.google.cloud.teleport.v2.templates.mysql; import static org.junit.Assert.assertTrue; @@ -29,6 +29,8 @@ import com.google.cloud.teleport.v2.spanner.migrations.schema.SyntheticPKey; import com.google.cloud.teleport.v2.spanner.migrations.utils.SessionFileReader; import com.google.cloud.teleport.v2.templates.changestream.TrimmedShardedDataChangeRecord; +import com.google.cloud.teleport.v2.templates.common.DMLGeneratorRequest; +import com.google.cloud.teleport.v2.templates.utils.InputRecordProcessor; import com.google.gson.FieldNamingPolicy; import com.google.gson.GsonBuilder; import java.io.InputStream; @@ -44,7 +46,7 @@ import org.junit.runners.JUnit4; @RunWith(JUnit4.class) -public final class DMLGeneratorTest { +public final class MySQLDMLGeneratorTest { @Test public void tableAndAllColumnNameTypesMatch() { @@ -59,9 +61,11 @@ public void tableAndAllColumnNameTypesMatch() { /*The expected sql is: "INSERT INTO Singers(SingerId,FirstName,LastName) VALUES (999,'kk','ll') ON DUPLICATE KEY" + " UPDATE FirstName = 'kk', LastName = 'll'";*/ + MySQLDMLGenerator mySQLDMLGenerator = new MySQLDMLGenerator(); String sql = - DMLGenerator.getDMLStatement( - modType, tableName, schema, newValuesJson, keyValuesJson, "+00:00"); + mySQLDMLGenerator.getDMLStatement( + new DMLGeneratorRequest( + modType, tableName, schema, newValuesJson, keyValuesJson, "+00:00")); assertTrue(sql.contains("FirstName = 'kk'")); assertTrue(sql.contains("LastName = 'll'")); @@ -80,9 +84,11 @@ public void tableNameMismatchAllColumnNameTypesMatch() { /* The expected sql is: "INSERT INTO Singers(SingerId,FirstName,LastName) VALUES (999,'kk','ll') ON DUPLICATE KEY" + " UPDATE FirstName = 'kk', LastName = 'll'";*/ + MySQLDMLGenerator mySQLDMLGenerator = new MySQLDMLGenerator(); String sql = - DMLGenerator.getDMLStatement( - modType, tableName, schema, newValuesJson, keyValuesJson, "+00:00"); + mySQLDMLGenerator.getDMLStatement( + new DMLGeneratorRequest( + modType, tableName, schema, newValuesJson, keyValuesJson, "+00:00")); assertTrue(sql.contains("FirstName = 'kk'")); assertTrue(sql.contains("LastName = 'll'")); @@ -100,9 +106,11 @@ public void tableNameMatchColumnNameTypeMismatch() { /*The expected sql is: "INSERT INTO Singers(SingerId,FirstName,LastName) VALUES ('999',222,'ll') ON DUPLICATE" + " KEY UPDATE FirstName = 222, LastName = 'll'";*/ + MySQLDMLGenerator mySQLDMLGenerator = new MySQLDMLGenerator(); String sql = - DMLGenerator.getDMLStatement( - modType, tableName, schema, newValuesJson, keyValuesJson, "+00:00"); + mySQLDMLGenerator.getDMLStatement( + new DMLGeneratorRequest( + modType, tableName, schema, newValuesJson, keyValuesJson, "+00:00")); assertTrue(sql.contains("FirstName = 222")); assertTrue(sql.contains("LastName = 'll'")); @@ -122,9 +130,11 @@ public void tableNameMatchSourceColumnNotPresentInSpanner() { /* The expected sql is: "INSERT INTO Singers(SingerId,FirstName,LastName) VALUES (999,'kk','ll') ON DUPLICATE KEY" + " UPDATE FirstName = 'kk', LastName = 'll'"; */ + MySQLDMLGenerator mySQLDMLGenerator = new MySQLDMLGenerator(); String sql = - DMLGenerator.getDMLStatement( - modType, tableName, schema, newValuesJson, keyValuesJson, "+00:00"); + mySQLDMLGenerator.getDMLStatement( + new DMLGeneratorRequest( + modType, tableName, schema, newValuesJson, keyValuesJson, "+00:00")); assertTrue(sql.contains("FirstName = 'kk'")); assertTrue(sql.contains("LastName = 'll'")); @@ -145,9 +155,11 @@ public void tableNameMatchSpannerColumnNotPresentInSource() { /* The expected sql is: "INSERT INTO Singers(SingerId,FirstName,LastName) VALUES (999,'kk','ll') ON DUPLICATE KEY" + " UPDATE FirstName = 'kk', LastName = 'll'";*/ + MySQLDMLGenerator mySQLDMLGenerator = new MySQLDMLGenerator(); String sql = - DMLGenerator.getDMLStatement( - modType, tableName, schema, newValuesJson, keyValuesJson, "+00:00"); + mySQLDMLGenerator.getDMLStatement( + new DMLGeneratorRequest( + modType, tableName, schema, newValuesJson, keyValuesJson, "+00:00")); assertTrue(sql.contains("FirstName = 'kk'")); assertTrue(sql.contains("LastName = 'll'")); @@ -164,9 +176,11 @@ public void primaryKeyNotFoundInJson() { String modType = "INSERT"; /* The expected sql is: ""*/ + MySQLDMLGenerator mySQLDMLGenerator = new MySQLDMLGenerator(); String sql = - DMLGenerator.getDMLStatement( - modType, tableName, schema, newValuesJson, keyValuesJson, "+00:00"); + mySQLDMLGenerator.getDMLStatement( + new DMLGeneratorRequest( + modType, tableName, schema, newValuesJson, keyValuesJson, "+00:00")); assertTrue(sql.isEmpty()); } @@ -182,10 +196,11 @@ public void primaryKeyNotPresentInSourceSchema() { String modType = "INSERT"; /* The expected sql is: ""*/ + MySQLDMLGenerator mySQLDMLGenerator = new MySQLDMLGenerator(); String sql = - sql = - DMLGenerator.getDMLStatement( - modType, tableName, schema, newValuesJson, keyValuesJson, "+00:00"); + mySQLDMLGenerator.getDMLStatement( + new DMLGeneratorRequest( + modType, tableName, schema, newValuesJson, keyValuesJson, "+00:00")); assertTrue(sql.isEmpty()); } @@ -204,9 +219,11 @@ public void timezoneOffsetMismatch() { "INSERT INTO Singers(SingerId,Bday) VALUES (999," + " CONVERT_TZ('2023-05-18T12:01:13.088397258','+00:00','+10:00')) ON DUPLICATE KEY" + " UPDATE Bday = CONVERT_TZ('2023-05-18T12:01:13.088397258','+00:00','+10:00')";*/ + MySQLDMLGenerator mySQLDMLGenerator = new MySQLDMLGenerator(); String sql = - DMLGenerator.getDMLStatement( - modType, tableName, schema, newValuesJson, keyValuesJson, "+10:00"); + mySQLDMLGenerator.getDMLStatement( + new DMLGeneratorRequest( + modType, tableName, schema, newValuesJson, keyValuesJson, "+10:00")); assertTrue( sql.contains("Bday = CONVERT_TZ('2023-05-18T12:01:13.088397258','+00:00','+10:00'")); @@ -225,9 +242,11 @@ public void primaryKeyMismatch() { /* The expected sql is: "INSERT INTO Singers(SingerId,FirstName,LastName) VALUES (999,'kk','ll') ON DUPLICATE KEY" + " UPDATE FirstName = 'kk', LastName = 'll'";*/ + MySQLDMLGenerator mySQLDMLGenerator = new MySQLDMLGenerator(); String sql = - DMLGenerator.getDMLStatement( - modType, tableName, schema, newValuesJson, keyValuesJson, "+00:00"); + mySQLDMLGenerator.getDMLStatement( + new DMLGeneratorRequest( + modType, tableName, schema, newValuesJson, keyValuesJson, "+00:00")); assertTrue(sql.contains("FirstName = 'kk'")); assertTrue(sql.contains("LastName = 'll'")); @@ -278,9 +297,11 @@ public void allDataypesDML() throws Exception { + " decimal_column = 444.222, bool_column = false, char_column = ' connectionPoolMap = new HashMap<>(); connectionPoolMap.put("urluserpass", mockHikariDataSource); - ConnectionHelper.setConnectionPoolMap(connectionPoolMap); + MySQLConnectionHelper.setConnectionPoolMap(connectionPoolMap); MySqlDao mySqlDao = new MySqlDao("url", "user", "pass"); mySqlDao.write("sql"); verify(mockStatement).executeUpdate(eq("sql")); diff --git a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/transforms/SourceWriterFnTest.java b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/transforms/SourceWriterFnTest.java index 990e190a04..65594590f1 100644 --- a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/transforms/SourceWriterFnTest.java +++ b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/transforms/SourceWriterFnTest.java @@ -41,6 +41,7 @@ import com.google.cloud.teleport.v2.spanner.migrations.utils.SessionFileReader; import com.google.cloud.teleport.v2.templates.changestream.ChangeStreamErrorRecord; import com.google.cloud.teleport.v2.templates.changestream.TrimmedShardedDataChangeRecord; +import com.google.cloud.teleport.v2.templates.common.ISourceDao; import com.google.cloud.teleport.v2.templates.constants.Constants; import com.google.cloud.teleport.v2.templates.mysql.MySqlDao; import com.google.cloud.teleport.v2.templates.utils.ShadowTableRecord; @@ -70,7 +71,7 @@ public class SourceWriterFnTest { @Rule public final MockitoRule mocktio = MockitoJUnit.rule(); @Mock private MySqlDao mockMySqlDao; @Mock private SpannerDao mockSpannerDao; - @Mock HashMap mockMySqlDaoMap; + @Mock HashMap mockSourceDaoMap; @Mock private SpannerConfig mockSpannerConfig; @Mock private DoFn.ProcessContext processContext; private static Gson gson = new Gson(); @@ -82,7 +83,7 @@ public class SourceWriterFnTest { @Before public void doBeforeEachTest() throws Exception { - when(mockMySqlDaoMap.get(any())).thenReturn(mockMySqlDao); + when(mockSourceDaoMap.get(any())).thenReturn(mockMySqlDao); when(mockSpannerDao.getShadowTableRecord(eq("shadow_parent1"), any())).thenReturn(null); when(mockSpannerDao.getShadowTableRecord(eq("shadow_tableName"), any())).thenReturn(null); when(mockSpannerDao.getShadowTableRecord(eq("shadow_parent2"), any())) @@ -137,12 +138,14 @@ public void testSourceIsAhead() throws Exception { testDdl, "shadow_", "skip", - 500); + 500, + "mysql", + Constants.SHARDING_MODE_MULTI_SHARD); ObjectMapper mapper = new ObjectMapper(); mapper.enable(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS); sourceWriterFn.setObjectMapper(mapper); sourceWriterFn.setSpannerDao(mockSpannerDao); - sourceWriterFn.setMySqlDaoMap(mockMySqlDaoMap); + sourceWriterFn.setSourceDaoMap(mockSourceDaoMap); sourceWriterFn.processElement(processContext); verify(mockSpannerDao, atLeast(1)).getShadowTableRecord(any(), any()); verify(mockMySqlDao, never()).write(any()); @@ -164,12 +167,14 @@ public void testSourceIsAheadWithSameCommitTimestamp() throws Exception { testDdl, "shadow_", "skip", - 500); + 500, + "mysql", + Constants.SHARDING_MODE_MULTI_SHARD); ObjectMapper mapper = new ObjectMapper(); mapper.enable(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS); sourceWriterFn.setObjectMapper(mapper); sourceWriterFn.setSpannerDao(mockSpannerDao); - sourceWriterFn.setMySqlDaoMap(mockMySqlDaoMap); + sourceWriterFn.setSourceDaoMap(mockSourceDaoMap); sourceWriterFn.processElement(processContext); verify(mockSpannerDao, atLeast(1)).getShadowTableRecord(any(), any()); verify(mockMySqlDao, never()).write(any()); @@ -190,12 +195,14 @@ public void testSourceIsBehind() throws Exception { testDdl, "shadow_", "skip", - 500); + 500, + "mysql", + Constants.SHARDING_MODE_MULTI_SHARD); ObjectMapper mapper = new ObjectMapper(); mapper.enable(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS); sourceWriterFn.setObjectMapper(mapper); sourceWriterFn.setSpannerDao(mockSpannerDao); - sourceWriterFn.setMySqlDaoMap(mockMySqlDaoMap); + sourceWriterFn.setSourceDaoMap(mockSourceDaoMap); sourceWriterFn.processElement(processContext); verify(mockSpannerDao, atLeast(1)).getShadowTableRecord(any(), any()); verify(mockMySqlDao, atLeast(1)).write(any()); @@ -215,12 +222,14 @@ public void testNoShard() throws Exception { testDdl, "shadow_", "skip", - 500); + 500, + "mysql", + Constants.SHARDING_MODE_MULTI_SHARD); ObjectMapper mapper = new ObjectMapper(); mapper.enable(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS); sourceWriterFn.setObjectMapper(mapper); sourceWriterFn.setSpannerDao(mockSpannerDao); - sourceWriterFn.setMySqlDaoMap(mockMySqlDaoMap); + sourceWriterFn.setSourceDaoMap(mockSourceDaoMap); sourceWriterFn.processElement(processContext); String jsonRec = gson.toJson(record, TrimmedShardedDataChangeRecord.class); ChangeStreamErrorRecord errorRecord = @@ -245,12 +254,14 @@ public void testSkipShard() throws Exception { testDdl, "shadow_", "skip", - 500); + 500, + "mysql", + Constants.SHARDING_MODE_MULTI_SHARD); ObjectMapper mapper = new ObjectMapper(); mapper.enable(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS); sourceWriterFn.setObjectMapper(mapper); sourceWriterFn.setSpannerDao(mockSpannerDao); - sourceWriterFn.setMySqlDaoMap(mockMySqlDaoMap); + sourceWriterFn.setSourceDaoMap(mockSourceDaoMap); sourceWriterFn.processElement(processContext); String jsonRec = gson.toJson(record, TrimmedShardedDataChangeRecord.class); ChangeStreamErrorRecord errorRecord = @@ -273,12 +284,14 @@ public void testPermanentError() throws Exception { testDdl, "shadow_", "skip", - 500); + 500, + "mysql", + Constants.SHARDING_MODE_MULTI_SHARD); ObjectMapper mapper = new ObjectMapper(); mapper.enable(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS); sourceWriterFn.setObjectMapper(mapper); sourceWriterFn.setSpannerDao(mockSpannerDao); - sourceWriterFn.setMySqlDaoMap(mockMySqlDaoMap); + sourceWriterFn.setSourceDaoMap(mockSourceDaoMap); sourceWriterFn.processElement(processContext); String jsonRec = gson.toJson(record, TrimmedShardedDataChangeRecord.class); ChangeStreamErrorRecord errorRecord = @@ -305,12 +318,14 @@ public void testRetryableError() throws Exception { testDdl, "shadow_", "skip", - 500); + 500, + "mysql", + Constants.SHARDING_MODE_MULTI_SHARD); ObjectMapper mapper = new ObjectMapper(); mapper.enable(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS); sourceWriterFn.setObjectMapper(mapper); sourceWriterFn.setSpannerDao(mockSpannerDao); - sourceWriterFn.setMySqlDaoMap(mockMySqlDaoMap); + sourceWriterFn.setSourceDaoMap(mockSourceDaoMap); sourceWriterFn.processElement(processContext); String jsonRec = gson.toJson(record, TrimmedShardedDataChangeRecord.class); ChangeStreamErrorRecord errorRecord = new ChangeStreamErrorRecord(jsonRec, "Test exception"); @@ -333,12 +348,14 @@ public void testRetryableErrorForForeignKey() throws Exception { testDdl, "shadow_", "skip", - 500); + 500, + "mysql", + Constants.SHARDING_MODE_MULTI_SHARD); ObjectMapper mapper = new ObjectMapper(); mapper.enable(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS); sourceWriterFn.setObjectMapper(mapper); sourceWriterFn.setSpannerDao(mockSpannerDao); - sourceWriterFn.setMySqlDaoMap(mockMySqlDaoMap); + sourceWriterFn.setSourceDaoMap(mockSourceDaoMap); sourceWriterFn.processElement(processContext); String jsonRec = gson.toJson(record, TrimmedShardedDataChangeRecord.class); ChangeStreamErrorRecord errorRecord = @@ -362,12 +379,14 @@ public void testRetryableErrorConnectionFailure() throws Exception { testDdl, "shadow_", "skip", - 500); + 500, + "mysql", + Constants.SHARDING_MODE_MULTI_SHARD); ObjectMapper mapper = new ObjectMapper(); mapper.enable(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS); sourceWriterFn.setObjectMapper(mapper); sourceWriterFn.setSpannerDao(mockSpannerDao); - sourceWriterFn.setMySqlDaoMap(mockMySqlDaoMap); + sourceWriterFn.setSourceDaoMap(mockSourceDaoMap); sourceWriterFn.processElement(processContext); String jsonRec = gson.toJson(record, TrimmedShardedDataChangeRecord.class); ChangeStreamErrorRecord errorRecord = @@ -391,12 +410,14 @@ public void testPermanentConnectionFailure() throws Exception { testDdl, "shadow_", "skip", - 500); + 500, + "mysql", + Constants.SHARDING_MODE_MULTI_SHARD); ObjectMapper mapper = new ObjectMapper(); mapper.enable(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS); sourceWriterFn.setObjectMapper(mapper); sourceWriterFn.setSpannerDao(mockSpannerDao); - sourceWriterFn.setMySqlDaoMap(mockMySqlDaoMap); + sourceWriterFn.setSourceDaoMap(mockSourceDaoMap); sourceWriterFn.processElement(processContext); String jsonRec = gson.toJson(record, TrimmedShardedDataChangeRecord.class); ChangeStreamErrorRecord errorRecord = @@ -420,12 +441,14 @@ public void testPermanentGenericException() throws Exception { testDdl, "shadow_", "skip", - 500); + 500, + "mysql", + Constants.SHARDING_MODE_MULTI_SHARD); ObjectMapper mapper = new ObjectMapper(); mapper.enable(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS); sourceWriterFn.setObjectMapper(mapper); sourceWriterFn.setSpannerDao(mockSpannerDao); - sourceWriterFn.setMySqlDaoMap(mockMySqlDaoMap); + sourceWriterFn.setSourceDaoMap(mockSourceDaoMap); sourceWriterFn.processElement(processContext); String jsonRec = gson.toJson(record, TrimmedShardedDataChangeRecord.class); ChangeStreamErrorRecord errorRecord = new ChangeStreamErrorRecord(jsonRec, "generic exception"); @@ -448,12 +471,14 @@ public void testDMLEmpty() throws Exception { testDdlForNullDML(), "shadow_", "skip", - 500); + 500, + "mysql", + Constants.SHARDING_MODE_MULTI_SHARD); ObjectMapper mapper = new ObjectMapper(); mapper.enable(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS); sourceWriterFn.setObjectMapper(mapper); sourceWriterFn.setSpannerDao(mockSpannerDao); - sourceWriterFn.setMySqlDaoMap(mockMySqlDaoMap); + sourceWriterFn.setSourceDaoMap(mockSourceDaoMap); sourceWriterFn.processElement(processContext); verify(mockMySqlDao, never()).write(contains("567890")); } From a00046efefb33b01c549defffd600826a1323500 Mon Sep 17 00:00:00 2001 From: Shreya Khajanchi Date: Mon, 11 Nov 2024 15:27:22 +0530 Subject: [PATCH 04/13] addressing some comments --- .../v2/templates/SpannerToSourceDb.java | 8 ++- .../common/DMLGeneratorRequest.java | 2 +- .../{ => source}/common/IDMLGenerator.java | 2 +- .../{ => source}/common/ISourceDao.java | 2 +- .../common/SourceProcessorFactory.java | 16 ++--- .../{ => source}/common/package-info.java | 2 +- .../sql/SQLConnectionHelper.java} | 21 ++++--- .../MySqlDao.java => source/sql/SqlDao.java} | 10 +-- .../sql}/mysql/MySQLDMLGenerator.java | 6 +- .../{ => source/sql}/mysql/package-info.java | 2 +- .../v2/templates/source/sql/package-info.java | 18 ++++++ .../templates/transforms/SourceWriterFn.java | 16 ++--- .../transforms/SourceWriterTransform.java | 8 +-- .../templates/utils/InputRecordProcessor.java | 11 ++-- .../common/DMLGeneratorRequestTest.java | 2 +- .../common/SourceProcessorFactoryTest.java | 18 +++--- .../sql/SqlDaoTest.java} | 14 ++--- .../sql}/mysql/MySQLDMLGeneratorTest.java | 4 +- .../transforms/SourceWriterFnTest.java | 62 ++++++++----------- 19 files changed, 117 insertions(+), 107 deletions(-) rename v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/{ => source}/common/DMLGeneratorRequest.java (96%) rename v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/{ => source}/common/IDMLGenerator.java (91%) rename v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/{ => source}/common/ISourceDao.java (93%) rename v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/{ => source}/common/SourceProcessorFactory.java (74%) rename v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/{ => source}/common/package-info.java (91%) rename v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/{mysql/MySQLConnectionHelper.java => source/sql/SQLConnectionHelper.java} (84%) rename v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/{mysql/MySqlDao.java => source/sql/SqlDao.java} (81%) rename v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/{ => source/sql}/mysql/MySQLDMLGenerator.java (98%) rename v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/{ => source/sql}/mysql/package-info.java (90%) create mode 100644 v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/source/sql/package-info.java rename v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/{ => source}/common/DMLGeneratorRequestTest.java (97%) rename v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/{ => source}/common/SourceProcessorFactoryTest.java (85%) rename v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/{mysql/MySqlDaoTest.java => source/sql/SqlDaoTest.java} (86%) rename v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/{ => source/sql}/mysql/MySQLDMLGeneratorTest.java (99%) diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/SpannerToSourceDb.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/SpannerToSourceDb.java index 8d65463452..e2cfa9a009 100644 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/SpannerToSourceDb.java +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/SpannerToSourceDb.java @@ -357,10 +357,11 @@ public interface Options extends PipelineOptions, StreamingOptions { void setDlqRetryMinutes(Integer value); - @TemplateParameter.Text( + @TemplateParameter.Enum( order = 24, optional = true, description = "Source database type, ex: mysql", + enumOptions = {@TemplateEnumOption("mysql")}, helpText = "The type of source database to reverse replicate to.") @Default.String("mysql") String getSourceType(); @@ -471,6 +472,8 @@ public static PipelineResult run(Options options) { Shard singleShard = shards.get(0); if (singleShard.getLogicalShardId() == null) { singleShard.setLogicalShardId(Constants.DEFAULT_SHARD_ID); + LOG.info( + "Logical shard id was not found, hence setting it to : " + Constants.DEFAULT_SHARD_ID); } } boolean isRegularMode = "regular".equals(options.getRunMode()); @@ -575,8 +578,7 @@ public static PipelineResult run(Options options) { options.getShadowTablePrefix(), options.getSkipDirectoryName(), connectionPoolSizePerWorker, - options.getSourceType(), - shardingMode)); + options.getSourceType())); PCollection> dlqPermErrorRecords = reconsumedElements diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/common/DMLGeneratorRequest.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/source/common/DMLGeneratorRequest.java similarity index 96% rename from v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/common/DMLGeneratorRequest.java rename to v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/source/common/DMLGeneratorRequest.java index 73716a1c5c..7cc683e71e 100644 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/common/DMLGeneratorRequest.java +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/source/common/DMLGeneratorRequest.java @@ -13,7 +13,7 @@ * License for the specific language governing permissions and limitations under * the License. */ -package com.google.cloud.teleport.v2.templates.common; +package com.google.cloud.teleport.v2.templates.source.common; import com.google.cloud.teleport.v2.spanner.migrations.schema.Schema; import org.json.JSONObject; diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/common/IDMLGenerator.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/source/common/IDMLGenerator.java similarity index 91% rename from v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/common/IDMLGenerator.java rename to v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/source/common/IDMLGenerator.java index 10f32ebc2c..89b72837ef 100644 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/common/IDMLGenerator.java +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/source/common/IDMLGenerator.java @@ -13,7 +13,7 @@ * License for the specific language governing permissions and limitations under * the License. */ -package com.google.cloud.teleport.v2.templates.common; +package com.google.cloud.teleport.v2.templates.source.common; public interface IDMLGenerator { String getDMLStatement(DMLGeneratorRequest dmlGeneratorRequest); diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/common/ISourceDao.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/source/common/ISourceDao.java similarity index 93% rename from v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/common/ISourceDao.java rename to v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/source/common/ISourceDao.java index 864cef9fca..c1450e778b 100644 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/common/ISourceDao.java +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/source/common/ISourceDao.java @@ -13,7 +13,7 @@ * License for the specific language governing permissions and limitations under * the License. */ -package com.google.cloud.teleport.v2.templates.common; +package com.google.cloud.teleport.v2.templates.source.common; public interface ISourceDao { diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/common/SourceProcessorFactory.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/source/common/SourceProcessorFactory.java similarity index 74% rename from v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/common/SourceProcessorFactory.java rename to v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/source/common/SourceProcessorFactory.java index f5c306615f..636550a3c1 100644 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/common/SourceProcessorFactory.java +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/source/common/SourceProcessorFactory.java @@ -13,13 +13,13 @@ * License for the specific language governing permissions and limitations under * the License. */ -package com.google.cloud.teleport.v2.templates.common; +package com.google.cloud.teleport.v2.templates.source.common; import com.google.cloud.teleport.v2.spanner.migrations.shard.Shard; import com.google.cloud.teleport.v2.templates.constants.Constants; -import com.google.cloud.teleport.v2.templates.mysql.MySQLConnectionHelper; -import com.google.cloud.teleport.v2.templates.mysql.MySQLDMLGenerator; -import com.google.cloud.teleport.v2.templates.mysql.MySqlDao; +import com.google.cloud.teleport.v2.templates.source.sql.SQLConnectionHelper; +import com.google.cloud.teleport.v2.templates.source.sql.SqlDao; +import com.google.cloud.teleport.v2.templates.source.sql.mysql.MySQLDMLGenerator; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -36,13 +36,13 @@ public static Map getSourceDaoMap( String source, List shards, int maxConnections) throws Exception { if (source.equalsIgnoreCase(Constants.SOURCE_MYSQL)) { Map sourceDaoMap = new HashMap<>(); - MySQLConnectionHelper.init(shards, null, maxConnections); + SQLConnectionHelper.init(shards, null, maxConnections, source, "com.mysql.cj.jdbc.Driver"); for (Shard shard : shards) { String sourceConnectionUrl = "jdbc:mysql://" + shard.getHost() + ":" + shard.getPort() + "/" + shard.getDbName(); - ISourceDao mySqlDao = - new MySqlDao(sourceConnectionUrl, shard.getUserName(), shard.getPassword()); - sourceDaoMap.put(shard.getLogicalShardId(), mySqlDao); + ISourceDao sqlDao = + new SqlDao(sourceConnectionUrl, shard.getUserName(), shard.getPassword()); + sourceDaoMap.put(shard.getLogicalShardId(), sqlDao); } return sourceDaoMap; } diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/common/package-info.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/source/common/package-info.java similarity index 91% rename from v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/common/package-info.java rename to v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/source/common/package-info.java index 14cd579bfb..a585ed54a5 100644 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/common/package-info.java +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/source/common/package-info.java @@ -15,4 +15,4 @@ */ /** Package info for common module. */ -package com.google.cloud.teleport.v2.templates.common; +package com.google.cloud.teleport.v2.templates.source.common; diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/mysql/MySQLConnectionHelper.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/source/sql/SQLConnectionHelper.java similarity index 84% rename from v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/mysql/MySQLConnectionHelper.java rename to v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/source/sql/SQLConnectionHelper.java index 61c2c9094a..dd810fcce3 100644 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/mysql/MySQLConnectionHelper.java +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/source/sql/SQLConnectionHelper.java @@ -13,7 +13,7 @@ * License for the specific language governing permissions and limitations under * the License. */ -package com.google.cloud.teleport.v2.templates.mysql; +package com.google.cloud.teleport.v2.templates.source.sql; import com.google.cloud.teleport.v2.spanner.migrations.shard.Shard; import com.zaxxer.hikari.HikariConfig; @@ -29,13 +29,13 @@ import org.slf4j.LoggerFactory; /** This is a per Dataflow worker singleton that holds connection pool. */ -public class MySQLConnectionHelper { +public class SQLConnectionHelper { - private static final Logger LOG = LoggerFactory.getLogger(MySQLConnectionHelper.class); - private static final String JDBC_DRIVER = "com.mysql.cj.jdbc.Driver"; + private static final Logger LOG = LoggerFactory.getLogger(SQLConnectionHelper.class); private static Map connectionPoolMap = null; - public static synchronized void init(List shards, String properties, int maxConnections) { + public static synchronized void init( + List shards, String properties, int maxConnections, String source, String jdbcDriver) { if (connectionPoolMap != null) { return; } @@ -43,12 +43,19 @@ public static synchronized void init(List shards, String properties, int connectionPoolMap = new HashMap<>(); for (Shard shard : shards) { String sourceConnectionUrl = - "jdbc:mysql://" + shard.getHost() + ":" + shard.getPort() + "/" + shard.getDbName(); + "jdbc:" + + source + + "://" + + shard.getHost() + + ":" + + shard.getPort() + + "/" + + shard.getDbName(); HikariConfig config = new HikariConfig(); config.setJdbcUrl(sourceConnectionUrl); config.setUsername(shard.getUserName()); config.setPassword(shard.getPassword()); - config.setDriverClassName(JDBC_DRIVER); + config.setDriverClassName(jdbcDriver); config.setMaximumPoolSize(maxConnections); config.setConnectionInitSql( "SET SESSION net_read_timeout=1200"); // to avoid timeouts at network level layer diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/mysql/MySqlDao.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/source/sql/SqlDao.java similarity index 81% rename from v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/mysql/MySqlDao.java rename to v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/source/sql/SqlDao.java index ae2b9edbe1..30c9e9246e 100644 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/mysql/MySqlDao.java +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/source/sql/SqlDao.java @@ -13,20 +13,20 @@ * License for the specific language governing permissions and limitations under * the License. */ -package com.google.cloud.teleport.v2.templates.mysql; +package com.google.cloud.teleport.v2.templates.source.sql; -import com.google.cloud.teleport.v2.templates.common.ISourceDao; +import com.google.cloud.teleport.v2.templates.source.common.ISourceDao; import com.google.cloud.teleport.v2.templates.utils.ConnectionException; import java.sql.Connection; import java.sql.SQLException; import java.sql.Statement; -public class MySqlDao implements ISourceDao { +public class SqlDao implements ISourceDao { private String sqlUrl; private String sqlUser; private String sqlPasswd; - public MySqlDao(String sqlUrl, String sqlUser, String sqlPasswd) { + public SqlDao(String sqlUrl, String sqlUser, String sqlPasswd) { this.sqlUrl = sqlUrl; this.sqlUser = sqlUser; this.sqlPasswd = sqlPasswd; @@ -42,7 +42,7 @@ public void write(String sqlStatement) throws SQLException, ConnectionException try { - connObj = MySQLConnectionHelper.getConnection(this.sqlUrl, this.sqlUser, this.sqlPasswd); + connObj = SQLConnectionHelper.getConnection(this.sqlUrl, this.sqlUser, this.sqlPasswd); if (connObj == null) { throw new ConnectionException("Connection is null"); } diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/mysql/MySQLDMLGenerator.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/source/sql/mysql/MySQLDMLGenerator.java similarity index 98% rename from v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/mysql/MySQLDMLGenerator.java rename to v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/source/sql/mysql/MySQLDMLGenerator.java index d433120024..b0a094ad07 100644 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/mysql/MySQLDMLGenerator.java +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/source/sql/mysql/MySQLDMLGenerator.java @@ -13,15 +13,15 @@ * License for the specific language governing permissions and limitations under * the License. */ -package com.google.cloud.teleport.v2.templates.mysql; +package com.google.cloud.teleport.v2.templates.source.sql.mysql; import com.google.cloud.teleport.v2.spanner.migrations.schema.ColumnPK; import com.google.cloud.teleport.v2.spanner.migrations.schema.SourceColumnDefinition; import com.google.cloud.teleport.v2.spanner.migrations.schema.SourceTable; import com.google.cloud.teleport.v2.spanner.migrations.schema.SpannerColumnDefinition; import com.google.cloud.teleport.v2.spanner.migrations.schema.SpannerTable; -import com.google.cloud.teleport.v2.templates.common.DMLGeneratorRequest; -import com.google.cloud.teleport.v2.templates.common.IDMLGenerator; +import com.google.cloud.teleport.v2.templates.source.common.DMLGeneratorRequest; +import com.google.cloud.teleport.v2.templates.source.common.IDMLGenerator; import java.util.HashMap; import java.util.Map; import java.util.Set; diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/mysql/package-info.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/source/sql/mysql/package-info.java similarity index 90% rename from v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/mysql/package-info.java rename to v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/source/sql/mysql/package-info.java index 6b4f0082dd..e5da8f92db 100644 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/mysql/package-info.java +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/source/sql/mysql/package-info.java @@ -15,4 +15,4 @@ */ /** Package info for mysql source module. */ -package com.google.cloud.teleport.v2.templates.mysql; +package com.google.cloud.teleport.v2.templates.source.sql.mysql; diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/source/sql/package-info.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/source/sql/package-info.java new file mode 100644 index 0000000000..dc47f6f7dd --- /dev/null +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/source/sql/package-info.java @@ -0,0 +1,18 @@ +/* + * Copyright (C) 2023 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +/** Package info for sql module. */ +package com.google.cloud.teleport.v2.templates.source.sql; diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/transforms/SourceWriterFn.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/transforms/SourceWriterFn.java index db74e38955..7e08dbf39d 100644 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/transforms/SourceWriterFn.java +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/transforms/SourceWriterFn.java @@ -29,9 +29,10 @@ import com.google.cloud.teleport.v2.spanner.migrations.shard.Shard; import com.google.cloud.teleport.v2.templates.changestream.ChangeStreamErrorRecord; import com.google.cloud.teleport.v2.templates.changestream.TrimmedShardedDataChangeRecord; -import com.google.cloud.teleport.v2.templates.common.ISourceDao; -import com.google.cloud.teleport.v2.templates.common.SourceProcessorFactory; import com.google.cloud.teleport.v2.templates.constants.Constants; +import com.google.cloud.teleport.v2.templates.source.common.IDMLGenerator; +import com.google.cloud.teleport.v2.templates.source.common.ISourceDao; +import com.google.cloud.teleport.v2.templates.source.common.SourceProcessorFactory; import com.google.cloud.teleport.v2.templates.utils.ConnectionException; import com.google.cloud.teleport.v2.templates.utils.InputRecordProcessor; import com.google.cloud.teleport.v2.templates.utils.ShadowTableRecord; @@ -86,7 +87,7 @@ public class SourceWriterFn extends DoFn shards, @@ -97,8 +98,7 @@ public SourceWriterFn( String shadowTablePrefix, String skipDirName, int maxThreadPerDataflowWorker, - String source, - String shardingMode) { + String source) { this.schema = schema; this.sourceDbTimezoneOffset = sourceDbTimezoneOffset; @@ -109,7 +109,6 @@ public SourceWriterFn( this.skipDirName = skipDirName; this.maxThreadPerDataflowWorker = maxThreadPerDataflowWorker; this.source = source; - this.shardingMode = shardingMode; } // for unit testing purposes @@ -135,6 +134,7 @@ public void setup() throws Exception { sourceDaoMap = SourceProcessorFactory.getSourceDaoMap(source, shards, maxThreadPerDataflowWorker); spannerDao = new SpannerDao(spannerConfig); + dmlGenerator = SourceProcessorFactory.getDMLGenerator(source); } /** Teardown function disconnects from the Cloud Spanner. */ @@ -153,7 +153,7 @@ public void processElement(ProcessContext c) { // no shard found, move to permanent error outputWithTag( c, Constants.PERMANENT_ERROR_TAG, Constants.SHARD_NOT_PRESENT_ERROR_MESSAGE, spannerRec); - } else if (shardId != null && shardId.equals(skipDirName)) { + } else if (shardId.equals(skipDirName)) { // the record is skipped skippedRecordCountMetric.inc(); outputWithTag(c, Constants.SKIPPED_TAG, Constants.SKIPPED_TAG_MESSAGE, spannerRec); @@ -189,7 +189,7 @@ public void processElement(ProcessContext c) { InputRecordProcessor inputRecordProcessor = new InputRecordProcessor(); inputRecordProcessor.processRecord( - spannerRec, schema, sourceDao, shardId, sourceDbTimezoneOffset, source); + spannerRec, schema, sourceDao, shardId, sourceDbTimezoneOffset, source, dmlGenerator); spannerDao.updateShadowTable( getShadowTableMutation( diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/transforms/SourceWriterTransform.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/transforms/SourceWriterTransform.java index 56429a46b6..e80f617eeb 100644 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/transforms/SourceWriterTransform.java +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/transforms/SourceWriterTransform.java @@ -52,7 +52,6 @@ public class SourceWriterTransform private final String skipDirName; private final int maxThreadPerDataflowWorker; private final String source; - private final String shardingMode; public SourceWriterTransform( List shards, @@ -63,8 +62,7 @@ public SourceWriterTransform( String shadowTablePrefix, String skipDirName, int maxThreadPerDataflowWorker, - String source, - String shardingMode) { + String source) { this.schema = schema; this.sourceDbTimezoneOffset = sourceDbTimezoneOffset; @@ -75,7 +73,6 @@ public SourceWriterTransform( this.skipDirName = skipDirName; this.maxThreadPerDataflowWorker = maxThreadPerDataflowWorker; this.source = source; - this.shardingMode = shardingMode; } @Override @@ -94,8 +91,7 @@ public SourceWriterTransform.Result expand( this.shadowTablePrefix, this.skipDirName, this.maxThreadPerDataflowWorker, - this.source, - this.shardingMode)) + this.source)) .withOutputTags( Constants.SUCCESS_TAG, TupleTagList.of(Constants.PERMANENT_ERROR_TAG) diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/InputRecordProcessor.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/InputRecordProcessor.java index 2f82023689..7304b83e34 100644 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/InputRecordProcessor.java +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/InputRecordProcessor.java @@ -17,10 +17,9 @@ import com.google.cloud.teleport.v2.spanner.migrations.schema.Schema; import com.google.cloud.teleport.v2.templates.changestream.TrimmedShardedDataChangeRecord; -import com.google.cloud.teleport.v2.templates.common.DMLGeneratorRequest; -import com.google.cloud.teleport.v2.templates.common.IDMLGenerator; -import com.google.cloud.teleport.v2.templates.common.ISourceDao; -import com.google.cloud.teleport.v2.templates.common.SourceProcessorFactory; +import com.google.cloud.teleport.v2.templates.source.common.DMLGeneratorRequest; +import com.google.cloud.teleport.v2.templates.source.common.IDMLGenerator; +import com.google.cloud.teleport.v2.templates.source.common.ISourceDao; import java.time.Instant; import java.time.temporal.ChronoUnit; import org.apache.beam.sdk.metrics.Counter; @@ -42,7 +41,8 @@ public void processRecord( ISourceDao dao, String shardId, String sourceDbTimezoneOffset, - String source) + String source, + IDMLGenerator dmlGenerator) throws Exception { try { @@ -58,7 +58,6 @@ public void processRecord( new DMLGeneratorRequest( modType, tableName, schema, newValuesJson, keysJson, sourceDbTimezoneOffset); - IDMLGenerator dmlGenerator = SourceProcessorFactory.getDMLGenerator(source); String dmlStatement = dmlGenerator.getDMLStatement(dmlGeneratorRequest); if (dmlStatement.isEmpty()) { LOG.warn("DML statement is empty for table: " + tableName); diff --git a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/common/DMLGeneratorRequestTest.java b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/source/common/DMLGeneratorRequestTest.java similarity index 97% rename from v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/common/DMLGeneratorRequestTest.java rename to v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/source/common/DMLGeneratorRequestTest.java index 14e7b01255..e59b0e0d1f 100644 --- a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/common/DMLGeneratorRequestTest.java +++ b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/source/common/DMLGeneratorRequestTest.java @@ -13,7 +13,7 @@ * License for the specific language governing permissions and limitations under * the License. */ -package com.google.cloud.teleport.v2.templates.common; +package com.google.cloud.teleport.v2.templates.source.common; import static org.junit.Assert.assertEquals; import static org.mockito.Mockito.mock; diff --git a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/common/SourceProcessorFactoryTest.java b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/source/common/SourceProcessorFactoryTest.java similarity index 85% rename from v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/common/SourceProcessorFactoryTest.java rename to v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/source/common/SourceProcessorFactoryTest.java index 19f752a1bd..cd960cf55c 100644 --- a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/common/SourceProcessorFactoryTest.java +++ b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/source/common/SourceProcessorFactoryTest.java @@ -13,7 +13,7 @@ * License for the specific language governing permissions and limitations under * the License. */ -package com.google.cloud.teleport.v2.templates.common; +package com.google.cloud.teleport.v2.templates.source.common; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; @@ -23,8 +23,8 @@ import com.google.cloud.teleport.v2.spanner.migrations.shard.Shard; import com.google.cloud.teleport.v2.templates.constants.Constants; -import com.google.cloud.teleport.v2.templates.mysql.MySQLDMLGenerator; -import com.google.cloud.teleport.v2.templates.mysql.MySqlDao; +import com.google.cloud.teleport.v2.templates.source.sql.SqlDao; +import com.google.cloud.teleport.v2.templates.source.sql.mysql.MySQLDMLGenerator; import java.util.Arrays; import java.util.List; import java.util.Map; @@ -76,15 +76,15 @@ public void testGetSourceDaoMap_MySQLSource() throws Exception { assertNotNull(sourceDaoMap); assertEquals(2, sourceDaoMap.size()); - ISourceDao mySqlDao1 = sourceDaoMap.get("shard1"); - assertTrue(mySqlDao1 instanceof MySqlDao); + ISourceDao sqlDao1 = sourceDaoMap.get("shard1"); + assertTrue(sqlDao1 instanceof SqlDao); assertEquals( - "jdbc:mysql://localhost:3306/test_db", ((MySqlDao) mySqlDao1).getSourceConnectionUrl()); + "jdbc:mysql://localhost:3306/test_db", ((SqlDao) sqlDao1).getSourceConnectionUrl()); - ISourceDao mySqlDao2 = sourceDaoMap.get("shard2"); - assertTrue(mySqlDao2 instanceof MySqlDao); + ISourceDao sqlDao2 = sourceDaoMap.get("shard2"); + assertTrue(sqlDao2 instanceof SqlDao); assertEquals( - "jdbc:mysql://localhost:3307/test_db_2", ((MySqlDao) mySqlDao2).getSourceConnectionUrl()); + "jdbc:mysql://localhost:3307/test_db_2", ((SqlDao) sqlDao2).getSourceConnectionUrl()); } @Test diff --git a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/mysql/MySqlDaoTest.java b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/source/sql/SqlDaoTest.java similarity index 86% rename from v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/mysql/MySqlDaoTest.java rename to v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/source/sql/SqlDaoTest.java index 5bb6fb5239..743acf8812 100644 --- a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/mysql/MySqlDaoTest.java +++ b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/source/sql/SqlDaoTest.java @@ -13,7 +13,7 @@ * License for the specific language governing permissions and limitations under * the License. */ -package com.google.cloud.teleport.v2.templates.mysql; +package com.google.cloud.teleport.v2.templates.source.sql; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; @@ -34,7 +34,7 @@ import org.mockito.junit.MockitoJUnit; import org.mockito.junit.MockitoRule; -public final class MySqlDaoTest { +public final class SqlDaoTest { @Rule public final MockitoRule mockito = MockitoJUnit.rule(); @Mock private HikariDataSource mockHikariDataSource; @@ -52,17 +52,17 @@ public void doBeforeEachTest() throws java.sql.SQLException { @Test(expected = ConnectionException.class) public void testNullConnection() throws java.sql.SQLException, ConnectionException { - MySqlDao mySqlDao = new MySqlDao("url", "user", "pass"); - mySqlDao.write("sql"); + SqlDao sqlDao = new SqlDao("url", "user", "pass"); + sqlDao.write("sql"); } @Test public void testSuccess() throws java.sql.SQLException, ConnectionException { Map connectionPoolMap = new HashMap<>(); connectionPoolMap.put("urluserpass", mockHikariDataSource); - MySQLConnectionHelper.setConnectionPoolMap(connectionPoolMap); - MySqlDao mySqlDao = new MySqlDao("url", "user", "pass"); - mySqlDao.write("sql"); + SQLConnectionHelper.setConnectionPoolMap(connectionPoolMap); + SqlDao sqlDao = new SqlDao("url", "user", "pass"); + sqlDao.write("sql"); verify(mockStatement).executeUpdate(eq("sql")); } } diff --git a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/mysql/MySQLDMLGeneratorTest.java b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/source/sql/mysql/MySQLDMLGeneratorTest.java similarity index 99% rename from v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/mysql/MySQLDMLGeneratorTest.java rename to v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/source/sql/mysql/MySQLDMLGeneratorTest.java index 338190c826..8a07911039 100644 --- a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/mysql/MySQLDMLGeneratorTest.java +++ b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/source/sql/mysql/MySQLDMLGeneratorTest.java @@ -13,7 +13,7 @@ * License for the specific language governing permissions and limitations under * the License. */ -package com.google.cloud.teleport.v2.templates.mysql; +package com.google.cloud.teleport.v2.templates.source.sql.mysql; import static org.junit.Assert.assertTrue; @@ -29,7 +29,7 @@ import com.google.cloud.teleport.v2.spanner.migrations.schema.SyntheticPKey; import com.google.cloud.teleport.v2.spanner.migrations.utils.SessionFileReader; import com.google.cloud.teleport.v2.templates.changestream.TrimmedShardedDataChangeRecord; -import com.google.cloud.teleport.v2.templates.common.DMLGeneratorRequest; +import com.google.cloud.teleport.v2.templates.source.common.DMLGeneratorRequest; import com.google.cloud.teleport.v2.templates.utils.InputRecordProcessor; import com.google.gson.FieldNamingPolicy; import com.google.gson.GsonBuilder; diff --git a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/transforms/SourceWriterFnTest.java b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/transforms/SourceWriterFnTest.java index 65594590f1..3a21af508a 100644 --- a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/transforms/SourceWriterFnTest.java +++ b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/transforms/SourceWriterFnTest.java @@ -41,9 +41,9 @@ import com.google.cloud.teleport.v2.spanner.migrations.utils.SessionFileReader; import com.google.cloud.teleport.v2.templates.changestream.ChangeStreamErrorRecord; import com.google.cloud.teleport.v2.templates.changestream.TrimmedShardedDataChangeRecord; -import com.google.cloud.teleport.v2.templates.common.ISourceDao; import com.google.cloud.teleport.v2.templates.constants.Constants; -import com.google.cloud.teleport.v2.templates.mysql.MySqlDao; +import com.google.cloud.teleport.v2.templates.source.common.ISourceDao; +import com.google.cloud.teleport.v2.templates.source.sql.SqlDao; import com.google.cloud.teleport.v2.templates.utils.ShadowTableRecord; import com.google.cloud.teleport.v2.templates.utils.SpannerDao; import com.google.common.collect.ImmutableList; @@ -69,7 +69,7 @@ public class SourceWriterFnTest { @Rule public final transient TestPipeline pipeline = TestPipeline.create(); @Rule public final MockitoRule mocktio = MockitoJUnit.rule(); - @Mock private MySqlDao mockMySqlDao; + @Mock private SqlDao mockSqlDao; @Mock private SpannerDao mockSpannerDao; @Mock HashMap mockSourceDaoMap; @Mock private SpannerConfig mockSpannerConfig; @@ -83,7 +83,7 @@ public class SourceWriterFnTest { @Before public void doBeforeEachTest() throws Exception { - when(mockSourceDaoMap.get(any())).thenReturn(mockMySqlDao); + when(mockSourceDaoMap.get(any())).thenReturn(mockSqlDao); when(mockSpannerDao.getShadowTableRecord(eq("shadow_parent1"), any())).thenReturn(null); when(mockSpannerDao.getShadowTableRecord(eq("shadow_tableName"), any())).thenReturn(null); when(mockSpannerDao.getShadowTableRecord(eq("shadow_parent2"), any())) @@ -93,24 +93,24 @@ public void doBeforeEachTest() throws Exception { when(mockSpannerDao.getShadowTableRecord(eq("shadow_child21"), any())).thenReturn(null); doNothing().when(mockSpannerDao).updateShadowTable(any()); doThrow(new java.sql.SQLIntegrityConstraintViolationException("a foreign key constraint fails")) - .when(mockMySqlDao) + .when(mockSqlDao) .write(contains("2300")); // This is the child_id for which we want to test the foreign key // constraint failure. doThrow( new java.sql.SQLNonTransientConnectionException( "transient connection error", "HY000", 1161)) - .when(mockMySqlDao) + .when(mockSqlDao) .write(contains("1161")); // This is the child_id for which we want to retryable // connection error doThrow( new java.sql.SQLNonTransientConnectionException( "permanent connection error", "HY000", 4242)) - .when(mockMySqlDao) + .when(mockSqlDao) .write(contains("4242")); // no retryable error doThrow(new RuntimeException("generic exception")) - .when(mockMySqlDao) + .when(mockSqlDao) .write(contains("12345")); // to test code path of generic exception - doNothing().when(mockMySqlDao).write(contains("parent1")); + doNothing().when(mockSqlDao).write(contains("parent1")); testShard = new Shard(); testShard.setLogicalShardId("shardA"); testShard.setUser("test"); @@ -139,8 +139,7 @@ public void testSourceIsAhead() throws Exception { "shadow_", "skip", 500, - "mysql", - Constants.SHARDING_MODE_MULTI_SHARD); + "mysql"); ObjectMapper mapper = new ObjectMapper(); mapper.enable(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS); sourceWriterFn.setObjectMapper(mapper); @@ -148,7 +147,7 @@ public void testSourceIsAhead() throws Exception { sourceWriterFn.setSourceDaoMap(mockSourceDaoMap); sourceWriterFn.processElement(processContext); verify(mockSpannerDao, atLeast(1)).getShadowTableRecord(any(), any()); - verify(mockMySqlDao, never()).write(any()); + verify(mockSqlDao, never()).write(any()); verify(mockSpannerDao, never()).updateShadowTable(any()); } @@ -168,8 +167,7 @@ public void testSourceIsAheadWithSameCommitTimestamp() throws Exception { "shadow_", "skip", 500, - "mysql", - Constants.SHARDING_MODE_MULTI_SHARD); + "mysql"); ObjectMapper mapper = new ObjectMapper(); mapper.enable(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS); sourceWriterFn.setObjectMapper(mapper); @@ -177,7 +175,7 @@ public void testSourceIsAheadWithSameCommitTimestamp() throws Exception { sourceWriterFn.setSourceDaoMap(mockSourceDaoMap); sourceWriterFn.processElement(processContext); verify(mockSpannerDao, atLeast(1)).getShadowTableRecord(any(), any()); - verify(mockMySqlDao, never()).write(any()); + verify(mockSqlDao, never()).write(any()); verify(mockSpannerDao, never()).updateShadowTable(any()); } @@ -196,8 +194,7 @@ public void testSourceIsBehind() throws Exception { "shadow_", "skip", 500, - "mysql", - Constants.SHARDING_MODE_MULTI_SHARD); + "mysql"); ObjectMapper mapper = new ObjectMapper(); mapper.enable(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS); sourceWriterFn.setObjectMapper(mapper); @@ -205,7 +202,7 @@ public void testSourceIsBehind() throws Exception { sourceWriterFn.setSourceDaoMap(mockSourceDaoMap); sourceWriterFn.processElement(processContext); verify(mockSpannerDao, atLeast(1)).getShadowTableRecord(any(), any()); - verify(mockMySqlDao, atLeast(1)).write(any()); + verify(mockSqlDao, atLeast(1)).write(any()); verify(mockSpannerDao, atLeast(1)).updateShadowTable(any()); } @@ -223,8 +220,7 @@ public void testNoShard() throws Exception { "shadow_", "skip", 500, - "mysql", - Constants.SHARDING_MODE_MULTI_SHARD); + "mysql"); ObjectMapper mapper = new ObjectMapper(); mapper.enable(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS); sourceWriterFn.setObjectMapper(mapper); @@ -255,8 +251,7 @@ public void testSkipShard() throws Exception { "shadow_", "skip", 500, - "mysql", - Constants.SHARDING_MODE_MULTI_SHARD); + "mysql"); ObjectMapper mapper = new ObjectMapper(); mapper.enable(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS); sourceWriterFn.setObjectMapper(mapper); @@ -285,8 +280,7 @@ public void testPermanentError() throws Exception { "shadow_", "skip", 500, - "mysql", - Constants.SHARDING_MODE_MULTI_SHARD); + "mysql"); ObjectMapper mapper = new ObjectMapper(); mapper.enable(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS); sourceWriterFn.setObjectMapper(mapper); @@ -319,8 +313,7 @@ public void testRetryableError() throws Exception { "shadow_", "skip", 500, - "mysql", - Constants.SHARDING_MODE_MULTI_SHARD); + "mysql"); ObjectMapper mapper = new ObjectMapper(); mapper.enable(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS); sourceWriterFn.setObjectMapper(mapper); @@ -349,8 +342,7 @@ public void testRetryableErrorForForeignKey() throws Exception { "shadow_", "skip", 500, - "mysql", - Constants.SHARDING_MODE_MULTI_SHARD); + "mysql"); ObjectMapper mapper = new ObjectMapper(); mapper.enable(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS); sourceWriterFn.setObjectMapper(mapper); @@ -380,8 +372,7 @@ public void testRetryableErrorConnectionFailure() throws Exception { "shadow_", "skip", 500, - "mysql", - Constants.SHARDING_MODE_MULTI_SHARD); + "mysql"); ObjectMapper mapper = new ObjectMapper(); mapper.enable(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS); sourceWriterFn.setObjectMapper(mapper); @@ -411,8 +402,7 @@ public void testPermanentConnectionFailure() throws Exception { "shadow_", "skip", 500, - "mysql", - Constants.SHARDING_MODE_MULTI_SHARD); + "mysql"); ObjectMapper mapper = new ObjectMapper(); mapper.enable(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS); sourceWriterFn.setObjectMapper(mapper); @@ -442,8 +432,7 @@ public void testPermanentGenericException() throws Exception { "shadow_", "skip", 500, - "mysql", - Constants.SHARDING_MODE_MULTI_SHARD); + "mysql"); ObjectMapper mapper = new ObjectMapper(); mapper.enable(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS); sourceWriterFn.setObjectMapper(mapper); @@ -472,15 +461,14 @@ public void testDMLEmpty() throws Exception { "shadow_", "skip", 500, - "mysql", - Constants.SHARDING_MODE_MULTI_SHARD); + "mysql"); ObjectMapper mapper = new ObjectMapper(); mapper.enable(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS); sourceWriterFn.setObjectMapper(mapper); sourceWriterFn.setSpannerDao(mockSpannerDao); sourceWriterFn.setSourceDaoMap(mockSourceDaoMap); sourceWriterFn.processElement(processContext); - verify(mockMySqlDao, never()).write(contains("567890")); + verify(mockSqlDao, never()).write(contains("567890")); } static Ddl getTestDdl() { From 18c8751a73e52379ef5cb3a68e2f229cea27eade Mon Sep 17 00:00:00 2001 From: Shreya Khajanchi Date: Fri, 15 Nov 2024 11:41:51 +0530 Subject: [PATCH 05/13] minor changes --- .../source/common/DMLGeneratorRequest.java | 4 + .../source/common/SourceProcessor.java | 61 ++++++++++++ .../source/common/SourceProcessorFactory.java | 96 ++++++++++++------- .../templates/transforms/SourceWriterFn.java | 18 ++-- .../common/SourceProcessorFactoryTest.java | 4 +- 5 files changed, 135 insertions(+), 48 deletions(-) create mode 100644 v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/source/common/SourceProcessor.java diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/source/common/DMLGeneratorRequest.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/source/common/DMLGeneratorRequest.java index 7cc683e71e..50ea236e50 100644 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/source/common/DMLGeneratorRequest.java +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/source/common/DMLGeneratorRequest.java @@ -18,6 +18,10 @@ import com.google.cloud.teleport.v2.spanner.migrations.schema.Schema; import org.json.JSONObject; +/** + * A request object representing the data necessary to generate DML + * statements for interacting with a source database. + */ public class DMLGeneratorRequest { private String modType; private String spannerTableName; diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/source/common/SourceProcessor.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/source/common/SourceProcessor.java new file mode 100644 index 0000000000..33a5ae7232 --- /dev/null +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/source/common/SourceProcessor.java @@ -0,0 +1,61 @@ +package com.google.cloud.teleport.v2.templates.source.common; + +import com.google.cloud.teleport.v2.templates.source.sql.SQLConnectionHelper; + +import java.util.HashMap; +import java.util.Map; + +public class SourceProcessor { + + private final IDMLGenerator dmlGenerator; + private final Map sourceDaoMap; + private final SQLConnectionHelper connectionHelper; + + private SourceProcessor(IDMLGenerator dmlGenerator, Map sourceDaoMap, SQLConnectionHelper connectionHelper) { + this.dmlGenerator = dmlGenerator; + this.sourceDaoMap = sourceDaoMap; + this.connectionHelper = connectionHelper; + } + + public IDMLGenerator getDmlGenerator() { + return dmlGenerator; + } + + public Map getSourceDaoMap() { + return sourceDaoMap; + } + + public SQLConnectionHelper getConnectionHelper() { + return connectionHelper; + } + + public static Builder builder() { + return new Builder(); + } + + public static class Builder { + private IDMLGenerator dmlGenerator; + private Map sourceDaoMap = new HashMap<>(); + private SQLConnectionHelper connectionHelper; + + public Builder dmlGenerator(IDMLGenerator dmlGenerator) { + this.dmlGenerator = dmlGenerator; + return this; + } + + public Builder sourceDaoMap(Map sourceDaoMap) { + this.sourceDaoMap = sourceDaoMap; + return this; + } + + public Builder connectionHelper(SQLConnectionHelper connectionHelper) { + this.connectionHelper = connectionHelper; + return this; + } + + public SourceProcessor build() { + return new SourceProcessor(dmlGenerator, sourceDaoMap, connectionHelper); + } + } +} + diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/source/common/SourceProcessorFactory.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/source/common/SourceProcessorFactory.java index 636550a3c1..bdd1d2ae4e 100644 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/source/common/SourceProcessorFactory.java +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/source/common/SourceProcessorFactory.java @@ -1,18 +1,3 @@ -/* - * Copyright (C) 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ package com.google.cloud.teleport.v2.templates.source.common; import com.google.cloud.teleport.v2.spanner.migrations.shard.Shard; @@ -23,29 +8,72 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; +import java.util.function.Function; public class SourceProcessorFactory { - public static IDMLGenerator getDMLGenerator(String source) throws Exception { - if (source.equalsIgnoreCase(Constants.SOURCE_MYSQL)) { - return new MySQLDMLGenerator(); - } - throw new Exception("Invalid source type: " + source); + private static final Map DML_GENERATOR_MAP = Map.of( + Constants.SOURCE_MYSQL, new MySQLDMLGenerator() + ); + + private static final Map DRIVER_MAP = Map.of( + Constants.SOURCE_MYSQL, "com.mysql.cj.jdbc.Driver" + ); + + private static final Map> CONNECTION_URL_GENERATORS = Map.of( + Constants.SOURCE_MYSQL, shard -> + "jdbc:mysql://" + shard.getHost() + ":" + shard.getPort() + "/" + shard.getDbName() + ); + + /** + * Creates a SourceProcessor instance for the specified source type. + * + * @param source the type of the source database + * @param shards the list of shards for the source + * @param maxConnections the maximum number of connections + * @return a configured SourceProcessor instance + * @throws Exception if the source type is invalid + */ + public static SourceProcessor createSourceProcessor(String source, List shards, int maxConnections) throws Exception { + IDMLGenerator dmlGenerator = getDMLGenerator(source); + String driver = getDriver(source); + SQLConnectionHelper connectionHelper = initializeConnectionHelper(source, shards, maxConnections, driver); + Map sourceDaoMap = createSourceDaoMap(source, shards); + + return SourceProcessor.builder() + .dmlGenerator(dmlGenerator) + .sourceDaoMap(sourceDaoMap) + .connectionHelper(connectionHelper) + .build(); + } + + private static IDMLGenerator getDMLGenerator(String source) throws Exception { + return Optional.ofNullable(DML_GENERATOR_MAP.get(source)) + .orElseThrow(() -> new Exception("Invalid source type for DML generator: " + source)); + } + + private static String getDriver(String source) throws Exception { + return Optional.ofNullable(DRIVER_MAP.get(source)) + .orElseThrow(() -> new Exception("Invalid source type for driver: " + source)); } - public static Map getSourceDaoMap( - String source, List shards, int maxConnections) throws Exception { - if (source.equalsIgnoreCase(Constants.SOURCE_MYSQL)) { - Map sourceDaoMap = new HashMap<>(); - SQLConnectionHelper.init(shards, null, maxConnections, source, "com.mysql.cj.jdbc.Driver"); - for (Shard shard : shards) { - String sourceConnectionUrl = - "jdbc:mysql://" + shard.getHost() + ":" + shard.getPort() + "/" + shard.getDbName(); - ISourceDao sqlDao = - new SqlDao(sourceConnectionUrl, shard.getUserName(), shard.getPassword()); - sourceDaoMap.put(shard.getLogicalShardId(), sqlDao); - } - return sourceDaoMap; + private static SQLConnectionHelper initializeConnectionHelper( + String source, List shards, int maxConnections, String driver) throws Exception { + SQLConnectionHelper connectionHelper = new SQLConnectionHelper(); + connectionHelper.init(shards, null, maxConnections, source, driver); + return connectionHelper; + } + + private static Map createSourceDaoMap(String source, List shards) throws Exception { + Function urlGenerator = Optional.ofNullable(CONNECTION_URL_GENERATORS.get(source)) + .orElseThrow(() -> new Exception("Invalid source type for URL generation: " + source)); + + Map sourceDaoMap = new HashMap<>(); + for (Shard shard : shards) { + String connectionUrl = urlGenerator.apply(shard); + ISourceDao sqlDao = new SqlDao(connectionUrl, shard.getUserName(), shard.getPassword()); + sourceDaoMap.put(shard.getLogicalShardId(), sqlDao); } - throw new Exception("Invalid source type: " + source); + return sourceDaoMap; } } diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/transforms/SourceWriterFn.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/transforms/SourceWriterFn.java index 7e08dbf39d..638a9d5fa5 100644 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/transforms/SourceWriterFn.java +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/transforms/SourceWriterFn.java @@ -32,6 +32,7 @@ import com.google.cloud.teleport.v2.templates.constants.Constants; import com.google.cloud.teleport.v2.templates.source.common.IDMLGenerator; import com.google.cloud.teleport.v2.templates.source.common.ISourceDao; +import com.google.cloud.teleport.v2.templates.source.common.SourceProcessor; import com.google.cloud.teleport.v2.templates.source.common.SourceProcessorFactory; import com.google.cloud.teleport.v2.templates.utils.ConnectionException; import com.google.cloud.teleport.v2.templates.utils.InputRecordProcessor; @@ -75,7 +76,6 @@ public class SourceWriterFn extends DoFn sourceDaoMap = new HashMap<>(); private final Schema schema; private final String sourceDbTimezoneOffset; @@ -87,7 +87,7 @@ public class SourceWriterFn extends DoFn shards, @@ -116,10 +116,6 @@ public void setSpannerDao(SpannerDao spannerDao) { this.spannerDao = spannerDao; } - // for unit testing purposes - public void setSourceDaoMap(Map sourceDaoMap) { - this.sourceDaoMap = sourceDaoMap; - } // for unit testing purposes public void setObjectMapper(ObjectMapper mapper) { @@ -131,17 +127,15 @@ public void setObjectMapper(ObjectMapper mapper) { public void setup() throws Exception { mapper = new ObjectMapper(); mapper.enable(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS); - sourceDaoMap = - SourceProcessorFactory.getSourceDaoMap(source, shards, maxThreadPerDataflowWorker); + sourceProcessor= SourceProcessorFactory.createSourceProcessor(source, shards, maxThreadPerDataflowWorker); spannerDao = new SpannerDao(spannerConfig); - dmlGenerator = SourceProcessorFactory.getDMLGenerator(source); + } /** Teardown function disconnects from the Cloud Spanner. */ @Teardown public void teardown() throws Exception { spannerDao.close(); - sourceDaoMap.clear(); } @ProcessElement @@ -185,11 +179,11 @@ public void processElement(ProcessContext c) { > Long.parseLong(spannerRec.getRecordSequence()))); if (!isSourceAhead) { - ISourceDao sourceDao = sourceDaoMap.get(shardId); + ISourceDao sourceDao = sourceProcessor.getSourceDaoMap().get(shardId); InputRecordProcessor inputRecordProcessor = new InputRecordProcessor(); inputRecordProcessor.processRecord( - spannerRec, schema, sourceDao, shardId, sourceDbTimezoneOffset, source, dmlGenerator); + spannerRec, schema, sourceDao, shardId, sourceDbTimezoneOffset, source, sourceProcessor.getDmlGenerator()); spannerDao.updateShadowTable( getShadowTableMutation( diff --git a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/source/common/SourceProcessorFactoryTest.java b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/source/common/SourceProcessorFactoryTest.java index cd960cf55c..ae4641a2c4 100644 --- a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/source/common/SourceProcessorFactoryTest.java +++ b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/source/common/SourceProcessorFactoryTest.java @@ -38,7 +38,7 @@ public class SourceProcessorFactoryTest { @Test public void testGetDMLGenerator_MySQLSource() throws Exception { - IDMLGenerator dmlGenerator = SourceProcessorFactory.getDMLGenerator(Constants.SOURCE_MYSQL); + IDMLGenerator dmlGenerator = SourceProcessorFactory.getSourceDMLGenerator(Constants.SOURCE_MYSQL); assertNotNull(dmlGenerator); assertTrue(dmlGenerator instanceof MySQLDMLGenerator); } @@ -47,7 +47,7 @@ public void testGetDMLGenerator_MySQLSource() throws Exception { public void testGetDMLGenerator_InvalidSource() throws Exception { expectedEx.expect(Exception.class); expectedEx.expectMessage("Invalid source type: invalidSource"); - SourceProcessorFactory.getDMLGenerator("invalidSource"); + SourceProcessorFactory.getSourceDMLGenerator("invalidSource"); } @Test From 54090698501785f0ff342c230fbd351ba4865349 Mon Sep 17 00:00:00 2001 From: Shreya Khajanchi Date: Wed, 20 Nov 2024 10:23:51 +0530 Subject: [PATCH 06/13] addressing comments --- pom.xml | 1 + .../{source/sql => dao}/package-info.java | 4 +- .../ISourceDao.java => dao/source/IDao.java} | 5 +- .../SqlDao.java => dao/source/JdbcDao.java} | 18 +- .../common => dao/source}/package-info.java | 4 +- .../{utils => dao/spanner}/SpannerDao.java | 3 +- .../templates/dao/spanner/package-info.java | 18 + .../{source/common => dml}/IDMLGenerator.java | 7 +- .../sql/mysql => dml}/MySQLDMLGenerator.java | 32 +- .../v2/templates/dml/package-info.java | 18 + .../models/ConnectionHelperRequest.java | 43 +++ .../templates/models/DMLGeneratorRequest.java | 96 +++++ .../models/DMLGeneratorResponse.java | 32 ++ .../v2/templates/models/package-info.java | 18 + .../templates/processor/SourceProcessor.java | 63 ++++ .../processor/SourceProcessorFactory.java | 105 ++++++ .../v2/templates/processor/package-info.java | 18 + .../source/common/DMLGeneratorRequest.java | 71 ---- .../source/common/SourceProcessor.java | 61 ---- .../source/common/SourceProcessorFactory.java | 79 ----- .../templates/transforms/SourceWriterFn.java | 32 +- .../templates/utils/InputRecordProcessor.java | 22 +- .../{ => connection}/ConnectionException.java | 2 +- .../utils/connection/IConnectionHelper.java | 24 ++ .../connection/MySQLConnectionHelper.java} | 62 ++-- .../connection}/package-info.java | 4 +- .../SqlDaoTest.java => dao/JdbcDaoTest.java} | 17 +- .../{utils => dao}/SpannerDaoTest.java | 4 +- .../mysql => dml}/MySQLDMLGeneratorTest.java | 329 ++++++++++++------ .../common/DMLGeneratorRequestTest.java | 65 ---- .../common/SourceProcessorFactoryTest.java | 103 ------ .../transforms/SourceWriterFnTest.java | 38 +- 32 files changed, 795 insertions(+), 603 deletions(-) rename v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/{source/sql => dao}/package-info.java (86%) rename v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/{source/common/ISourceDao.java => dao/source/IDao.java} (89%) rename v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/{source/sql/SqlDao.java => dao/source/JdbcDao.java} (71%) rename v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/{source/common => dao/source}/package-info.java (85%) rename v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/{utils => dao/spanner}/SpannerDao.java (95%) create mode 100644 v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dao/spanner/package-info.java rename v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/{source/common => dml}/IDMLGenerator.java (69%) rename v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/{source/sql/mysql => dml}/MySQLDMLGenerator.java (94%) create mode 100644 v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dml/package-info.java create mode 100644 v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/models/ConnectionHelperRequest.java create mode 100644 v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/models/DMLGeneratorRequest.java create mode 100644 v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/models/DMLGeneratorResponse.java create mode 100644 v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/models/package-info.java create mode 100644 v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/processor/SourceProcessor.java create mode 100644 v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/processor/SourceProcessorFactory.java create mode 100644 v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/processor/package-info.java delete mode 100644 v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/source/common/DMLGeneratorRequest.java delete mode 100644 v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/source/common/SourceProcessor.java delete mode 100644 v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/source/common/SourceProcessorFactory.java rename v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/{ => connection}/ConnectionException.java (93%) create mode 100644 v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/connection/IConnectionHelper.java rename v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/{source/sql/SQLConnectionHelper.java => utils/connection/MySQLConnectionHelper.java} (56%) rename v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/{source/sql/mysql => utils/connection}/package-info.java (84%) rename v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/{source/sql/SqlDaoTest.java => dao/JdbcDaoTest.java} (75%) rename v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/{utils => dao}/SpannerDaoTest.java (95%) rename v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/{source/sql/mysql => dml}/MySQLDMLGeneratorTest.java (78%) delete mode 100644 v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/source/common/DMLGeneratorRequestTest.java delete mode 100644 v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/source/common/SourceProcessorFactoryTest.java diff --git a/pom.xml b/pom.xml index d196736d7b..5e3f942820 100644 --- a/pom.xml +++ b/pom.xml @@ -367,6 +367,7 @@ **/CustomTransformationImplFetcher.* **/JarFileReader.* **/CustomTransformationWithShardFor*IT.* + **/models/* diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/source/sql/package-info.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dao/package-info.java similarity index 86% rename from v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/source/sql/package-info.java rename to v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dao/package-info.java index dc47f6f7dd..61fcc6af59 100644 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/source/sql/package-info.java +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dao/package-info.java @@ -14,5 +14,5 @@ * the License. */ -/** Package info for sql module. */ -package com.google.cloud.teleport.v2.templates.source.sql; +/** Package info for dao module. */ +package com.google.cloud.teleport.v2.templates.dao; diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/source/common/ISourceDao.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dao/source/IDao.java similarity index 89% rename from v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/source/common/ISourceDao.java rename to v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dao/source/IDao.java index c1450e778b..e761cce69d 100644 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/source/common/ISourceDao.java +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dao/source/IDao.java @@ -13,10 +13,9 @@ * License for the specific language governing permissions and limitations under * the License. */ -package com.google.cloud.teleport.v2.templates.source.common; - -public interface ISourceDao { +package com.google.cloud.teleport.v2.templates.dao.source; +public interface IDao { /** * Executes a given write statement. * diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/source/sql/SqlDao.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dao/source/JdbcDao.java similarity index 71% rename from v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/source/sql/SqlDao.java rename to v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dao/source/JdbcDao.java index 30c9e9246e..102f6b50bc 100644 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/source/sql/SqlDao.java +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dao/source/JdbcDao.java @@ -13,36 +13,38 @@ * License for the specific language governing permissions and limitations under * the License. */ -package com.google.cloud.teleport.v2.templates.source.sql; +package com.google.cloud.teleport.v2.templates.dao.source; -import com.google.cloud.teleport.v2.templates.source.common.ISourceDao; -import com.google.cloud.teleport.v2.templates.utils.ConnectionException; +import com.google.cloud.teleport.v2.templates.utils.connection.ConnectionException; +import com.google.cloud.teleport.v2.templates.utils.connection.IConnectionHelper; import java.sql.Connection; import java.sql.SQLException; import java.sql.Statement; -public class SqlDao implements ISourceDao { +public class JdbcDao implements IDao { private String sqlUrl; private String sqlUser; - private String sqlPasswd; - public SqlDao(String sqlUrl, String sqlUser, String sqlPasswd) { + private final IConnectionHelper connectionHelper; + + public JdbcDao(String sqlUrl, String sqlUser, IConnectionHelper connectionHelper) { this.sqlUrl = sqlUrl; this.sqlUser = sqlUser; - this.sqlPasswd = sqlPasswd; + this.connectionHelper = connectionHelper; } public String getSourceConnectionUrl() { return sqlUrl; } + @Override public void write(String sqlStatement) throws SQLException, ConnectionException { Connection connObj = null; Statement statement = null; try { - connObj = SQLConnectionHelper.getConnection(this.sqlUrl, this.sqlUser, this.sqlPasswd); + connObj = (Connection) connectionHelper.getConnection(this.sqlUrl + "/" + this.sqlUser); if (connObj == null) { throw new ConnectionException("Connection is null"); } diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/source/common/package-info.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dao/source/package-info.java similarity index 85% rename from v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/source/common/package-info.java rename to v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dao/source/package-info.java index a585ed54a5..e3fb254651 100644 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/source/common/package-info.java +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dao/source/package-info.java @@ -14,5 +14,5 @@ * the License. */ -/** Package info for common module. */ -package com.google.cloud.teleport.v2.templates.source.common; +/** Package info for source module. */ +package com.google.cloud.teleport.v2.templates.dao.source; diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/SpannerDao.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dao/spanner/SpannerDao.java similarity index 95% rename from v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/SpannerDao.java rename to v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dao/spanner/SpannerDao.java index e0283f9619..8638851f12 100644 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/SpannerDao.java +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dao/spanner/SpannerDao.java @@ -13,12 +13,13 @@ * License for the specific language governing permissions and limitations under * the License. */ -package com.google.cloud.teleport.v2.templates.utils; +package com.google.cloud.teleport.v2.templates.dao.spanner; import com.google.cloud.spanner.DatabaseClient; import com.google.cloud.spanner.Mutation; import com.google.cloud.spanner.Struct; import com.google.cloud.teleport.v2.templates.constants.Constants; +import com.google.cloud.teleport.v2.templates.utils.ShadowTableRecord; import java.util.ArrayList; import java.util.Arrays; import java.util.List; diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dao/spanner/package-info.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dao/spanner/package-info.java new file mode 100644 index 0000000000..6ec95edbed --- /dev/null +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dao/spanner/package-info.java @@ -0,0 +1,18 @@ +/* + * Copyright (C) 2023 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +/** Package info for spanner module. */ +package com.google.cloud.teleport.v2.templates.dao.spanner; diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/source/common/IDMLGenerator.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dml/IDMLGenerator.java similarity index 69% rename from v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/source/common/IDMLGenerator.java rename to v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dml/IDMLGenerator.java index 89b72837ef..3dc34a240b 100644 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/source/common/IDMLGenerator.java +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dml/IDMLGenerator.java @@ -13,8 +13,11 @@ * License for the specific language governing permissions and limitations under * the License. */ -package com.google.cloud.teleport.v2.templates.source.common; +package com.google.cloud.teleport.v2.templates.dml; + +import com.google.cloud.teleport.v2.templates.models.DMLGeneratorRequest; +import com.google.cloud.teleport.v2.templates.models.DMLGeneratorResponse; public interface IDMLGenerator { - String getDMLStatement(DMLGeneratorRequest dmlGeneratorRequest); + DMLGeneratorResponse getDMLStatement(DMLGeneratorRequest dmlGeneratorRequest); } diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/source/sql/mysql/MySQLDMLGenerator.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dml/MySQLDMLGenerator.java similarity index 94% rename from v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/source/sql/mysql/MySQLDMLGenerator.java rename to v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dml/MySQLDMLGenerator.java index b0a094ad07..0cea58f276 100644 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/source/sql/mysql/MySQLDMLGenerator.java +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dml/MySQLDMLGenerator.java @@ -13,15 +13,15 @@ * License for the specific language governing permissions and limitations under * the License. */ -package com.google.cloud.teleport.v2.templates.source.sql.mysql; +package com.google.cloud.teleport.v2.templates.dml; import com.google.cloud.teleport.v2.spanner.migrations.schema.ColumnPK; import com.google.cloud.teleport.v2.spanner.migrations.schema.SourceColumnDefinition; import com.google.cloud.teleport.v2.spanner.migrations.schema.SourceTable; import com.google.cloud.teleport.v2.spanner.migrations.schema.SpannerColumnDefinition; import com.google.cloud.teleport.v2.spanner.migrations.schema.SpannerTable; -import com.google.cloud.teleport.v2.templates.source.common.DMLGeneratorRequest; -import com.google.cloud.teleport.v2.templates.source.common.IDMLGenerator; +import com.google.cloud.teleport.v2.templates.models.DMLGeneratorRequest; +import com.google.cloud.teleport.v2.templates.models.DMLGeneratorResponse; import java.util.HashMap; import java.util.Map; import java.util.Set; @@ -35,7 +35,7 @@ public class MySQLDMLGenerator implements IDMLGenerator { private static final Logger LOG = LoggerFactory.getLogger(MySQLDMLGenerator.class); - public String getDMLStatement(DMLGeneratorRequest dmlGeneratorRequest) { + public DMLGeneratorResponse getDMLStatement(DMLGeneratorRequest dmlGeneratorRequest) { if (dmlGeneratorRequest .getSchema() @@ -45,7 +45,7 @@ public String getDMLStatement(DMLGeneratorRequest dmlGeneratorRequest) { LOG.warn( "The spanner table {} was not found in session file, dropping the record", dmlGeneratorRequest.getSpannerTableName()); - return ""; + return new DMLGeneratorResponse(""); } String spannerTableId = @@ -60,20 +60,20 @@ public String getDMLStatement(DMLGeneratorRequest dmlGeneratorRequest) { LOG.warn( "The spanner table {} was not found in session file, dropping the record", dmlGeneratorRequest.getSpannerTableName()); - return ""; + return new DMLGeneratorResponse(""); } SourceTable sourceTable = dmlGeneratorRequest.getSchema().getSrcSchema().get(spannerTableId); if (sourceTable == null) { LOG.warn("The table {} was not found in source", dmlGeneratorRequest.getSpannerTableName()); - return ""; + return new DMLGeneratorResponse(""); } if (sourceTable.getPrimaryKeys() == null || sourceTable.getPrimaryKeys().length == 0) { LOG.warn( "Cannot reverse replicate for table {} without primary key, skipping the record", sourceTable.getName()); - return ""; + return new DMLGeneratorResponse(""); } if ("INSERT".equals(dmlGeneratorRequest.getModType()) @@ -89,7 +89,7 @@ public String getDMLStatement(DMLGeneratorRequest dmlGeneratorRequest) { LOG.warn( "Cannot reverse replicate for table {} without primary key, skipping the record", sourceTable.getName()); - return ""; + return new DMLGeneratorResponse(""); } Map columnNameValues = getColumnValues( @@ -116,16 +116,16 @@ public String getDMLStatement(DMLGeneratorRequest dmlGeneratorRequest) { LOG.warn( "Cannot reverse replicate for table {} without primary key, skipping the record", sourceTable.getName()); - return ""; + return new DMLGeneratorResponse(""); } return getDeleteStatement(sourceTable.getName(), pkcolumnNameValues); } else { LOG.warn("Unsupported modType: " + dmlGeneratorRequest.getModType()); - return ""; + return new DMLGeneratorResponse(""); } } - private static String getUpsertStatement( + private static DMLGeneratorResponse getUpsertStatement( String tableName, Set primaryKeys, Map columnNameValues, @@ -150,7 +150,7 @@ private static String getUpsertStatement( String returnVal = "INSERT INTO `" + tableName + "`(" + allColumns + ")" + " VALUES (" + allValues + ") "; - return returnVal; + return new DMLGeneratorResponse(returnVal); } int index = 0; @@ -182,10 +182,10 @@ private static String getUpsertStatement( + "ON DUPLICATE KEY UPDATE " + updateValues; - return returnVal; + return new DMLGeneratorResponse(returnVal); } - private static String getDeleteStatement( + private static DMLGeneratorResponse getDeleteStatement( String tableName, Map pkcolumnNameValues) { String deleteValues = ""; @@ -202,7 +202,7 @@ private static String getDeleteStatement( } String returnVal = "DELETE FROM `" + tableName + "` WHERE " + deleteValues; - return returnVal; + return new DMLGeneratorResponse(returnVal); } private static Map getColumnValues( diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dml/package-info.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dml/package-info.java new file mode 100644 index 0000000000..e0cc0c996c --- /dev/null +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dml/package-info.java @@ -0,0 +1,18 @@ +/* + * Copyright (C) 2023 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +/** Package info for dml module. */ +package com.google.cloud.teleport.v2.templates.dml; diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/models/ConnectionHelperRequest.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/models/ConnectionHelperRequest.java new file mode 100644 index 0000000000..6e1817695e --- /dev/null +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/models/ConnectionHelperRequest.java @@ -0,0 +1,43 @@ +/* + * Copyright (C) 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.templates.models; + +import com.google.cloud.teleport.v2.spanner.migrations.shard.Shard; +import java.util.List; + +public class ConnectionHelperRequest { + private List shards; + private String properties; + private int maxConnections; + + public List getShards() { + return shards; + } + + public String getProperties() { + return properties; + } + + public int getMaxConnections() { + return maxConnections; + } + + public ConnectionHelperRequest(List shards, String properties, int maxConnections) { + this.shards = shards; + this.properties = properties; + this.maxConnections = maxConnections; + } +} diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/models/DMLGeneratorRequest.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/models/DMLGeneratorRequest.java new file mode 100644 index 0000000000..f6615b9d35 --- /dev/null +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/models/DMLGeneratorRequest.java @@ -0,0 +1,96 @@ +/* + * Copyright (C) 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.templates.models; + +import com.google.cloud.teleport.v2.spanner.migrations.schema.Schema; +import org.json.JSONObject; + +/** + * A request object representing the data necessary to generate DML statements for interacting with + * a source database. + */ +public class DMLGeneratorRequest { + private final String modType; + private final String spannerTableName; + private final Schema schema; + private final JSONObject newValuesJson; + private final JSONObject keyValuesJson; + private final String sourceDbTimezoneOffset; + + public DMLGeneratorRequest(Builder builder) { + this.modType = builder.modType; + this.spannerTableName = builder.spannerTableName; + this.schema = builder.schema; + this.newValuesJson = builder.newValuesJson; + this.keyValuesJson = builder.keyValuesJson; + this.sourceDbTimezoneOffset = builder.sourceDbTimezoneOffset; + } + + public String getModType() { + return modType; + } + + public String getSpannerTableName() { + return spannerTableName; + } + + public Schema getSchema() { + return schema; + } + + public JSONObject getNewValuesJson() { + return newValuesJson; + } + + public JSONObject getKeyValuesJson() { + return keyValuesJson; + } + + public String getSourceDbTimezoneOffset() { + return sourceDbTimezoneOffset; + } + + public static class Builder { + private final String modType; + private final String spannerTableName; + private final JSONObject newValuesJson; + private final JSONObject keyValuesJson; + private final String sourceDbTimezoneOffset; + private Schema schema; + + public Builder( + String modType, + String spannerTableName, + JSONObject newValuesJson, + JSONObject keyValuesJson, + String sourceDbTimezoneOffset) { + this.modType = modType; + this.spannerTableName = spannerTableName; + this.newValuesJson = newValuesJson; + this.keyValuesJson = keyValuesJson; + this.sourceDbTimezoneOffset = sourceDbTimezoneOffset; + } + + public Builder setSchema(Schema schema) { + this.schema = schema; + return this; + } + + public DMLGeneratorRequest build() { + return new DMLGeneratorRequest(this); + } + } +} diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/models/DMLGeneratorResponse.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/models/DMLGeneratorResponse.java new file mode 100644 index 0000000000..42479a362d --- /dev/null +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/models/DMLGeneratorResponse.java @@ -0,0 +1,32 @@ +/* + * Copyright (C) 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.templates.models; + +public class DMLGeneratorResponse { + private String dmlStatement; + + public String getDmlStatement() { + return dmlStatement; + } + + public void setDmlStatement(String dmlStatement) { + this.dmlStatement = dmlStatement; + } + + public DMLGeneratorResponse(String dmlStatement) { + this.dmlStatement = dmlStatement; + } +} diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/models/package-info.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/models/package-info.java new file mode 100644 index 0000000000..d49a3a3fe9 --- /dev/null +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/models/package-info.java @@ -0,0 +1,18 @@ +/* + * Copyright (C) 2023 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +/** Package info for models module. */ +package com.google.cloud.teleport.v2.templates.models; diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/processor/SourceProcessor.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/processor/SourceProcessor.java new file mode 100644 index 0000000000..173be89c3e --- /dev/null +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/processor/SourceProcessor.java @@ -0,0 +1,63 @@ +/* + * Copyright (C) 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.templates.processor; + +import com.google.cloud.teleport.v2.templates.dao.source.IDao; +import com.google.cloud.teleport.v2.templates.dml.IDMLGenerator; +import java.util.HashMap; +import java.util.Map; + +public class SourceProcessor { + + private final IDMLGenerator dmlGenerator; + private final Map sourceDaoMap; + + private SourceProcessor(IDMLGenerator dmlGenerator, Map sourceDaoMap) { + this.dmlGenerator = dmlGenerator; + this.sourceDaoMap = sourceDaoMap; + } + + public IDMLGenerator getDmlGenerator() { + return dmlGenerator; + } + + public Map getSourceDaoMap() { + return sourceDaoMap; + } + + public static Builder builder() { + return new Builder(); + } + + public static class Builder { + private IDMLGenerator dmlGenerator; + private Map sourceDaoMap = new HashMap<>(); + + public Builder dmlGenerator(IDMLGenerator dmlGenerator) { + this.dmlGenerator = dmlGenerator; + return this; + } + + public Builder sourceDaoMap(Map sourceDaoMap) { + this.sourceDaoMap = sourceDaoMap; + return this; + } + + public SourceProcessor build() { + return new SourceProcessor(dmlGenerator, sourceDaoMap); + } + } +} diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/processor/SourceProcessorFactory.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/processor/SourceProcessorFactory.java new file mode 100644 index 0000000000..24ce9b9eaf --- /dev/null +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/processor/SourceProcessorFactory.java @@ -0,0 +1,105 @@ +/* + * Copyright (C) 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.templates.processor; + +import com.google.cloud.teleport.v2.spanner.migrations.shard.Shard; +import com.google.cloud.teleport.v2.templates.constants.Constants; +import com.google.cloud.teleport.v2.templates.dao.source.IDao; +import com.google.cloud.teleport.v2.templates.dao.source.JdbcDao; +import com.google.cloud.teleport.v2.templates.dml.IDMLGenerator; +import com.google.cloud.teleport.v2.templates.dml.MySQLDMLGenerator; +import com.google.cloud.teleport.v2.templates.models.ConnectionHelperRequest; +import com.google.cloud.teleport.v2.templates.utils.connection.IConnectionHelper; +import com.google.cloud.teleport.v2.templates.utils.connection.MySQLConnectionHelper; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.function.Function; + +public class SourceProcessorFactory { + private static final Map DML_GENERATOR_MAP = + Map.of(Constants.SOURCE_MYSQL, new MySQLDMLGenerator()); + + private static final Map CONNECTION_HELPER_MAP = + Map.of(Constants.SOURCE_MYSQL, new MySQLConnectionHelper()); + + private static final Map DRIVER_MAP = + Map.of(Constants.SOURCE_MYSQL, "com.mysql.cj.jdbc.Driver"); + + private static final Map> CONNECTION_URL_GENERATORS = + Map.of( + Constants.SOURCE_MYSQL, + shard -> + "jdbc:mysql://" + shard.getHost() + ":" + shard.getPort() + "/" + shard.getDbName()); + + /** + * Creates a SourceProcessor instance for the specified source type. + * + * @param source the type of the source database + * @param shards the list of shards for the source + * @param maxConnections the maximum number of connections + * @return a configured SourceProcessor instance + * @throws Exception if the source type is invalid + */ + public static SourceProcessor createSourceProcessor( + String source, List shards, int maxConnections) throws Exception { + IDMLGenerator dmlGenerator = getDMLGenerator(source); + String driver = getDriver(source); + IConnectionHelper connectionHelper = + initializeConnectionHelper(source, shards, maxConnections, driver); + Map sourceDaoMap = createSourceDaoMap(source, shards, connectionHelper); + + return SourceProcessor.builder().dmlGenerator(dmlGenerator).sourceDaoMap(sourceDaoMap).build(); + } + + private static IDMLGenerator getDMLGenerator(String source) throws Exception { + return Optional.ofNullable(DML_GENERATOR_MAP.get(source)) + .orElseThrow(() -> new Exception("Invalid source type for DML generator: " + source)); + } + + private static String getDriver(String source) throws Exception { + return Optional.ofNullable(DRIVER_MAP.get(source)) + .orElseThrow(() -> new Exception("Invalid source type for driver: " + source)); + } + + private static IConnectionHelper getConnectionHelper(String source) throws Exception { + return Optional.ofNullable(CONNECTION_HELPER_MAP.get(source)) + .orElseThrow(() -> new Exception("Invalid source type for connection helper: " + source)); + } + + private static IConnectionHelper initializeConnectionHelper( + String source, List shards, int maxConnections, String driver) throws Exception { + IConnectionHelper connectionHelper = getConnectionHelper(source); + connectionHelper.init(new ConnectionHelperRequest(shards, null, maxConnections)); + return connectionHelper; + } + + private static Map createSourceDaoMap( + String source, List shards, IConnectionHelper connectionHelper) throws Exception { + Function urlGenerator = + Optional.ofNullable(CONNECTION_URL_GENERATORS.get(source)) + .orElseThrow(() -> new Exception("Invalid source type for URL generation: " + source)); + + Map sourceDaoMap = new HashMap<>(); + for (Shard shard : shards) { + String connectionUrl = urlGenerator.apply(shard); + IDao sqlDao = new JdbcDao(connectionUrl, shard.getUserName(), connectionHelper); + sourceDaoMap.put(shard.getLogicalShardId(), sqlDao); + } + return sourceDaoMap; + } +} diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/processor/package-info.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/processor/package-info.java new file mode 100644 index 0000000000..80fe4035aa --- /dev/null +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/processor/package-info.java @@ -0,0 +1,18 @@ +/* + * Copyright (C) 2023 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +/** Package info for processor module. */ +package com.google.cloud.teleport.v2.templates.processor; diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/source/common/DMLGeneratorRequest.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/source/common/DMLGeneratorRequest.java deleted file mode 100644 index 50ea236e50..0000000000 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/source/common/DMLGeneratorRequest.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright (C) 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ -package com.google.cloud.teleport.v2.templates.source.common; - -import com.google.cloud.teleport.v2.spanner.migrations.schema.Schema; -import org.json.JSONObject; - -/** - * A request object representing the data necessary to generate DML - * statements for interacting with a source database. - */ -public class DMLGeneratorRequest { - private String modType; - private String spannerTableName; - private Schema schema; - private JSONObject newValuesJson; - private JSONObject keyValuesJson; - private String sourceDbTimezoneOffset; - - public String getModType() { - return modType; - } - - public String getSpannerTableName() { - return spannerTableName; - } - - public Schema getSchema() { - return schema; - } - - public JSONObject getNewValuesJson() { - return newValuesJson; - } - - public JSONObject getKeyValuesJson() { - return keyValuesJson; - } - - public String getSourceDbTimezoneOffset() { - return sourceDbTimezoneOffset; - } - - public DMLGeneratorRequest( - String modType, - String spannerTableName, - Schema schema, - JSONObject newValuesJson, - JSONObject keyValuesJson, - String sourceDbTimezoneOffset) { - this.modType = modType; - this.spannerTableName = spannerTableName; - this.schema = schema; - this.newValuesJson = newValuesJson; - this.keyValuesJson = keyValuesJson; - this.sourceDbTimezoneOffset = sourceDbTimezoneOffset; - } -} diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/source/common/SourceProcessor.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/source/common/SourceProcessor.java deleted file mode 100644 index 33a5ae7232..0000000000 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/source/common/SourceProcessor.java +++ /dev/null @@ -1,61 +0,0 @@ -package com.google.cloud.teleport.v2.templates.source.common; - -import com.google.cloud.teleport.v2.templates.source.sql.SQLConnectionHelper; - -import java.util.HashMap; -import java.util.Map; - -public class SourceProcessor { - - private final IDMLGenerator dmlGenerator; - private final Map sourceDaoMap; - private final SQLConnectionHelper connectionHelper; - - private SourceProcessor(IDMLGenerator dmlGenerator, Map sourceDaoMap, SQLConnectionHelper connectionHelper) { - this.dmlGenerator = dmlGenerator; - this.sourceDaoMap = sourceDaoMap; - this.connectionHelper = connectionHelper; - } - - public IDMLGenerator getDmlGenerator() { - return dmlGenerator; - } - - public Map getSourceDaoMap() { - return sourceDaoMap; - } - - public SQLConnectionHelper getConnectionHelper() { - return connectionHelper; - } - - public static Builder builder() { - return new Builder(); - } - - public static class Builder { - private IDMLGenerator dmlGenerator; - private Map sourceDaoMap = new HashMap<>(); - private SQLConnectionHelper connectionHelper; - - public Builder dmlGenerator(IDMLGenerator dmlGenerator) { - this.dmlGenerator = dmlGenerator; - return this; - } - - public Builder sourceDaoMap(Map sourceDaoMap) { - this.sourceDaoMap = sourceDaoMap; - return this; - } - - public Builder connectionHelper(SQLConnectionHelper connectionHelper) { - this.connectionHelper = connectionHelper; - return this; - } - - public SourceProcessor build() { - return new SourceProcessor(dmlGenerator, sourceDaoMap, connectionHelper); - } - } -} - diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/source/common/SourceProcessorFactory.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/source/common/SourceProcessorFactory.java deleted file mode 100644 index bdd1d2ae4e..0000000000 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/source/common/SourceProcessorFactory.java +++ /dev/null @@ -1,79 +0,0 @@ -package com.google.cloud.teleport.v2.templates.source.common; - -import com.google.cloud.teleport.v2.spanner.migrations.shard.Shard; -import com.google.cloud.teleport.v2.templates.constants.Constants; -import com.google.cloud.teleport.v2.templates.source.sql.SQLConnectionHelper; -import com.google.cloud.teleport.v2.templates.source.sql.SqlDao; -import com.google.cloud.teleport.v2.templates.source.sql.mysql.MySQLDMLGenerator; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.function.Function; - -public class SourceProcessorFactory { - private static final Map DML_GENERATOR_MAP = Map.of( - Constants.SOURCE_MYSQL, new MySQLDMLGenerator() - ); - - private static final Map DRIVER_MAP = Map.of( - Constants.SOURCE_MYSQL, "com.mysql.cj.jdbc.Driver" - ); - - private static final Map> CONNECTION_URL_GENERATORS = Map.of( - Constants.SOURCE_MYSQL, shard -> - "jdbc:mysql://" + shard.getHost() + ":" + shard.getPort() + "/" + shard.getDbName() - ); - - /** - * Creates a SourceProcessor instance for the specified source type. - * - * @param source the type of the source database - * @param shards the list of shards for the source - * @param maxConnections the maximum number of connections - * @return a configured SourceProcessor instance - * @throws Exception if the source type is invalid - */ - public static SourceProcessor createSourceProcessor(String source, List shards, int maxConnections) throws Exception { - IDMLGenerator dmlGenerator = getDMLGenerator(source); - String driver = getDriver(source); - SQLConnectionHelper connectionHelper = initializeConnectionHelper(source, shards, maxConnections, driver); - Map sourceDaoMap = createSourceDaoMap(source, shards); - - return SourceProcessor.builder() - .dmlGenerator(dmlGenerator) - .sourceDaoMap(sourceDaoMap) - .connectionHelper(connectionHelper) - .build(); - } - - private static IDMLGenerator getDMLGenerator(String source) throws Exception { - return Optional.ofNullable(DML_GENERATOR_MAP.get(source)) - .orElseThrow(() -> new Exception("Invalid source type for DML generator: " + source)); - } - - private static String getDriver(String source) throws Exception { - return Optional.ofNullable(DRIVER_MAP.get(source)) - .orElseThrow(() -> new Exception("Invalid source type for driver: " + source)); - } - - private static SQLConnectionHelper initializeConnectionHelper( - String source, List shards, int maxConnections, String driver) throws Exception { - SQLConnectionHelper connectionHelper = new SQLConnectionHelper(); - connectionHelper.init(shards, null, maxConnections, source, driver); - return connectionHelper; - } - - private static Map createSourceDaoMap(String source, List shards) throws Exception { - Function urlGenerator = Optional.ofNullable(CONNECTION_URL_GENERATORS.get(source)) - .orElseThrow(() -> new Exception("Invalid source type for URL generation: " + source)); - - Map sourceDaoMap = new HashMap<>(); - for (Shard shard : shards) { - String connectionUrl = urlGenerator.apply(shard); - ISourceDao sqlDao = new SqlDao(connectionUrl, shard.getUserName(), shard.getPassword()); - sourceDaoMap.put(shard.getLogicalShardId(), sqlDao); - } - return sourceDaoMap; - } -} diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/transforms/SourceWriterFn.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/transforms/SourceWriterFn.java index 638a9d5fa5..8844ea14f3 100644 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/transforms/SourceWriterFn.java +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/transforms/SourceWriterFn.java @@ -30,21 +30,18 @@ import com.google.cloud.teleport.v2.templates.changestream.ChangeStreamErrorRecord; import com.google.cloud.teleport.v2.templates.changestream.TrimmedShardedDataChangeRecord; import com.google.cloud.teleport.v2.templates.constants.Constants; -import com.google.cloud.teleport.v2.templates.source.common.IDMLGenerator; -import com.google.cloud.teleport.v2.templates.source.common.ISourceDao; -import com.google.cloud.teleport.v2.templates.source.common.SourceProcessor; -import com.google.cloud.teleport.v2.templates.source.common.SourceProcessorFactory; -import com.google.cloud.teleport.v2.templates.utils.ConnectionException; +import com.google.cloud.teleport.v2.templates.dao.source.IDao; +import com.google.cloud.teleport.v2.templates.dao.spanner.SpannerDao; +import com.google.cloud.teleport.v2.templates.processor.SourceProcessor; +import com.google.cloud.teleport.v2.templates.processor.SourceProcessorFactory; import com.google.cloud.teleport.v2.templates.utils.InputRecordProcessor; import com.google.cloud.teleport.v2.templates.utils.ShadowTableRecord; -import com.google.cloud.teleport.v2.templates.utils.SpannerDao; +import com.google.cloud.teleport.v2.templates.utils.connection.ConnectionException; import com.google.common.collect.ImmutableList; import com.google.gson.Gson; import java.io.Serializable; -import java.util.HashMap; import java.util.HashSet; import java.util.List; -import java.util.Map; import java.util.Set; import java.util.stream.Collectors; import org.apache.beam.sdk.io.gcp.spanner.SpannerConfig; @@ -116,20 +113,24 @@ public void setSpannerDao(SpannerDao spannerDao) { this.spannerDao = spannerDao; } - // for unit testing purposes public void setObjectMapper(ObjectMapper mapper) { this.mapper = mapper; } + // for unit testing purposes + public void setSourceProcessor(SourceProcessor sourceProcessor) { + this.sourceProcessor = sourceProcessor; + } + /** Setup function connects to Cloud Spanner. */ @Setup public void setup() throws Exception { mapper = new ObjectMapper(); mapper.enable(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS); - sourceProcessor= SourceProcessorFactory.createSourceProcessor(source, shards, maxThreadPerDataflowWorker); + sourceProcessor = + SourceProcessorFactory.createSourceProcessor(source, shards, maxThreadPerDataflowWorker); spannerDao = new SpannerDao(spannerConfig); - } /** Teardown function disconnects from the Cloud Spanner. */ @@ -179,11 +180,16 @@ public void processElement(ProcessContext c) { > Long.parseLong(spannerRec.getRecordSequence()))); if (!isSourceAhead) { - ISourceDao sourceDao = sourceProcessor.getSourceDaoMap().get(shardId); + IDao sourceDao = sourceProcessor.getSourceDaoMap().get(shardId); InputRecordProcessor inputRecordProcessor = new InputRecordProcessor(); inputRecordProcessor.processRecord( - spannerRec, schema, sourceDao, shardId, sourceDbTimezoneOffset, source, sourceProcessor.getDmlGenerator()); + spannerRec, + schema, + sourceDao, + shardId, + sourceDbTimezoneOffset, + sourceProcessor.getDmlGenerator()); spannerDao.updateShadowTable( getShadowTableMutation( diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/InputRecordProcessor.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/InputRecordProcessor.java index 7304b83e34..c7fccd630a 100644 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/InputRecordProcessor.java +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/InputRecordProcessor.java @@ -17,9 +17,10 @@ import com.google.cloud.teleport.v2.spanner.migrations.schema.Schema; import com.google.cloud.teleport.v2.templates.changestream.TrimmedShardedDataChangeRecord; -import com.google.cloud.teleport.v2.templates.source.common.DMLGeneratorRequest; -import com.google.cloud.teleport.v2.templates.source.common.IDMLGenerator; -import com.google.cloud.teleport.v2.templates.source.common.ISourceDao; +import com.google.cloud.teleport.v2.templates.dao.source.IDao; +import com.google.cloud.teleport.v2.templates.dml.IDMLGenerator; +import com.google.cloud.teleport.v2.templates.models.DMLGeneratorRequest; +import com.google.cloud.teleport.v2.templates.models.DMLGeneratorResponse; import java.time.Instant; import java.time.temporal.ChronoUnit; import org.apache.beam.sdk.metrics.Counter; @@ -38,10 +39,9 @@ public class InputRecordProcessor { public void processRecord( TrimmedShardedDataChangeRecord spannerRecord, Schema schema, - ISourceDao dao, + IDao dao, String shardId, String sourceDbTimezoneOffset, - String source, IDMLGenerator dmlGenerator) throws Exception { @@ -55,15 +55,17 @@ public void processRecord( JSONObject keysJson = new JSONObject(keysJsonStr); DMLGeneratorRequest dmlGeneratorRequest = - new DMLGeneratorRequest( - modType, tableName, schema, newValuesJson, keysJson, sourceDbTimezoneOffset); + new DMLGeneratorRequest.Builder( + modType, tableName, newValuesJson, keysJson, sourceDbTimezoneOffset) + .setSchema(schema) + .build(); - String dmlStatement = dmlGenerator.getDMLStatement(dmlGeneratorRequest); - if (dmlStatement.isEmpty()) { + DMLGeneratorResponse dmlGeneratorResponse = dmlGenerator.getDMLStatement(dmlGeneratorRequest); + if (dmlGeneratorResponse.getDmlStatement().isEmpty()) { LOG.warn("DML statement is empty for table: " + tableName); return; } - dao.write(dmlStatement); + dao.write(dmlGeneratorResponse.getDmlStatement()); Counter numRecProcessedMetric = Metrics.counter(shardId, "records_written_to_source_" + shardId); diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/ConnectionException.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/connection/ConnectionException.java similarity index 93% rename from v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/ConnectionException.java rename to v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/connection/ConnectionException.java index aa54c23308..3c93235dcc 100644 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/ConnectionException.java +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/connection/ConnectionException.java @@ -13,7 +13,7 @@ * License for the specific language governing permissions and limitations under * the License. */ -package com.google.cloud.teleport.v2.templates.utils; +package com.google.cloud.teleport.v2.templates.utils.connection; /** Exception when connecting to the source database. */ public class ConnectionException extends Exception { diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/connection/IConnectionHelper.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/connection/IConnectionHelper.java new file mode 100644 index 0000000000..3cfe0b8c80 --- /dev/null +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/connection/IConnectionHelper.java @@ -0,0 +1,24 @@ +/* + * Copyright (C) 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.templates.utils.connection; + +import com.google.cloud.teleport.v2.templates.models.ConnectionHelperRequest; + +public interface IConnectionHelper { + void init(ConnectionHelperRequest connectionHelperRequest); + + T getConnection(String connectionRequestKey) throws ConnectionException; +} diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/source/sql/SQLConnectionHelper.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/connection/MySQLConnectionHelper.java similarity index 56% rename from v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/source/sql/SQLConnectionHelper.java rename to v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/connection/MySQLConnectionHelper.java index dd810fcce3..3f264e5b2a 100644 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/source/sql/SQLConnectionHelper.java +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/connection/MySQLConnectionHelper.java @@ -13,38 +13,39 @@ * License for the specific language governing permissions and limitations under * the License. */ -package com.google.cloud.teleport.v2.templates.source.sql; +package com.google.cloud.teleport.v2.templates.utils.connection; import com.google.cloud.teleport.v2.spanner.migrations.shard.Shard; +import com.google.cloud.teleport.v2.templates.models.ConnectionHelperRequest; import com.zaxxer.hikari.HikariConfig; import com.zaxxer.hikari.HikariDataSource; import java.io.IOException; import java.io.StringReader; import java.sql.Connection; import java.util.HashMap; -import java.util.List; import java.util.Map; import java.util.Properties; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** This is a per Dataflow worker singleton that holds connection pool. */ -public class SQLConnectionHelper { +public class MySQLConnectionHelper implements IConnectionHelper { - private static final Logger LOG = LoggerFactory.getLogger(SQLConnectionHelper.class); + private static final Logger LOG = LoggerFactory.getLogger(MySQLConnectionHelper.class); private static Map connectionPoolMap = null; + private static final String MYSQL_JDBC_DRIVER = "com.mysql.cj.jdbc.Driver"; - public static synchronized void init( - List shards, String properties, int maxConnections, String source, String jdbcDriver) { + @Override + public synchronized void init(ConnectionHelperRequest connectionHelperRequest) { if (connectionPoolMap != null) { return; } - LOG.info("Initializing connection pool with size: ", maxConnections); + LOG.info( + "Initializing connection pool with size: ", connectionHelperRequest.getMaxConnections()); connectionPoolMap = new HashMap<>(); - for (Shard shard : shards) { + for (Shard shard : connectionHelperRequest.getShards()) { String sourceConnectionUrl = - "jdbc:" - + source + "jdbc:mysql://" + "://" + shard.getHost() + ":" @@ -55,13 +56,14 @@ public static synchronized void init( config.setJdbcUrl(sourceConnectionUrl); config.setUsername(shard.getUserName()); config.setPassword(shard.getPassword()); - config.setDriverClassName(jdbcDriver); - config.setMaximumPoolSize(maxConnections); + config.setDriverClassName(MYSQL_JDBC_DRIVER); + config.setMaximumPoolSize(connectionHelperRequest.getMaxConnections()); config.setConnectionInitSql( "SET SESSION net_read_timeout=1200"); // to avoid timeouts at network level layer Properties jdbcProperties = new Properties(); - if (properties != null && !properties.isEmpty()) { - try (StringReader reader = new StringReader(properties)) { + if (connectionHelperRequest.getProperties() != null + && !connectionHelperRequest.getProperties().isEmpty()) { + try (StringReader reader = new StringReader(connectionHelperRequest.getProperties())) { jdbcProperties.load(reader); } catch (IOException e) { LOG.error("Error converting string to properties: {}", e.getMessage()); @@ -74,27 +76,31 @@ public static synchronized void init( } HikariDataSource ds = new HikariDataSource(config); - connectionPoolMap.put(sourceConnectionUrl + shard.getUserName() + shard.getPassword(), ds); + connectionPoolMap.put(sourceConnectionUrl + "/" + shard.getUserName(), ds); } } - public static Connection getConnection(String sourceConnectionUrl, String user, String password) - throws java.sql.SQLException { - if (connectionPoolMap == null) { - LOG.warn("Connection pool not initialized"); - return null; - } - HikariDataSource ds = connectionPoolMap.get(sourceConnectionUrl + user + password); - if (ds == null) { - LOG.warn("Connection pool not found for source connection url: {}", sourceConnectionUrl); - return null; - } + @Override + public Connection getConnection(String connectionRequestKey) throws ConnectionException { + try { + if (connectionPoolMap == null) { + LOG.warn("Connection pool not initialized"); + return null; + } + HikariDataSource ds = connectionPoolMap.get(connectionRequestKey); + if (ds == null) { + LOG.warn("Connection pool not found for source connection : {}", connectionRequestKey); + return null; + } - return ds.getConnection(); + return ds.getConnection(); + } catch (Exception e) { + throw new ConnectionException(e); + } } // for unit testing - public static void setConnectionPoolMap(Map inputMap) { + public void setConnectionPoolMap(Map inputMap) { connectionPoolMap = inputMap; } } diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/source/sql/mysql/package-info.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/connection/package-info.java similarity index 84% rename from v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/source/sql/mysql/package-info.java rename to v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/connection/package-info.java index e5da8f92db..0cb2b03850 100644 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/source/sql/mysql/package-info.java +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/connection/package-info.java @@ -14,5 +14,5 @@ * the License. */ -/** Package info for mysql source module. */ -package com.google.cloud.teleport.v2.templates.source.sql.mysql; +/** Package info for connection module. */ +package com.google.cloud.teleport.v2.templates.utils.connection; diff --git a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/source/sql/SqlDaoTest.java b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/dao/JdbcDaoTest.java similarity index 75% rename from v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/source/sql/SqlDaoTest.java rename to v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/dao/JdbcDaoTest.java index 743acf8812..7afddcd9e0 100644 --- a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/source/sql/SqlDaoTest.java +++ b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/dao/JdbcDaoTest.java @@ -13,7 +13,7 @@ * License for the specific language governing permissions and limitations under * the License. */ -package com.google.cloud.teleport.v2.templates.source.sql; +package com.google.cloud.teleport.v2.templates.dao; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; @@ -21,7 +21,9 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -import com.google.cloud.teleport.v2.templates.utils.ConnectionException; +import com.google.cloud.teleport.v2.templates.dao.source.JdbcDao; +import com.google.cloud.teleport.v2.templates.utils.connection.ConnectionException; +import com.google.cloud.teleport.v2.templates.utils.connection.MySQLConnectionHelper; import com.zaxxer.hikari.HikariDataSource; import java.sql.Connection; import java.sql.Statement; @@ -34,7 +36,7 @@ import org.mockito.junit.MockitoJUnit; import org.mockito.junit.MockitoRule; -public final class SqlDaoTest { +public final class JdbcDaoTest { @Rule public final MockitoRule mockito = MockitoJUnit.rule(); @Mock private HikariDataSource mockHikariDataSource; @@ -52,16 +54,17 @@ public void doBeforeEachTest() throws java.sql.SQLException { @Test(expected = ConnectionException.class) public void testNullConnection() throws java.sql.SQLException, ConnectionException { - SqlDao sqlDao = new SqlDao("url", "user", "pass"); + JdbcDao sqlDao = new JdbcDao("url", "user", new MySQLConnectionHelper()); sqlDao.write("sql"); } @Test public void testSuccess() throws java.sql.SQLException, ConnectionException { Map connectionPoolMap = new HashMap<>(); - connectionPoolMap.put("urluserpass", mockHikariDataSource); - SQLConnectionHelper.setConnectionPoolMap(connectionPoolMap); - SqlDao sqlDao = new SqlDao("url", "user", "pass"); + connectionPoolMap.put("url/user", mockHikariDataSource); + MySQLConnectionHelper mySQLConnectionHelper = new MySQLConnectionHelper(); + mySQLConnectionHelper.setConnectionPoolMap(connectionPoolMap); + JdbcDao sqlDao = new JdbcDao("url", "user", mySQLConnectionHelper); sqlDao.write("sql"); verify(mockStatement).executeUpdate(eq("sql")); } diff --git a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/utils/SpannerDaoTest.java b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/dao/SpannerDaoTest.java similarity index 95% rename from v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/utils/SpannerDaoTest.java rename to v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/dao/SpannerDaoTest.java index 2d3c0496ae..e191e23db0 100644 --- a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/utils/SpannerDaoTest.java +++ b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/dao/SpannerDaoTest.java @@ -13,7 +13,7 @@ * License for the specific language governing permissions and limitations under * the License. */ -package com.google.cloud.teleport.v2.templates.utils; +package com.google.cloud.teleport.v2.templates.dao; import static com.google.common.truth.Truth.assertThat; import static org.mockito.ArgumentMatchers.any; @@ -30,6 +30,8 @@ import com.google.cloud.spanner.ReadOnlyTransaction; import com.google.cloud.spanner.Struct; import com.google.cloud.teleport.v2.templates.constants.Constants; +import com.google.cloud.teleport.v2.templates.dao.spanner.SpannerDao; +import com.google.cloud.teleport.v2.templates.utils.ShadowTableRecord; import com.google.common.collect.ImmutableList; import org.apache.beam.sdk.io.gcp.spanner.SpannerAccessor; import org.junit.Before; diff --git a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/source/sql/mysql/MySQLDMLGeneratorTest.java b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/dml/MySQLDMLGeneratorTest.java similarity index 78% rename from v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/source/sql/mysql/MySQLDMLGeneratorTest.java rename to v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/dml/MySQLDMLGeneratorTest.java index 8a07911039..e07dd9eaf2 100644 --- a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/source/sql/mysql/MySQLDMLGeneratorTest.java +++ b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/dml/MySQLDMLGeneratorTest.java @@ -13,7 +13,7 @@ * License for the specific language governing permissions and limitations under * the License. */ -package com.google.cloud.teleport.v2.templates.source.sql.mysql; +package com.google.cloud.teleport.v2.templates.dml; import static org.junit.Assert.assertTrue; @@ -29,7 +29,8 @@ import com.google.cloud.teleport.v2.spanner.migrations.schema.SyntheticPKey; import com.google.cloud.teleport.v2.spanner.migrations.utils.SessionFileReader; import com.google.cloud.teleport.v2.templates.changestream.TrimmedShardedDataChangeRecord; -import com.google.cloud.teleport.v2.templates.source.common.DMLGeneratorRequest; +import com.google.cloud.teleport.v2.templates.models.DMLGeneratorRequest; +import com.google.cloud.teleport.v2.templates.models.DMLGeneratorResponse; import com.google.cloud.teleport.v2.templates.utils.InputRecordProcessor; import com.google.gson.FieldNamingPolicy; import com.google.gson.GsonBuilder; @@ -62,10 +63,13 @@ public void tableAndAllColumnNameTypesMatch() { "INSERT INTO Singers(SingerId,FirstName,LastName) VALUES (999,'kk','ll') ON DUPLICATE KEY" + " UPDATE FirstName = 'kk', LastName = 'll'";*/ MySQLDMLGenerator mySQLDMLGenerator = new MySQLDMLGenerator(); - String sql = + DMLGeneratorResponse dmlGeneratorResponse = mySQLDMLGenerator.getDMLStatement( - new DMLGeneratorRequest( - modType, tableName, schema, newValuesJson, keyValuesJson, "+00:00")); + new DMLGeneratorRequest.Builder( + modType, tableName, newValuesJson, keyValuesJson, "+00:00") + .setSchema(schema) + .build()); + String sql = dmlGeneratorResponse.getDmlStatement(); assertTrue(sql.contains("`FirstName` = 'kk'")); assertTrue(sql.contains("`LastName` = 'll'")); @@ -85,10 +89,13 @@ public void tableNameMismatchAllColumnNameTypesMatch() { "INSERT INTO Singers(SingerId,FirstName,LastName) VALUES (999,'kk','ll') ON DUPLICATE KEY" + " UPDATE FirstName = 'kk', LastName = 'll'";*/ MySQLDMLGenerator mySQLDMLGenerator = new MySQLDMLGenerator(); - String sql = + DMLGeneratorResponse dmlGeneratorResponse = mySQLDMLGenerator.getDMLStatement( - new DMLGeneratorRequest( - modType, tableName, schema, newValuesJson, keyValuesJson, "+00:00")); + new DMLGeneratorRequest.Builder( + modType, tableName, newValuesJson, keyValuesJson, "+00:00") + .setSchema(schema) + .build()); + String sql = dmlGeneratorResponse.getDmlStatement(); assertTrue(sql.contains("`FirstName` = 'kk'")); assertTrue(sql.contains("`LastName` = 'll'")); @@ -107,10 +114,13 @@ public void tableNameMatchColumnNameTypeMismatch() { "INSERT INTO Singers(SingerId,FirstName,LastName) VALUES ('999',222,'ll') ON DUPLICATE" + " KEY UPDATE FirstName = 222, LastName = 'll'";*/ MySQLDMLGenerator mySQLDMLGenerator = new MySQLDMLGenerator(); - String sql = + DMLGeneratorResponse dmlGeneratorResponse = mySQLDMLGenerator.getDMLStatement( - new DMLGeneratorRequest( - modType, tableName, schema, newValuesJson, keyValuesJson, "+00:00")); + new DMLGeneratorRequest.Builder( + modType, tableName, newValuesJson, keyValuesJson, "+00:00") + .setSchema(schema) + .build()); + String sql = dmlGeneratorResponse.getDmlStatement(); assertTrue(sql.contains("`FirstName` = 222")); assertTrue(sql.contains("`LastName` = 'll'")); @@ -131,10 +141,13 @@ public void tableNameMatchSourceColumnNotPresentInSpanner() { "INSERT INTO Singers(SingerId,FirstName,LastName) VALUES (999,'kk','ll') ON DUPLICATE KEY" + " UPDATE FirstName = 'kk', LastName = 'll'"; */ MySQLDMLGenerator mySQLDMLGenerator = new MySQLDMLGenerator(); - String sql = + DMLGeneratorResponse dmlGeneratorResponse = mySQLDMLGenerator.getDMLStatement( - new DMLGeneratorRequest( - modType, tableName, schema, newValuesJson, keyValuesJson, "+00:00")); + new DMLGeneratorRequest.Builder( + modType, tableName, newValuesJson, keyValuesJson, "+00:00") + .setSchema(schema) + .build()); + String sql = dmlGeneratorResponse.getDmlStatement(); assertTrue(sql.contains("`FirstName` = 'kk'")); assertTrue(sql.contains("`LastName` = 'll'")); @@ -156,10 +169,13 @@ public void tableNameMatchSpannerColumnNotPresentInSource() { "INSERT INTO Singers(SingerId,FirstName,LastName) VALUES (999,'kk','ll') ON DUPLICATE KEY" + " UPDATE FirstName = 'kk', LastName = 'll'";*/ MySQLDMLGenerator mySQLDMLGenerator = new MySQLDMLGenerator(); - String sql = + DMLGeneratorResponse dmlGeneratorResponse = mySQLDMLGenerator.getDMLStatement( - new DMLGeneratorRequest( - modType, tableName, schema, newValuesJson, keyValuesJson, "+00:00")); + new DMLGeneratorRequest.Builder( + modType, tableName, newValuesJson, keyValuesJson, "+00:00") + .setSchema(schema) + .build()); + String sql = dmlGeneratorResponse.getDmlStatement(); assertTrue(sql.contains("`FirstName` = 'kk'")); assertTrue(sql.contains("`LastName` = 'll'")); @@ -177,10 +193,13 @@ public void primaryKeyNotFoundInJson() { /* The expected sql is: ""*/ MySQLDMLGenerator mySQLDMLGenerator = new MySQLDMLGenerator(); - String sql = + DMLGeneratorResponse dmlGeneratorResponse = mySQLDMLGenerator.getDMLStatement( - new DMLGeneratorRequest( - modType, tableName, schema, newValuesJson, keyValuesJson, "+00:00")); + new DMLGeneratorRequest.Builder( + modType, tableName, newValuesJson, keyValuesJson, "+00:00") + .setSchema(schema) + .build()); + String sql = dmlGeneratorResponse.getDmlStatement(); assertTrue(sql.isEmpty()); } @@ -197,10 +216,13 @@ public void primaryKeyNotPresentInSourceSchema() { /* The expected sql is: ""*/ MySQLDMLGenerator mySQLDMLGenerator = new MySQLDMLGenerator(); - String sql = + DMLGeneratorResponse dmlGeneratorResponse = mySQLDMLGenerator.getDMLStatement( - new DMLGeneratorRequest( - modType, tableName, schema, newValuesJson, keyValuesJson, "+00:00")); + new DMLGeneratorRequest.Builder( + modType, tableName, newValuesJson, keyValuesJson, "+00:00") + .setSchema(schema) + .build()); + String sql = dmlGeneratorResponse.getDmlStatement(); assertTrue(sql.isEmpty()); } @@ -220,10 +242,13 @@ public void timezoneOffsetMismatch() { + " CONVERT_TZ('2023-05-18T12:01:13.088397258','+00:00','+10:00')) ON DUPLICATE KEY" + " UPDATE Bday = CONVERT_TZ('2023-05-18T12:01:13.088397258','+00:00','+10:00')";*/ MySQLDMLGenerator mySQLDMLGenerator = new MySQLDMLGenerator(); - String sql = + DMLGeneratorResponse dmlGeneratorResponse = mySQLDMLGenerator.getDMLStatement( - new DMLGeneratorRequest( - modType, tableName, schema, newValuesJson, keyValuesJson, "+10:00")); + new DMLGeneratorRequest.Builder( + modType, tableName, newValuesJson, keyValuesJson, "+10:00") + .setSchema(schema) + .build()); + String sql = dmlGeneratorResponse.getDmlStatement(); assertTrue( sql.contains("`Bday` = CONVERT_TZ('2023-05-18T12:01:13.088397258','+00:00','+10:00'")); @@ -243,10 +268,13 @@ public void primaryKeyMismatch() { "INSERT INTO Singers(SingerId,FirstName,LastName) VALUES (999,'kk','ll') ON DUPLICATE KEY" + " UPDATE FirstName = 'kk', LastName = 'll'";*/ MySQLDMLGenerator mySQLDMLGenerator = new MySQLDMLGenerator(); - String sql = + DMLGeneratorResponse dmlGeneratorResponse = mySQLDMLGenerator.getDMLStatement( - new DMLGeneratorRequest( - modType, tableName, schema, newValuesJson, keyValuesJson, "+00:00")); + new DMLGeneratorRequest.Builder( + modType, tableName, newValuesJson, keyValuesJson, "+00:00") + .setSchema(schema) + .build()); + String sql = dmlGeneratorResponse.getDmlStatement(); assertTrue(sql.contains("`FirstName` = 'kk'")); assertTrue(sql.contains("`LastName` = 'll'")); @@ -298,10 +326,13 @@ public void allDataypesDML() throws Exception { + " = '2023-05-18', double_column = 42.42, smallint_column = 22, varchar_column =" + " 'abc'"; */ MySQLDMLGenerator mySQLDMLGenerator = new MySQLDMLGenerator(); - String sql = + DMLGeneratorResponse dmlGeneratorResponse = mySQLDMLGenerator.getDMLStatement( - new DMLGeneratorRequest( - modType, tableName, schema, newValuesJson, keyValuesJson, "+00:00")); + new DMLGeneratorRequest.Builder( + modType, tableName, newValuesJson, keyValuesJson, "+00:00") + .setSchema(schema) + .build()); + String sql = dmlGeneratorResponse.getDmlStatement(); assertTrue(sql.contains("`mediumint_column` = 333")); assertTrue(sql.contains("`tinyblob_column` = FROM_BASE64('YWJj')")); @@ -360,10 +391,13 @@ public void updateToNull() { "INSERT INTO Singers(SingerId,FirstName,LastName) VALUES (999,'kk',NULL) ON DUPLICATE KEY" + " UPDATE FirstName = 'kk', LastName = NULL";*/ MySQLDMLGenerator mySQLDMLGenerator = new MySQLDMLGenerator(); - String sql = + DMLGeneratorResponse dmlGeneratorResponse = mySQLDMLGenerator.getDMLStatement( - new DMLGeneratorRequest( - modType, tableName, schema, newValuesJson, keyValuesJson, "+00:00")); + new DMLGeneratorRequest.Builder( + modType, tableName, newValuesJson, keyValuesJson, "+00:00") + .setSchema(schema) + .build()); + String sql = dmlGeneratorResponse.getDmlStatement(); assertTrue(sql.contains("`FirstName` = 'kk'")); assertTrue(sql.contains("`LastName` = NULL")); @@ -382,10 +416,13 @@ public void deleteMultiplePKColumns() { /* The expected sql is: "DELETE FROM Singers WHERE FirstName = 'kk' AND SingerId = 999";*/ MySQLDMLGenerator mySQLDMLGenerator = new MySQLDMLGenerator(); - String sql = + DMLGeneratorResponse dmlGeneratorResponse = mySQLDMLGenerator.getDMLStatement( - new DMLGeneratorRequest( - modType, tableName, schema, newValuesJson, keyValuesJson, "+00:00")); + new DMLGeneratorRequest.Builder( + modType, tableName, newValuesJson, keyValuesJson, "+00:00") + .setSchema(schema) + .build()); + String sql = dmlGeneratorResponse.getDmlStatement(); assertTrue(sql.contains("`FirstName` = 'kk'")); assertTrue(sql.contains("`SingerId` = 999")); @@ -406,10 +443,13 @@ public void testSingleQuoteMatch() { "INSERT INTO Singers(SingerId,FirstName,LastName) VALUES (999,'k''k','ll') ON DUPLICATE KEY" + " UPDATE FirstName = 'k''k', LastName = 'll'"; */ MySQLDMLGenerator mySQLDMLGenerator = new MySQLDMLGenerator(); - String sql = + DMLGeneratorResponse dmlGeneratorResponse = mySQLDMLGenerator.getDMLStatement( - new DMLGeneratorRequest( - modType, tableName, schema, newValuesJson, keyValuesJson, "+00:00")); + new DMLGeneratorRequest.Builder( + modType, tableName, newValuesJson, keyValuesJson, "+00:00") + .setSchema(schema) + .build()); + String sql = dmlGeneratorResponse.getDmlStatement(); assertTrue(sql.contains("`FirstName` = 'k''k'")); assertTrue(sql.contains("`LastName` = 'll'")); @@ -435,10 +475,13 @@ public void singleQuoteBytesDML() throws Exception { + " VALUES (12,'''',FROM_BASE64('Jw=='))" + " ON DUPLICATE KEY UPDATE varchar_column = '''', blob_column = FROM_BASE64('Jw==')";*/ MySQLDMLGenerator mySQLDMLGenerator = new MySQLDMLGenerator(); - String sql = + DMLGeneratorResponse dmlGeneratorResponse = mySQLDMLGenerator.getDMLStatement( - new DMLGeneratorRequest( - modType, tableName, schema, newValuesJson, keyValuesJson, "+00:00")); + new DMLGeneratorRequest.Builder( + modType, tableName, newValuesJson, keyValuesJson, "+00:00") + .setSchema(schema) + .build()); + String sql = dmlGeneratorResponse.getDmlStatement(); assertTrue(sql.contains("`varchar_column` = '''")); assertTrue(sql.contains("`blob_column` = FROM_BASE64('Jw==')")); } @@ -463,10 +506,13 @@ public void twoSingleEscapedQuoteDML() throws Exception { + " (12,'''''',FROM_BASE64('Jyc=')) ON DUPLICATE KEY UPDATE varchar_column = ''''''," + " blob_column = FROM_BASE64('Jyc=')";*/ MySQLDMLGenerator mySQLDMLGenerator = new MySQLDMLGenerator(); - String sql = + DMLGeneratorResponse dmlGeneratorResponse = mySQLDMLGenerator.getDMLStatement( - new DMLGeneratorRequest( - modType, tableName, schema, newValuesJson, keyValuesJson, "+00:00")); + new DMLGeneratorRequest.Builder( + modType, tableName, newValuesJson, keyValuesJson, "+00:00") + .setSchema(schema) + .build()); + String sql = dmlGeneratorResponse.getDmlStatement(); assertTrue(sql.contains("`varchar_column` = '''''")); assertTrue(sql.contains("`blob_column` = FROM_BASE64('Jyc=')")); @@ -492,10 +538,13 @@ public void threeEscapesAndSingleQuoteDML() throws Exception { + " (12,'\\\\''',FROM_BASE64('XCc=')) ON DUPLICATE KEY UPDATE varchar_column =" + " '\\\\''', blob_column = FROM_BASE64('XCc=')";*/ MySQLDMLGenerator mySQLDMLGenerator = new MySQLDMLGenerator(); - String sql = + DMLGeneratorResponse dmlGeneratorResponse = mySQLDMLGenerator.getDMLStatement( - new DMLGeneratorRequest( - modType, tableName, schema, newValuesJson, keyValuesJson, "+00:00")); + new DMLGeneratorRequest.Builder( + modType, tableName, newValuesJson, keyValuesJson, "+00:00") + .setSchema(schema) + .build()); + String sql = dmlGeneratorResponse.getDmlStatement(); assertTrue(sql.contains("`varchar_column` = '\\\\'")); assertTrue(sql.contains("`blob_column` = FROM_BASE64('XCc=')")); @@ -522,10 +571,13 @@ public void tabEscapeDML() throws Exception { + "\t',FROM_BASE64('CQ==')) ON DUPLICATE KEY UPDATE varchar_column = '\t', blob_column" + " = FROM_BASE64('CQ==')"; */ MySQLDMLGenerator mySQLDMLGenerator = new MySQLDMLGenerator(); - String sql = + DMLGeneratorResponse dmlGeneratorResponse = mySQLDMLGenerator.getDMLStatement( - new DMLGeneratorRequest( - modType, tableName, schema, newValuesJson, keyValuesJson, "+00:00")); + new DMLGeneratorRequest.Builder( + modType, tableName, newValuesJson, keyValuesJson, "+00:00") + .setSchema(schema) + .build()); + String sql = dmlGeneratorResponse.getDmlStatement(); assertTrue(sql.contains("`varchar_column` = '\t'")); assertTrue(sql.contains("`blob_column` = FROM_BASE64('CQ==')")); @@ -552,10 +604,13 @@ public void backSpaceEscapeDML() throws Exception { + " (12,'\b',FROM_BASE64('CA==')) ON DUPLICATE KEY UPDATE varchar_column = '\b'," + " blob_column = FROM_BASE64('CA==')";*/ MySQLDMLGenerator mySQLDMLGenerator = new MySQLDMLGenerator(); - String sql = + DMLGeneratorResponse dmlGeneratorResponse = mySQLDMLGenerator.getDMLStatement( - new DMLGeneratorRequest( - modType, tableName, schema, newValuesJson, keyValuesJson, "+00:00")); + new DMLGeneratorRequest.Builder( + modType, tableName, newValuesJson, keyValuesJson, "+00:00") + .setSchema(schema) + .build()); + String sql = dmlGeneratorResponse.getDmlStatement(); assertTrue(sql.contains("`varchar_column` = '\b'")); assertTrue(sql.contains("`blob_column` = FROM_BASE64('CA==')")); @@ -582,10 +637,13 @@ public void newLineEscapeDML() throws Exception { + "',FROM_BASE64('Cg==')) ON DUPLICATE KEY UPDATE varchar_column = '\n" + "', blob_column = FROM_BASE64('Cg==')";*/ MySQLDMLGenerator mySQLDMLGenerator = new MySQLDMLGenerator(); - String sql = + DMLGeneratorResponse dmlGeneratorResponse = mySQLDMLGenerator.getDMLStatement( - new DMLGeneratorRequest( - modType, tableName, schema, newValuesJson, keyValuesJson, "+00:00")); + new DMLGeneratorRequest.Builder( + modType, tableName, newValuesJson, keyValuesJson, "+00:00") + .setSchema(schema) + .build()); + String sql = dmlGeneratorResponse.getDmlStatement(); assertTrue(sql.contains("`varchar_column` = '\n'")); assertTrue(sql.contains("`blob_column` = FROM_BASE64('Cg==')")); @@ -612,10 +670,13 @@ public void carriageReturnEscapeDML() throws Exception { + "',FROM_BASE64('DQ==')) ON DUPLICATE KEY UPDATE varchar_column = '\r" + "', blob_column = FROM_BASE64('DQ==')";*/ MySQLDMLGenerator mySQLDMLGenerator = new MySQLDMLGenerator(); - String sql = + DMLGeneratorResponse dmlGeneratorResponse = mySQLDMLGenerator.getDMLStatement( - new DMLGeneratorRequest( - modType, tableName, schema, newValuesJson, keyValuesJson, "+00:00")); + new DMLGeneratorRequest.Builder( + modType, tableName, newValuesJson, keyValuesJson, "+00:00") + .setSchema(schema) + .build()); + String sql = dmlGeneratorResponse.getDmlStatement(); assertTrue(sql.contains("`varchar_column` = '\r'")); assertTrue(sql.contains("`blob_column` = FROM_BASE64('DQ==')")); @@ -642,10 +703,13 @@ public void formFeedEscapeDML() throws Exception { + " (12,'\f',FROM_BASE64('DA==')) ON DUPLICATE KEY UPDATE varchar_column = '\f'," + " blob_column = FROM_BASE64('DA==')";*/ MySQLDMLGenerator mySQLDMLGenerator = new MySQLDMLGenerator(); - String sql = + DMLGeneratorResponse dmlGeneratorResponse = mySQLDMLGenerator.getDMLStatement( - new DMLGeneratorRequest( - modType, tableName, schema, newValuesJson, keyValuesJson, "+00:00")); + new DMLGeneratorRequest.Builder( + modType, tableName, newValuesJson, keyValuesJson, "+00:00") + .setSchema(schema) + .build()); + String sql = dmlGeneratorResponse.getDmlStatement(); assertTrue(sql.contains("`varchar_column` = '\f'")); assertTrue(sql.contains("`blob_column` = FROM_BASE64('DA==')")); @@ -672,10 +736,13 @@ public void doubleQuoteEscapeDML() throws Exception { + " (12,'\"',FROM_BASE64('Ig==')) ON DUPLICATE KEY UPDATE varchar_column = '\"'," + " blob_column = FROM_BASE64('Ig==')";*/ MySQLDMLGenerator mySQLDMLGenerator = new MySQLDMLGenerator(); - String sql = + DMLGeneratorResponse dmlGeneratorResponse = mySQLDMLGenerator.getDMLStatement( - new DMLGeneratorRequest( - modType, tableName, schema, newValuesJson, keyValuesJson, "+00:00")); + new DMLGeneratorRequest.Builder( + modType, tableName, newValuesJson, keyValuesJson, "+00:00") + .setSchema(schema) + .build()); + String sql = dmlGeneratorResponse.getDmlStatement(); assertTrue(sql.contains("`varchar_column` = '\"'")); assertTrue(sql.contains("`blob_column` = FROM_BASE64('Ig==')")); @@ -702,10 +769,13 @@ public void backSlashEscapeDML() throws Exception { + " (12,'\\\\',FROM_BASE64('XA==')) ON DUPLICATE KEY UPDATE varchar_column = '\\\\'," + " blob_column = FROM_BASE64('XA==')";*/ MySQLDMLGenerator mySQLDMLGenerator = new MySQLDMLGenerator(); - String sql = + DMLGeneratorResponse dmlGeneratorResponse = mySQLDMLGenerator.getDMLStatement( - new DMLGeneratorRequest( - modType, tableName, schema, newValuesJson, keyValuesJson, "+00:00")); + new DMLGeneratorRequest.Builder( + modType, tableName, newValuesJson, keyValuesJson, "+00:00") + .setSchema(schema) + .build()); + String sql = dmlGeneratorResponse.getDmlStatement(); assertTrue(sql.contains("`varchar_column` = '\\\\'")); assertTrue(sql.contains("`blob_column` = FROM_BASE64('XA==')")); @@ -726,10 +796,13 @@ public void bitColumnSql() { + " (999,'kk',BINARY(FROM_BASE64('YmlsX2NvbA=='))) ON DUPLICATE KEY UPDATE FirstName =" + " 'kk', LastName = BINARY(FROM_BASE64('YmlsX2NvbA=='))"; */ MySQLDMLGenerator mySQLDMLGenerator = new MySQLDMLGenerator(); - String sql = + DMLGeneratorResponse dmlGeneratorResponse = mySQLDMLGenerator.getDMLStatement( - new DMLGeneratorRequest( - modType, tableName, schema, newValuesJson, keyValuesJson, "+00:00")); + new DMLGeneratorRequest.Builder( + modType, tableName, newValuesJson, keyValuesJson, "+00:00") + .setSchema(schema) + .build()); + String sql = dmlGeneratorResponse.getDmlStatement(); assertTrue(sql.contains("`LastName` = BINARY(FROM_BASE64('YmlsX2NvbA=='))")); } @@ -745,10 +818,13 @@ public void testSpannerTableNotInSchema() { String modType = "INSERT"; MySQLDMLGenerator mySQLDMLGenerator = new MySQLDMLGenerator(); - String sql = + DMLGeneratorResponse dmlGeneratorResponse = mySQLDMLGenerator.getDMLStatement( - new DMLGeneratorRequest( - modType, tableName, schema, newValuesJson, keyValuesJson, "+00:00")); + new DMLGeneratorRequest.Builder( + modType, tableName, newValuesJson, keyValuesJson, "+00:00") + .setSchema(schema) + .build()); + String sql = dmlGeneratorResponse.getDmlStatement(); assertTrue(sql.isEmpty()); } @@ -764,10 +840,13 @@ public void testSpannerKeyIsNull() { String modType = "INSERT"; MySQLDMLGenerator mySQLDMLGenerator = new MySQLDMLGenerator(); - String sql = + DMLGeneratorResponse dmlGeneratorResponse = mySQLDMLGenerator.getDMLStatement( - new DMLGeneratorRequest( - modType, tableName, schema, newValuesJson, keyValuesJson, "+00:00")); + new DMLGeneratorRequest.Builder( + modType, tableName, newValuesJson, keyValuesJson, "+00:00") + .setSchema(schema) + .build()); + String sql = dmlGeneratorResponse.getDmlStatement(); assertTrue( sql.contains( @@ -785,10 +864,13 @@ public void testKeyInNewValuesJson() { String modType = "INSERT"; MySQLDMLGenerator mySQLDMLGenerator = new MySQLDMLGenerator(); - String sql = + DMLGeneratorResponse dmlGeneratorResponse = mySQLDMLGenerator.getDMLStatement( - new DMLGeneratorRequest( - modType, tableName, schema, newValuesJson, keyValuesJson, "+00:00")); + new DMLGeneratorRequest.Builder( + modType, tableName, newValuesJson, keyValuesJson, "+00:00") + .setSchema(schema) + .build()); + String sql = dmlGeneratorResponse.getDmlStatement(); assertTrue( sql.contains( "INSERT INTO `Singers`(`SingerId`,`FirstName`,`LastName`) VALUES (NULL,'kk','ll')")); @@ -805,10 +887,13 @@ public void testSourcePKNotInSpanner() { String modType = "DELETE"; MySQLDMLGenerator mySQLDMLGenerator = new MySQLDMLGenerator(); - String sql = + DMLGeneratorResponse dmlGeneratorResponse = mySQLDMLGenerator.getDMLStatement( - new DMLGeneratorRequest( - modType, tableName, schema, newValuesJson, keyValuesJson, "+00:00")); + new DMLGeneratorRequest.Builder( + modType, tableName, newValuesJson, keyValuesJson, "+00:00") + .setSchema(schema) + .build()); + String sql = dmlGeneratorResponse.getDmlStatement(); assertTrue(sql.isEmpty()); } @@ -827,10 +912,13 @@ public void primaryKeyMismatchSpannerNull() { "INSERT INTO Singers(SingerId,FirstName,LastName) VALUES (999,NULL,'ll') ON DUPLICATE KEY" + " UPDATE FirstName = NULL , LastName = 'll'";*/ MySQLDMLGenerator mySQLDMLGenerator = new MySQLDMLGenerator(); - String sql = + DMLGeneratorResponse dmlGeneratorResponse = mySQLDMLGenerator.getDMLStatement( - new DMLGeneratorRequest( - modType, tableName, schema, newValuesJson, keyValuesJson, "+00:00")); + new DMLGeneratorRequest.Builder( + modType, tableName, newValuesJson, keyValuesJson, "+00:00") + .setSchema(schema) + .build()); + String sql = dmlGeneratorResponse.getDmlStatement(); assertTrue(sql.contains("`FirstName` = NULL")); } @@ -846,10 +934,13 @@ public void testUnsupportedModType() { String modType = "JUNK"; MySQLDMLGenerator mySQLDMLGenerator = new MySQLDMLGenerator(); - String sql = + DMLGeneratorResponse dmlGeneratorResponse = mySQLDMLGenerator.getDMLStatement( - new DMLGeneratorRequest( - modType, tableName, schema, newValuesJson, keyValuesJson, "+00:00")); + new DMLGeneratorRequest.Builder( + modType, tableName, newValuesJson, keyValuesJson, "+00:00") + .setSchema(schema) + .build()); + String sql = dmlGeneratorResponse.getDmlStatement(); assertTrue(sql.isEmpty()); } @@ -868,10 +959,13 @@ public void testUpdateModType() { "INSERT INTO Singers(SingerId,FirstName,LastName) VALUES (999,'kk','ll') ON DUPLICATE KEY" + " UPDATE FirstName = 'kk', LastName = 'll'";*/ MySQLDMLGenerator mySQLDMLGenerator = new MySQLDMLGenerator(); - String sql = + DMLGeneratorResponse dmlGeneratorResponse = mySQLDMLGenerator.getDMLStatement( - new DMLGeneratorRequest( - modType, tableName, schema, newValuesJson, keyValuesJson, "+00:00")); + new DMLGeneratorRequest.Builder( + modType, tableName, newValuesJson, keyValuesJson, "+00:00") + .setSchema(schema) + .build()); + String sql = dmlGeneratorResponse.getDmlStatement(); assertTrue(sql.contains("`FirstName` = 'kk'")); assertTrue(sql.contains("`LastName` = 'll'")); @@ -888,10 +982,13 @@ public void testSpannerTableIdMismatch() { String modType = "DELETE"; MySQLDMLGenerator mySQLDMLGenerator = new MySQLDMLGenerator(); - String sql = + DMLGeneratorResponse dmlGeneratorResponse = mySQLDMLGenerator.getDMLStatement( - new DMLGeneratorRequest( - modType, tableName, schema, newValuesJson, keyValuesJson, "+00:00")); + new DMLGeneratorRequest.Builder( + modType, tableName, newValuesJson, keyValuesJson, "+00:00") + .setSchema(schema) + .build()); + String sql = dmlGeneratorResponse.getDmlStatement(); assertTrue(sql.isEmpty()); } @@ -907,10 +1004,13 @@ public void testSourcePkNull() { String modType = "INSERT"; MySQLDMLGenerator mySQLDMLGenerator = new MySQLDMLGenerator(); - String sql = + DMLGeneratorResponse dmlGeneratorResponse = mySQLDMLGenerator.getDMLStatement( - new DMLGeneratorRequest( - modType, tableName, schema, newValuesJson, keyValuesJson, "+00:00")); + new DMLGeneratorRequest.Builder( + modType, tableName, newValuesJson, keyValuesJson, "+00:00") + .setSchema(schema) + .build()); + String sql = dmlGeneratorResponse.getDmlStatement(); assertTrue(sql.isEmpty()); } @@ -926,10 +1026,13 @@ public void testSourceTableNotInSchema() { String modType = "INSERT"; MySQLDMLGenerator mySQLDMLGenerator = new MySQLDMLGenerator(); - String sql = + DMLGeneratorResponse dmlGeneratorResponse = mySQLDMLGenerator.getDMLStatement( - new DMLGeneratorRequest( - modType, tableName, schema, newValuesJson, keyValuesJson, "+00:00")); + new DMLGeneratorRequest.Builder( + modType, tableName, newValuesJson, keyValuesJson, "+00:00") + .setSchema(schema) + .build()); + String sql = dmlGeneratorResponse.getDmlStatement(); assertTrue(sql.isEmpty()); } @@ -946,10 +1049,13 @@ public void testSpannerTableNotInSchemaObject() { String modType = "INSERT"; MySQLDMLGenerator mySQLDMLGenerator = new MySQLDMLGenerator(); - String sql = + DMLGeneratorResponse dmlGeneratorResponse = mySQLDMLGenerator.getDMLStatement( - new DMLGeneratorRequest( - modType, tableName, schema, newValuesJson, keyValuesJson, "+00:00")); + new DMLGeneratorRequest.Builder( + modType, tableName, newValuesJson, keyValuesJson, "+00:00") + .setSchema(schema) + .build()); + String sql = dmlGeneratorResponse.getDmlStatement(); assertTrue(sql.isEmpty()); } @@ -969,10 +1075,13 @@ public void testSpannerColDefsNull() { String modType = "INSERT"; MySQLDMLGenerator mySQLDMLGenerator = new MySQLDMLGenerator(); - String sql = + DMLGeneratorResponse dmlGeneratorResponse = mySQLDMLGenerator.getDMLStatement( - new DMLGeneratorRequest( - modType, tableName, schema, newValuesJson, keyValuesJson, "+00:00")); + new DMLGeneratorRequest.Builder( + modType, tableName, newValuesJson, keyValuesJson, "+00:00") + .setSchema(schema) + .build()); + String sql = dmlGeneratorResponse.getDmlStatement(); MySQLDMLGenerator test = new MySQLDMLGenerator(); // to add that last bit of code coverage InputRecordProcessor test2 = new InputRecordProcessor(); // to add that last bit of code coverage diff --git a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/source/common/DMLGeneratorRequestTest.java b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/source/common/DMLGeneratorRequestTest.java deleted file mode 100644 index e59b0e0d1f..0000000000 --- a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/source/common/DMLGeneratorRequestTest.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Copyright (C) 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ -package com.google.cloud.teleport.v2.templates.source.common; - -import static org.junit.Assert.assertEquals; -import static org.mockito.Mockito.mock; - -import com.google.cloud.teleport.v2.spanner.migrations.schema.Schema; -import org.json.JSONObject; -import org.junit.Before; -import org.junit.Test; - -public class DMLGeneratorRequestTest { - private String modType; - private String spannerTableName; - private Schema schema; - private JSONObject newValuesJson; - private JSONObject keyValuesJson; - private String sourceDbTimezoneOffset; - - @Before - public void setUp() { - // Initialize mock data - modType = "INSERT"; - spannerTableName = "my_table"; - schema = mock(Schema.class); - newValuesJson = new JSONObject(); - newValuesJson.put("column1", "value1"); - keyValuesJson = new JSONObject(); - keyValuesJson.put("column1", "key1"); - sourceDbTimezoneOffset = "+00:00"; - } - - @Test - public void testConstructorAndGetters() { - DMLGeneratorRequest request = - new DMLGeneratorRequest( - modType, - spannerTableName, - schema, - newValuesJson, - keyValuesJson, - sourceDbTimezoneOffset); - - assertEquals(modType, request.getModType()); - assertEquals(spannerTableName, request.getSpannerTableName()); - assertEquals(schema, request.getSchema()); - assertEquals(newValuesJson.toString(), request.getNewValuesJson().toString()); - assertEquals(keyValuesJson.toString(), request.getKeyValuesJson().toString()); - assertEquals(sourceDbTimezoneOffset, request.getSourceDbTimezoneOffset()); - } -} diff --git a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/source/common/SourceProcessorFactoryTest.java b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/source/common/SourceProcessorFactoryTest.java deleted file mode 100644 index ae4641a2c4..0000000000 --- a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/source/common/SourceProcessorFactoryTest.java +++ /dev/null @@ -1,103 +0,0 @@ -/* - * Copyright (C) 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ -package com.google.cloud.teleport.v2.templates.source.common; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import com.google.cloud.teleport.v2.spanner.migrations.shard.Shard; -import com.google.cloud.teleport.v2.templates.constants.Constants; -import com.google.cloud.teleport.v2.templates.source.sql.SqlDao; -import com.google.cloud.teleport.v2.templates.source.sql.mysql.MySQLDMLGenerator; -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.ExpectedException; - -public class SourceProcessorFactoryTest { - - @Rule public ExpectedException expectedEx = ExpectedException.none(); - - @Test - public void testGetDMLGenerator_MySQLSource() throws Exception { - IDMLGenerator dmlGenerator = SourceProcessorFactory.getSourceDMLGenerator(Constants.SOURCE_MYSQL); - assertNotNull(dmlGenerator); - assertTrue(dmlGenerator instanceof MySQLDMLGenerator); - } - - @Test - public void testGetDMLGenerator_InvalidSource() throws Exception { - expectedEx.expect(Exception.class); - expectedEx.expectMessage("Invalid source type: invalidSource"); - SourceProcessorFactory.getSourceDMLGenerator("invalidSource"); - } - - @Test - public void testGetSourceDaoMap_MySQLSource() throws Exception { - Shard shard1 = mock(Shard.class); - when(shard1.getHost()).thenReturn("localhost"); - when(shard1.getPort()).thenReturn("3306"); - when(shard1.getDbName()).thenReturn("test_db"); - when(shard1.getLogicalShardId()).thenReturn("shard1"); - when(shard1.getUserName()).thenReturn("root"); - when(shard1.getPassword()).thenReturn("password"); - - Shard shard2 = mock(Shard.class); - when(shard2.getHost()).thenReturn("localhost"); - when(shard2.getPort()).thenReturn("3307"); - when(shard2.getDbName()).thenReturn("test_db_2"); - when(shard2.getLogicalShardId()).thenReturn("shard2"); - when(shard2.getUserName()).thenReturn("root"); - when(shard2.getPassword()).thenReturn("password"); - - List shards = Arrays.asList(shard1, shard2); - - Map sourceDaoMap = - SourceProcessorFactory.getSourceDaoMap(Constants.SOURCE_MYSQL, shards, 10); - - assertNotNull(sourceDaoMap); - assertEquals(2, sourceDaoMap.size()); - - ISourceDao sqlDao1 = sourceDaoMap.get("shard1"); - assertTrue(sqlDao1 instanceof SqlDao); - assertEquals( - "jdbc:mysql://localhost:3306/test_db", ((SqlDao) sqlDao1).getSourceConnectionUrl()); - - ISourceDao sqlDao2 = sourceDaoMap.get("shard2"); - assertTrue(sqlDao2 instanceof SqlDao); - assertEquals( - "jdbc:mysql://localhost:3307/test_db_2", ((SqlDao) sqlDao2).getSourceConnectionUrl()); - } - - @Test - public void testGetSourceDaoMap_InvalidSource() throws Exception { - expectedEx.expect(Exception.class); - expectedEx.expectMessage("Invalid source type: invalidSource"); - List shards = Arrays.asList(mock(Shard.class), mock(Shard.class)); - SourceProcessorFactory.getSourceDaoMap("invalidSource", shards, 10); - } - - @Test - public void testGetSourceDaoMap_NullShards() throws Exception { - expectedEx.expect(NullPointerException.class); - SourceProcessorFactory.getSourceDaoMap(Constants.SOURCE_MYSQL, null, 10); - } -} diff --git a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/transforms/SourceWriterFnTest.java b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/transforms/SourceWriterFnTest.java index 3a21af508a..7489ccbd0d 100644 --- a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/transforms/SourceWriterFnTest.java +++ b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/transforms/SourceWriterFnTest.java @@ -42,10 +42,12 @@ import com.google.cloud.teleport.v2.templates.changestream.ChangeStreamErrorRecord; import com.google.cloud.teleport.v2.templates.changestream.TrimmedShardedDataChangeRecord; import com.google.cloud.teleport.v2.templates.constants.Constants; -import com.google.cloud.teleport.v2.templates.source.common.ISourceDao; -import com.google.cloud.teleport.v2.templates.source.sql.SqlDao; +import com.google.cloud.teleport.v2.templates.dao.source.IDao; +import com.google.cloud.teleport.v2.templates.dao.source.JdbcDao; +import com.google.cloud.teleport.v2.templates.dao.spanner.SpannerDao; +import com.google.cloud.teleport.v2.templates.dml.MySQLDMLGenerator; +import com.google.cloud.teleport.v2.templates.processor.SourceProcessor; import com.google.cloud.teleport.v2.templates.utils.ShadowTableRecord; -import com.google.cloud.teleport.v2.templates.utils.SpannerDao; import com.google.common.collect.ImmutableList; import com.google.gson.Gson; import java.util.HashMap; @@ -69,9 +71,9 @@ public class SourceWriterFnTest { @Rule public final transient TestPipeline pipeline = TestPipeline.create(); @Rule public final MockitoRule mocktio = MockitoJUnit.rule(); - @Mock private SqlDao mockSqlDao; + @Mock private JdbcDao mockSqlDao; @Mock private SpannerDao mockSpannerDao; - @Mock HashMap mockSourceDaoMap; + @Mock HashMap mockDaoMap; @Mock private SpannerConfig mockSpannerConfig; @Mock private DoFn.ProcessContext processContext; private static Gson gson = new Gson(); @@ -81,9 +83,11 @@ public class SourceWriterFnTest { private Ddl testDdl; private String testSourceDbTimezoneOffset; + private SourceProcessor sourceProcessor; + @Before public void doBeforeEachTest() throws Exception { - when(mockSourceDaoMap.get(any())).thenReturn(mockSqlDao); + when(mockDaoMap.get(any())).thenReturn(mockSqlDao); when(mockSpannerDao.getShadowTableRecord(eq("shadow_parent1"), any())).thenReturn(null); when(mockSpannerDao.getShadowTableRecord(eq("shadow_tableName"), any())).thenReturn(null); when(mockSpannerDao.getShadowTableRecord(eq("shadow_parent2"), any())) @@ -122,6 +126,11 @@ public void doBeforeEachTest() throws Exception { testSchema = SessionFileReader.read("src/test/resources/sourceWriterUTSession.json"); testSourceDbTimezoneOffset = "+00:00"; testDdl = getTestDdl(); + sourceProcessor = + SourceProcessor.builder() + .dmlGenerator(new MySQLDMLGenerator()) + .sourceDaoMap(mockDaoMap) + .build(); } @Test @@ -144,7 +153,6 @@ public void testSourceIsAhead() throws Exception { mapper.enable(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS); sourceWriterFn.setObjectMapper(mapper); sourceWriterFn.setSpannerDao(mockSpannerDao); - sourceWriterFn.setSourceDaoMap(mockSourceDaoMap); sourceWriterFn.processElement(processContext); verify(mockSpannerDao, atLeast(1)).getShadowTableRecord(any(), any()); verify(mockSqlDao, never()).write(any()); @@ -172,7 +180,6 @@ public void testSourceIsAheadWithSameCommitTimestamp() throws Exception { mapper.enable(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS); sourceWriterFn.setObjectMapper(mapper); sourceWriterFn.setSpannerDao(mockSpannerDao); - sourceWriterFn.setSourceDaoMap(mockSourceDaoMap); sourceWriterFn.processElement(processContext); verify(mockSpannerDao, atLeast(1)).getShadowTableRecord(any(), any()); verify(mockSqlDao, never()).write(any()); @@ -198,8 +205,8 @@ public void testSourceIsBehind() throws Exception { ObjectMapper mapper = new ObjectMapper(); mapper.enable(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS); sourceWriterFn.setObjectMapper(mapper); + sourceWriterFn.setSourceProcessor(sourceProcessor); sourceWriterFn.setSpannerDao(mockSpannerDao); - sourceWriterFn.setSourceDaoMap(mockSourceDaoMap); sourceWriterFn.processElement(processContext); verify(mockSpannerDao, atLeast(1)).getShadowTableRecord(any(), any()); verify(mockSqlDao, atLeast(1)).write(any()); @@ -225,7 +232,6 @@ public void testNoShard() throws Exception { mapper.enable(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS); sourceWriterFn.setObjectMapper(mapper); sourceWriterFn.setSpannerDao(mockSpannerDao); - sourceWriterFn.setSourceDaoMap(mockSourceDaoMap); sourceWriterFn.processElement(processContext); String jsonRec = gson.toJson(record, TrimmedShardedDataChangeRecord.class); ChangeStreamErrorRecord errorRecord = @@ -256,7 +262,6 @@ public void testSkipShard() throws Exception { mapper.enable(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS); sourceWriterFn.setObjectMapper(mapper); sourceWriterFn.setSpannerDao(mockSpannerDao); - sourceWriterFn.setSourceDaoMap(mockSourceDaoMap); sourceWriterFn.processElement(processContext); String jsonRec = gson.toJson(record, TrimmedShardedDataChangeRecord.class); ChangeStreamErrorRecord errorRecord = @@ -285,7 +290,6 @@ public void testPermanentError() throws Exception { mapper.enable(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS); sourceWriterFn.setObjectMapper(mapper); sourceWriterFn.setSpannerDao(mockSpannerDao); - sourceWriterFn.setSourceDaoMap(mockSourceDaoMap); sourceWriterFn.processElement(processContext); String jsonRec = gson.toJson(record, TrimmedShardedDataChangeRecord.class); ChangeStreamErrorRecord errorRecord = @@ -318,7 +322,6 @@ public void testRetryableError() throws Exception { mapper.enable(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS); sourceWriterFn.setObjectMapper(mapper); sourceWriterFn.setSpannerDao(mockSpannerDao); - sourceWriterFn.setSourceDaoMap(mockSourceDaoMap); sourceWriterFn.processElement(processContext); String jsonRec = gson.toJson(record, TrimmedShardedDataChangeRecord.class); ChangeStreamErrorRecord errorRecord = new ChangeStreamErrorRecord(jsonRec, "Test exception"); @@ -346,8 +349,8 @@ public void testRetryableErrorForForeignKey() throws Exception { ObjectMapper mapper = new ObjectMapper(); mapper.enable(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS); sourceWriterFn.setObjectMapper(mapper); + sourceWriterFn.setSourceProcessor(sourceProcessor); sourceWriterFn.setSpannerDao(mockSpannerDao); - sourceWriterFn.setSourceDaoMap(mockSourceDaoMap); sourceWriterFn.processElement(processContext); String jsonRec = gson.toJson(record, TrimmedShardedDataChangeRecord.class); ChangeStreamErrorRecord errorRecord = @@ -376,8 +379,8 @@ public void testRetryableErrorConnectionFailure() throws Exception { ObjectMapper mapper = new ObjectMapper(); mapper.enable(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS); sourceWriterFn.setObjectMapper(mapper); + sourceWriterFn.setSourceProcessor(sourceProcessor); sourceWriterFn.setSpannerDao(mockSpannerDao); - sourceWriterFn.setSourceDaoMap(mockSourceDaoMap); sourceWriterFn.processElement(processContext); String jsonRec = gson.toJson(record, TrimmedShardedDataChangeRecord.class); ChangeStreamErrorRecord errorRecord = @@ -406,8 +409,8 @@ public void testPermanentConnectionFailure() throws Exception { ObjectMapper mapper = new ObjectMapper(); mapper.enable(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS); sourceWriterFn.setObjectMapper(mapper); + sourceWriterFn.setSourceProcessor(sourceProcessor); sourceWriterFn.setSpannerDao(mockSpannerDao); - sourceWriterFn.setSourceDaoMap(mockSourceDaoMap); sourceWriterFn.processElement(processContext); String jsonRec = gson.toJson(record, TrimmedShardedDataChangeRecord.class); ChangeStreamErrorRecord errorRecord = @@ -436,8 +439,8 @@ public void testPermanentGenericException() throws Exception { ObjectMapper mapper = new ObjectMapper(); mapper.enable(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS); sourceWriterFn.setObjectMapper(mapper); + sourceWriterFn.setSourceProcessor(sourceProcessor); sourceWriterFn.setSpannerDao(mockSpannerDao); - sourceWriterFn.setSourceDaoMap(mockSourceDaoMap); sourceWriterFn.processElement(processContext); String jsonRec = gson.toJson(record, TrimmedShardedDataChangeRecord.class); ChangeStreamErrorRecord errorRecord = new ChangeStreamErrorRecord(jsonRec, "generic exception"); @@ -466,7 +469,6 @@ public void testDMLEmpty() throws Exception { mapper.enable(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS); sourceWriterFn.setObjectMapper(mapper); sourceWriterFn.setSpannerDao(mockSpannerDao); - sourceWriterFn.setSourceDaoMap(mockSourceDaoMap); sourceWriterFn.processElement(processContext); verify(mockSqlDao, never()).write(contains("567890")); } From c99a2541ada18471a9058dd7998b84b659a59e55 Mon Sep 17 00:00:00 2001 From: Shreya Khajanchi Date: Wed, 20 Nov 2024 12:17:12 +0530 Subject: [PATCH 07/13] adding comments --- pom.xml | 1 + .../v2/templates/dao/source/IDao.java | 6 +- .../v2/templates/dao/source/JdbcDao.java | 8 +- .../v2/templates/dml/IDMLGenerator.java | 17 ++++ .../ConnectionException.java | 2 +- .../exceptions/InvalidSourceException.java | 31 +++++++ .../v2/templates/exceptions/package-info.java | 17 ++++ .../models/ConnectionHelperRequest.java | 12 +++ .../templates/models/DMLGeneratorRequest.java | 25 +++++- .../templates/processor/SourceProcessor.java | 11 +++ .../processor/SourceProcessorFactory.java | 53 +++++++----- .../templates/transforms/SourceWriterFn.java | 2 +- .../utils/connection/IConnectionHelper.java | 1 + .../connection/MySQLConnectionHelper.java | 9 +- .../v2/templates/dao/JdbcDaoTest.java | 2 +- .../processor/SourceProcessorFactoryTest.java | 85 +++++++++++++++++++ 16 files changed, 238 insertions(+), 44 deletions(-) rename v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/{utils/connection => exceptions}/ConnectionException.java (93%) create mode 100644 v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/exceptions/InvalidSourceException.java create mode 100644 v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/exceptions/package-info.java create mode 100644 v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/processor/SourceProcessorFactoryTest.java diff --git a/pom.xml b/pom.xml index 5e3f942820..f3951f5336 100644 --- a/pom.xml +++ b/pom.xml @@ -368,6 +368,7 @@ **/JarFileReader.* **/CustomTransformationWithShardFor*IT.* **/models/* + **/exceptions/* diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dao/source/IDao.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dao/source/IDao.java index e761cce69d..13974c1f10 100644 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dao/source/IDao.java +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dao/source/IDao.java @@ -15,12 +15,12 @@ */ package com.google.cloud.teleport.v2.templates.dao.source; -public interface IDao { +public interface IDao { /** - * Executes a given write statement. + * Executes a given write statement against the data source. * * @param statement The SQL or query statement. * @throws Exception If there is an error executing the statement. */ - void write(String statement) throws Exception; + void write(T statement) throws Exception; } diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dao/source/JdbcDao.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dao/source/JdbcDao.java index 102f6b50bc..d3dc7552ee 100644 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dao/source/JdbcDao.java +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dao/source/JdbcDao.java @@ -15,13 +15,13 @@ */ package com.google.cloud.teleport.v2.templates.dao.source; -import com.google.cloud.teleport.v2.templates.utils.connection.ConnectionException; +import com.google.cloud.teleport.v2.templates.exceptions.ConnectionException; import com.google.cloud.teleport.v2.templates.utils.connection.IConnectionHelper; import java.sql.Connection; import java.sql.SQLException; import java.sql.Statement; -public class JdbcDao implements IDao { +public class JdbcDao implements IDao { private String sqlUrl; private String sqlUser; @@ -33,10 +33,6 @@ public JdbcDao(String sqlUrl, String sqlUser, IConnectionHelper connectionHelper this.connectionHelper = connectionHelper; } - public String getSourceConnectionUrl() { - return sqlUrl; - } - @Override public void write(String sqlStatement) throws SQLException, ConnectionException { Connection connObj = null; diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dml/IDMLGenerator.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dml/IDMLGenerator.java index 3dc34a240b..31923f6d6d 100644 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dml/IDMLGenerator.java +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dml/IDMLGenerator.java @@ -18,6 +18,23 @@ import com.google.cloud.teleport.v2.templates.models.DMLGeneratorRequest; import com.google.cloud.teleport.v2.templates.models.DMLGeneratorResponse; +/** + * Interface for generating DML statements. + * + *

This interface provides a contract for implementing classes to define the logic for generating + * DML statements, such as INSERT, UPDATE, and DELETE, based on a provided {@link + * DMLGeneratorRequest}. + * + *

Classes implementing this interface should handle the construction of DML statements tailored + * to the specific requirements of the target database. + */ public interface IDMLGenerator { + /** + * Generates a DML statement based on the provided {@link DMLGeneratorRequest}. + * + * @param dmlGeneratorRequest the request containing necessary information to construct the DML + * statement, including modification type, table schema, new values, and key values. + * @return a {@link DMLGeneratorResponse} object containing the generated DML statement. + */ DMLGeneratorResponse getDMLStatement(DMLGeneratorRequest dmlGeneratorRequest); } diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/connection/ConnectionException.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/exceptions/ConnectionException.java similarity index 93% rename from v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/connection/ConnectionException.java rename to v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/exceptions/ConnectionException.java index 3c93235dcc..949fb4f4a4 100644 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/connection/ConnectionException.java +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/exceptions/ConnectionException.java @@ -13,7 +13,7 @@ * License for the specific language governing permissions and limitations under * the License. */ -package com.google.cloud.teleport.v2.templates.utils.connection; +package com.google.cloud.teleport.v2.templates.exceptions; /** Exception when connecting to the source database. */ public class ConnectionException extends Exception { diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/exceptions/InvalidSourceException.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/exceptions/InvalidSourceException.java new file mode 100644 index 0000000000..58e224ae79 --- /dev/null +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/exceptions/InvalidSourceException.java @@ -0,0 +1,31 @@ +/* + * Copyright (C) 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.templates.exceptions; + +/** Exception in case of unsupported source database. */ +public class InvalidSourceException extends Exception { + public InvalidSourceException(Exception e) { + super(e); + } + + public InvalidSourceException(String message) { + super(message); + } + + public InvalidSourceException(String message, Exception e) { + super(message, e); + } +} diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/exceptions/package-info.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/exceptions/package-info.java new file mode 100644 index 0000000000..60d38f6de9 --- /dev/null +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/exceptions/package-info.java @@ -0,0 +1,17 @@ +/* + * Copyright (C) 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +/** Package info for exceptions module. */ +package com.google.cloud.teleport.v2.templates.exceptions; diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/models/ConnectionHelperRequest.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/models/ConnectionHelperRequest.java index 6e1817695e..ec19a5c20a 100644 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/models/ConnectionHelperRequest.java +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/models/ConnectionHelperRequest.java @@ -18,6 +18,18 @@ import com.google.cloud.teleport.v2.spanner.migrations.shard.Shard; import java.util.List; +/** + * Represents a request to initialize a connection helper with the necessary parameters. + * + *

This class encapsulates the essential information required for establishing connections to a + * database or a data source. It includes: + * + *

    + *
  • A list of {@link Shard} objects representing the database shards. + *
  • Optional connection properties as a {@link String}. + *
  • The maximum number of connections allowed. + *
+ */ public class ConnectionHelperRequest { private List shards; private String properties; diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/models/DMLGeneratorRequest.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/models/DMLGeneratorRequest.java index f6615b9d35..661f05038d 100644 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/models/DMLGeneratorRequest.java +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/models/DMLGeneratorRequest.java @@ -19,15 +19,36 @@ import org.json.JSONObject; /** - * A request object representing the data necessary to generate DML statements for interacting with - * a source database. + * Represents a request object containing the data required to generate DML (Data Manipulation + * Language) statements for interacting with a source database. + * + *

This class is immutable and is built using the {@link Builder} class. It includes: + * + *

    + *
  • Modification type (e.g., INSERT, UPDATE, DELETE). + *
  • The corresponding Spanner table name. + *
  • The schema definition of the table. + *
  • JSON objects for new values and key values. + *
  • The timezone offset of the source database. + *
*/ public class DMLGeneratorRequest { + // The type of DML operation (e.g., "INSERT", "UPDATE", "DELETE"). private final String modType; + + // The name of the Spanner table associated with the DML operation. private final String spannerTableName; + + // The schema of the source and spanner table, providing details about the table's structure. private final Schema schema; + + // JSON object containing the new values for the operation (e.g., updated or inserted values). private final JSONObject newValuesJson; + + // JSON object containing the key values for identifying records . private final JSONObject keyValuesJson; + + // The timezone offset of the source database, used for handling timezone-specific data. private final String sourceDbTimezoneOffset; public DMLGeneratorRequest(Builder builder) { diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/processor/SourceProcessor.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/processor/SourceProcessor.java index 173be89c3e..344a73162e 100644 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/processor/SourceProcessor.java +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/processor/SourceProcessor.java @@ -20,11 +20,21 @@ import java.util.HashMap; import java.util.Map; +/** + * Represents a processor responsible for managing source database operations. This class + * encapsulates a DML generator and a map of DAOs for interacting with source databases. + */ public class SourceProcessor { private final IDMLGenerator dmlGenerator; private final Map sourceDaoMap; + /** + * Constructs a SourceProcessor with the specified DML generator and source DAO map. + * + * @param dmlGenerator the DML generator for the source + * @param sourceDaoMap the map of shard ID to DAO for the source + */ private SourceProcessor(IDMLGenerator dmlGenerator, Map sourceDaoMap) { this.dmlGenerator = dmlGenerator; this.sourceDaoMap = sourceDaoMap; @@ -42,6 +52,7 @@ public static Builder builder() { return new Builder(); } + /** Builder class for constructing instances of {@link SourceProcessor}. */ public static class Builder { private IDMLGenerator dmlGenerator; private Map sourceDaoMap = new HashMap<>(); diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/processor/SourceProcessorFactory.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/processor/SourceProcessorFactory.java index 24ce9b9eaf..a754232863 100644 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/processor/SourceProcessorFactory.java +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/processor/SourceProcessorFactory.java @@ -21,6 +21,7 @@ import com.google.cloud.teleport.v2.templates.dao.source.JdbcDao; import com.google.cloud.teleport.v2.templates.dml.IDMLGenerator; import com.google.cloud.teleport.v2.templates.dml.MySQLDMLGenerator; +import com.google.cloud.teleport.v2.templates.exceptions.InvalidSourceException; import com.google.cloud.teleport.v2.templates.models.ConnectionHelperRequest; import com.google.cloud.teleport.v2.templates.utils.connection.IConnectionHelper; import com.google.cloud.teleport.v2.templates.utils.connection.MySQLConnectionHelper; @@ -31,21 +32,26 @@ import java.util.function.Function; public class SourceProcessorFactory { - private static final Map DML_GENERATOR_MAP = + private static Map dmlGeneratorMap = Map.of(Constants.SOURCE_MYSQL, new MySQLDMLGenerator()); - private static final Map CONNECTION_HELPER_MAP = + private static Map connectionHelperMap = Map.of(Constants.SOURCE_MYSQL, new MySQLConnectionHelper()); - private static final Map DRIVER_MAP = + private static Map driverMap = Map.of(Constants.SOURCE_MYSQL, "com.mysql.cj.jdbc.Driver"); - private static final Map> CONNECTION_URL_GENERATORS = + private static Map> connectionUrl = Map.of( Constants.SOURCE_MYSQL, shard -> "jdbc:mysql://" + shard.getHost() + ":" + shard.getPort() + "/" + shard.getDbName()); + // for unit testing purposes + public static void setConnectionHelperMap(Map connectionHelper) { + connectionHelperMap = connectionHelper; + } + /** * Creates a SourceProcessor instance for the specified source type. * @@ -56,43 +62,44 @@ public class SourceProcessorFactory { * @throws Exception if the source type is invalid */ public static SourceProcessor createSourceProcessor( - String source, List shards, int maxConnections) throws Exception { + String source, List shards, int maxConnections) throws InvalidSourceException { IDMLGenerator dmlGenerator = getDMLGenerator(source); - String driver = getDriver(source); - IConnectionHelper connectionHelper = - initializeConnectionHelper(source, shards, maxConnections, driver); + IConnectionHelper connectionHelper = initializeConnectionHelper(source, shards, maxConnections); Map sourceDaoMap = createSourceDaoMap(source, shards, connectionHelper); return SourceProcessor.builder().dmlGenerator(dmlGenerator).sourceDaoMap(sourceDaoMap).build(); } - private static IDMLGenerator getDMLGenerator(String source) throws Exception { - return Optional.ofNullable(DML_GENERATOR_MAP.get(source)) - .orElseThrow(() -> new Exception("Invalid source type for DML generator: " + source)); - } - - private static String getDriver(String source) throws Exception { - return Optional.ofNullable(DRIVER_MAP.get(source)) - .orElseThrow(() -> new Exception("Invalid source type for driver: " + source)); + private static IDMLGenerator getDMLGenerator(String source) throws InvalidSourceException { + return Optional.ofNullable(dmlGeneratorMap.get(source)) + .orElseThrow( + () -> new InvalidSourceException("Invalid source type for DML generator: " + source)); } - private static IConnectionHelper getConnectionHelper(String source) throws Exception { - return Optional.ofNullable(CONNECTION_HELPER_MAP.get(source)) - .orElseThrow(() -> new Exception("Invalid source type for connection helper: " + source)); + private static IConnectionHelper getConnectionHelper(String source) + throws InvalidSourceException { + return Optional.ofNullable(connectionHelperMap.get(source)) + .orElseThrow( + () -> + new InvalidSourceException("Invalid source type for connection helper: " + source)); } private static IConnectionHelper initializeConnectionHelper( - String source, List shards, int maxConnections, String driver) throws Exception { + String source, List shards, int maxConnections) throws InvalidSourceException { IConnectionHelper connectionHelper = getConnectionHelper(source); connectionHelper.init(new ConnectionHelperRequest(shards, null, maxConnections)); return connectionHelper; } private static Map createSourceDaoMap( - String source, List shards, IConnectionHelper connectionHelper) throws Exception { + String source, List shards, IConnectionHelper connectionHelper) + throws InvalidSourceException { Function urlGenerator = - Optional.ofNullable(CONNECTION_URL_GENERATORS.get(source)) - .orElseThrow(() -> new Exception("Invalid source type for URL generation: " + source)); + Optional.ofNullable(connectionUrl.get(source)) + .orElseThrow( + () -> + new InvalidSourceException( + "Invalid source type for URL generation: " + source)); Map sourceDaoMap = new HashMap<>(); for (Shard shard : shards) { diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/transforms/SourceWriterFn.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/transforms/SourceWriterFn.java index 8844ea14f3..beddf13369 100644 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/transforms/SourceWriterFn.java +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/transforms/SourceWriterFn.java @@ -32,11 +32,11 @@ import com.google.cloud.teleport.v2.templates.constants.Constants; import com.google.cloud.teleport.v2.templates.dao.source.IDao; import com.google.cloud.teleport.v2.templates.dao.spanner.SpannerDao; +import com.google.cloud.teleport.v2.templates.exceptions.ConnectionException; import com.google.cloud.teleport.v2.templates.processor.SourceProcessor; import com.google.cloud.teleport.v2.templates.processor.SourceProcessorFactory; import com.google.cloud.teleport.v2.templates.utils.InputRecordProcessor; import com.google.cloud.teleport.v2.templates.utils.ShadowTableRecord; -import com.google.cloud.teleport.v2.templates.utils.connection.ConnectionException; import com.google.common.collect.ImmutableList; import com.google.gson.Gson; import java.io.Serializable; diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/connection/IConnectionHelper.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/connection/IConnectionHelper.java index 3cfe0b8c80..aeaf49b882 100644 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/connection/IConnectionHelper.java +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/connection/IConnectionHelper.java @@ -15,6 +15,7 @@ */ package com.google.cloud.teleport.v2.templates.utils.connection; +import com.google.cloud.teleport.v2.templates.exceptions.ConnectionException; import com.google.cloud.teleport.v2.templates.models.ConnectionHelperRequest; public interface IConnectionHelper { diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/connection/MySQLConnectionHelper.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/connection/MySQLConnectionHelper.java index 3f264e5b2a..29bf4a5c42 100644 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/connection/MySQLConnectionHelper.java +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/connection/MySQLConnectionHelper.java @@ -16,6 +16,7 @@ package com.google.cloud.teleport.v2.templates.utils.connection; import com.google.cloud.teleport.v2.spanner.migrations.shard.Shard; +import com.google.cloud.teleport.v2.templates.exceptions.ConnectionException; import com.google.cloud.teleport.v2.templates.models.ConnectionHelperRequest; import com.zaxxer.hikari.HikariConfig; import com.zaxxer.hikari.HikariDataSource; @@ -45,13 +46,7 @@ public synchronized void init(ConnectionHelperRequest connectionHelperRequest) { connectionPoolMap = new HashMap<>(); for (Shard shard : connectionHelperRequest.getShards()) { String sourceConnectionUrl = - "jdbc:mysql://" - + "://" - + shard.getHost() - + ":" - + shard.getPort() - + "/" - + shard.getDbName(); + "jdbc:mysql://" + shard.getHost() + ":" + shard.getPort() + "/" + shard.getDbName(); HikariConfig config = new HikariConfig(); config.setJdbcUrl(sourceConnectionUrl); config.setUsername(shard.getUserName()); diff --git a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/dao/JdbcDaoTest.java b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/dao/JdbcDaoTest.java index 7afddcd9e0..746e5c922d 100644 --- a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/dao/JdbcDaoTest.java +++ b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/dao/JdbcDaoTest.java @@ -22,7 +22,7 @@ import static org.mockito.Mockito.when; import com.google.cloud.teleport.v2.templates.dao.source.JdbcDao; -import com.google.cloud.teleport.v2.templates.utils.connection.ConnectionException; +import com.google.cloud.teleport.v2.templates.exceptions.ConnectionException; import com.google.cloud.teleport.v2.templates.utils.connection.MySQLConnectionHelper; import com.zaxxer.hikari.HikariDataSource; import java.sql.Connection; diff --git a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/processor/SourceProcessorFactoryTest.java b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/processor/SourceProcessorFactoryTest.java new file mode 100644 index 0000000000..dd0a9870f4 --- /dev/null +++ b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/processor/SourceProcessorFactoryTest.java @@ -0,0 +1,85 @@ +/* + * Copyright (C) 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.templates.processor; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doNothing; + +import com.google.cloud.teleport.v2.spanner.migrations.shard.Shard; +import com.google.cloud.teleport.v2.templates.constants.Constants; +import com.google.cloud.teleport.v2.templates.dao.source.JdbcDao; +import com.google.cloud.teleport.v2.templates.dml.MySQLDMLGenerator; +import com.google.cloud.teleport.v2.templates.exceptions.InvalidSourceException; +import com.google.cloud.teleport.v2.templates.utils.connection.MySQLConnectionHelper; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.mockito.Mockito; + +@RunWith(JUnit4.class) +public class SourceProcessorFactoryTest { + @Test + public void testCreateSourceProcessor_validSource() throws Exception { + List shards = + Arrays.asList( + new Shard( + "shard1", + "localhost", + "3306", + "myuser", + "mypassword", + "mydatabase", + "mynamespace", + "projects/myproject/secrets/mysecret/versions/latest", + "")); + int maxConnections = 10; + MySQLConnectionHelper mockConnectionHelper = Mockito.mock(MySQLConnectionHelper.class); + doNothing().when(mockConnectionHelper).init(any()); + SourceProcessorFactory.setConnectionHelperMap( + Map.of(Constants.SOURCE_MYSQL, mockConnectionHelper)); + SourceProcessor processor = + SourceProcessorFactory.createSourceProcessor( + Constants.SOURCE_MYSQL, shards, maxConnections); + + Assert.assertNotNull(processor); + Assert.assertTrue(processor.getDmlGenerator() instanceof MySQLDMLGenerator); + Assert.assertEquals(1, processor.getSourceDaoMap().size()); + Assert.assertTrue(processor.getSourceDaoMap().get("shard1") instanceof JdbcDao); + } + + @Test(expected = InvalidSourceException.class) + public void testCreateSourceProcessor_invalidSource() throws Exception { + List shards = + Arrays.asList( + new Shard( + "shard1", + "localhost", + "3306", + "myuser", + "mypassword", + "mydatabase", + "mynamespace", + "projects/myproject/secrets/mysecret/versions/latest", + "")); + int maxConnections = 10; + + SourceProcessorFactory.createSourceProcessor("invalid_source", shards, maxConnections); + } +} From f561fd89e3d1a47bfc07c60855eb19a88aa4faf9 Mon Sep 17 00:00:00 2001 From: Shreya Khajanchi Date: Wed, 20 Nov 2024 15:29:12 +0530 Subject: [PATCH 08/13] addressing some comments --- .../v2/templates/constants/package-info.java | 2 +- .../v2/templates/dao/package-info.java | 2 +- .../v2/templates/dao/source/package-info.java | 2 +- .../templates/dao/spanner/package-info.java | 2 +- .../v2/templates/dml/MySQLDMLGenerator.java | 75 +++++++++---------- .../v2/templates/dml/package-info.java | 2 +- .../v2/templates/exceptions/package-info.java | 2 +- .../v2/templates/models/package-info.java | 2 +- .../v2/templates/processor/package-info.java | 2 +- .../templates/transforms/SourceWriterFn.java | 8 +- .../templates/utils/InputRecordProcessor.java | 2 +- .../utils/connection/package-info.java | 2 +- .../connection/MySQLConnectionHelperTest.java | 4 + 13 files changed, 54 insertions(+), 53 deletions(-) create mode 100644 v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/utils/connection/MySQLConnectionHelperTest.java diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/constants/package-info.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/constants/package-info.java index 048d1a51b6..1f0ad4724e 100644 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/constants/package-info.java +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/constants/package-info.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2023 Google LLC + * Copyright (C) 2024 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dao/package-info.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dao/package-info.java index 61fcc6af59..2a53019c17 100644 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dao/package-info.java +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dao/package-info.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2023 Google Inc. + * Copyright (C) 2024 Google Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dao/source/package-info.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dao/source/package-info.java index e3fb254651..dbd1d5f6d1 100644 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dao/source/package-info.java +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dao/source/package-info.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2023 Google Inc. + * Copyright (C) 2024 Google Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dao/spanner/package-info.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dao/spanner/package-info.java index 6ec95edbed..9473df0800 100644 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dao/spanner/package-info.java +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dao/spanner/package-info.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2023 Google Inc. + * Copyright (C) 2024 Google Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dml/MySQLDMLGenerator.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dml/MySQLDMLGenerator.java index 0cea58f276..f6f1f394e3 100644 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dml/MySQLDMLGenerator.java +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dml/MySQLDMLGenerator.java @@ -76,48 +76,26 @@ public DMLGeneratorResponse getDMLStatement(DMLGeneratorRequest dmlGeneratorRequ return new DMLGeneratorResponse(""); } + Map pkcolumnNameValues = + getPkColumnValues( + spannerTable, + sourceTable, + dmlGeneratorRequest.getNewValuesJson(), + dmlGeneratorRequest.getKeyValuesJson(), + dmlGeneratorRequest.getSourceDbTimezoneOffset()); + if (pkcolumnNameValues == null) { + LOG.warn( + "Cannot reverse replicate for table {} without primary key, skipping the record", + sourceTable.getName()); + return new DMLGeneratorResponse(""); + } + if ("INSERT".equals(dmlGeneratorRequest.getModType()) || "UPDATE".equals(dmlGeneratorRequest.getModType())) { - Map pkcolumnNameValues = - getPkColumnValues( - spannerTable, - sourceTable, - dmlGeneratorRequest.getNewValuesJson(), - dmlGeneratorRequest.getKeyValuesJson(), - dmlGeneratorRequest.getSourceDbTimezoneOffset()); - if (pkcolumnNameValues == null) { - LOG.warn( - "Cannot reverse replicate for table {} without primary key, skipping the record", - sourceTable.getName()); - return new DMLGeneratorResponse(""); - } - Map columnNameValues = - getColumnValues( - spannerTable, - sourceTable, - dmlGeneratorRequest.getNewValuesJson(), - dmlGeneratorRequest.getKeyValuesJson(), - dmlGeneratorRequest.getSourceDbTimezoneOffset()); - return getUpsertStatement( - sourceTable.getName(), - sourceTable.getPrimaryKeySet(), - columnNameValues, - pkcolumnNameValues); - } else if ("DELETE".equals(dmlGeneratorRequest.getModType())) { + return generateUpsertStatement( + spannerTable, sourceTable, dmlGeneratorRequest, pkcolumnNameValues); - Map pkcolumnNameValues = - getPkColumnValues( - spannerTable, - sourceTable, - dmlGeneratorRequest.getNewValuesJson(), - dmlGeneratorRequest.getKeyValuesJson(), - dmlGeneratorRequest.getSourceDbTimezoneOffset()); - if (pkcolumnNameValues == null) { - LOG.warn( - "Cannot reverse replicate for table {} without primary key, skipping the record", - sourceTable.getName()); - return new DMLGeneratorResponse(""); - } + } else if ("DELETE".equals(dmlGeneratorRequest.getModType())) { return getDeleteStatement(sourceTable.getName(), pkcolumnNameValues); } else { LOG.warn("Unsupported modType: " + dmlGeneratorRequest.getModType()); @@ -205,6 +183,25 @@ private static DMLGeneratorResponse getDeleteStatement( return new DMLGeneratorResponse(returnVal); } + private static DMLGeneratorResponse generateUpsertStatement( + SpannerTable spannerTable, + SourceTable sourceTable, + DMLGeneratorRequest dmlGeneratorRequest, + Map pkcolumnNameValues) { + Map columnNameValues = + getColumnValues( + spannerTable, + sourceTable, + dmlGeneratorRequest.getNewValuesJson(), + dmlGeneratorRequest.getKeyValuesJson(), + dmlGeneratorRequest.getSourceDbTimezoneOffset()); + return getUpsertStatement( + sourceTable.getName(), + sourceTable.getPrimaryKeySet(), + columnNameValues, + pkcolumnNameValues); + } + private static Map getColumnValues( SpannerTable spannerTable, SourceTable sourceTable, diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dml/package-info.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dml/package-info.java index e0cc0c996c..c3444a47fc 100644 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dml/package-info.java +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dml/package-info.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2023 Google Inc. + * Copyright (C) 2024 Google Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/exceptions/package-info.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/exceptions/package-info.java index 60d38f6de9..64f8540d24 100644 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/exceptions/package-info.java +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/exceptions/package-info.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2023 Google LLC + * Copyright (C) 2024 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/models/package-info.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/models/package-info.java index d49a3a3fe9..6e350acc80 100644 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/models/package-info.java +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/models/package-info.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2023 Google Inc. + * Copyright (C) 2024 Google Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/processor/package-info.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/processor/package-info.java index 80fe4035aa..c9b5e85854 100644 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/processor/package-info.java +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/processor/package-info.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2023 Google Inc. + * Copyright (C) 2024 Google Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/transforms/SourceWriterFn.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/transforms/SourceWriterFn.java index beddf13369..630273427e 100644 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/transforms/SourceWriterFn.java +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/transforms/SourceWriterFn.java @@ -33,6 +33,7 @@ import com.google.cloud.teleport.v2.templates.dao.source.IDao; import com.google.cloud.teleport.v2.templates.dao.spanner.SpannerDao; import com.google.cloud.teleport.v2.templates.exceptions.ConnectionException; +import com.google.cloud.teleport.v2.templates.exceptions.InvalidSourceException; import com.google.cloud.teleport.v2.templates.processor.SourceProcessor; import com.google.cloud.teleport.v2.templates.processor.SourceProcessorFactory; import com.google.cloud.teleport.v2.templates.utils.InputRecordProcessor; @@ -84,7 +85,7 @@ public class SourceWriterFn extends DoFn shards, @@ -125,7 +126,7 @@ public void setSourceProcessor(SourceProcessor sourceProcessor) { /** Setup function connects to Cloud Spanner. */ @Setup - public void setup() throws Exception { + public void setup() throws InvalidSourceException { mapper = new ObjectMapper(); mapper.enable(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS); sourceProcessor = @@ -182,8 +183,7 @@ public void processElement(ProcessContext c) { if (!isSourceAhead) { IDao sourceDao = sourceProcessor.getSourceDaoMap().get(shardId); - InputRecordProcessor inputRecordProcessor = new InputRecordProcessor(); - inputRecordProcessor.processRecord( + InputRecordProcessor.processRecord( spannerRec, schema, sourceDao, diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/InputRecordProcessor.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/InputRecordProcessor.java index c7fccd630a..fd26e3b97f 100644 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/InputRecordProcessor.java +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/InputRecordProcessor.java @@ -36,7 +36,7 @@ public class InputRecordProcessor { private static final Logger LOG = LoggerFactory.getLogger(InputRecordProcessor.class); - public void processRecord( + public static void processRecord( TrimmedShardedDataChangeRecord spannerRecord, Schema schema, IDao dao, diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/connection/package-info.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/connection/package-info.java index 0cb2b03850..e5e22823ee 100644 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/connection/package-info.java +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/connection/package-info.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2023 Google Inc. + * Copyright (C) 2024 Google Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of diff --git a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/utils/connection/MySQLConnectionHelperTest.java b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/utils/connection/MySQLConnectionHelperTest.java new file mode 100644 index 0000000000..16a3aa9895 --- /dev/null +++ b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/utils/connection/MySQLConnectionHelperTest.java @@ -0,0 +1,4 @@ +package com.google.cloud.teleport.v2.templates.utils.connection; + +public class MySQLConnectionHelperTest { +} From e37979603287440524a8dc44844d36181ae297cf Mon Sep 17 00:00:00 2001 From: Akash Thawait Date: Wed, 27 Nov 2024 15:03:22 +0530 Subject: [PATCH 09/13] Added connection class for cassandra and created connection helper --- .../migrations/cassandra/CassandraConfig.java | 209 ++++++++++++++++++ .../migrations/cassandra/package-info.java | 1 + .../v2/templates/dao/source/CassandraDao.java | 36 +++ .../models/ConnectionHelperRequest.java | 13 +- .../connection/CassandraConnectionHelper.java | 70 ++++++ 5 files changed, 328 insertions(+), 1 deletion(-) create mode 100644 v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/migrations/cassandra/CassandraConfig.java create mode 100644 v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/migrations/cassandra/package-info.java create mode 100644 v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dao/source/CassandraDao.java create mode 100644 v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/connection/CassandraConnectionHelper.java diff --git a/v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/migrations/cassandra/CassandraConfig.java b/v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/migrations/cassandra/CassandraConfig.java new file mode 100644 index 0000000000..8b1b085116 --- /dev/null +++ b/v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/migrations/cassandra/CassandraConfig.java @@ -0,0 +1,209 @@ +package com.google.cloud.teleport.v2.spanner.migrations.cassandra; + +import java.io.Serializable; +import java.util.Objects; + +public class CassandraConfig implements Serializable { + + private String host; + private String port; + private String username; + private String password; + private String keyspace; + private String consistencyLevel = "LOCAL_QUORUM"; + private boolean sslOptions = false; + private String protocolVersion = "v5"; + private String dataCenter = "datacenter1"; + private int localPoolSize = 1024; + private int remotePoolSize = 256; + + public CassandraConfig() {} + + public CassandraConfig( + String host, + String port, + String username, + String password, + String keyspace, + String consistencyLevel, + boolean sslOptions, + String protocolVersion, + String dataCenter, + int localPoolSize, + int remotePoolSize) { + this.host = host; + this.port = port; + this.username = username; + this.password = password; + this.keyspace = keyspace; + this.consistencyLevel = consistencyLevel; + this.sslOptions = sslOptions; + this.protocolVersion = protocolVersion; + this.dataCenter = dataCenter; + this.localPoolSize = localPoolSize; + this.remotePoolSize = remotePoolSize; + } + + // Getter and Setter methods for all fields + + public String getHost() { + return host; + } + + public void setHost(String host) { + this.host = host; + } + + public String getPort() { + return port; + } + + public void setPort(String port) { + this.port = port; + } + + public String getUsername() { + return username; + } + + public void setUsername(String username) { + this.username = username; + } + + public String getPassword() { + return password; + } + + public void setPassword(String password) { + this.password = password; + } + + public String getKeyspace() { + return keyspace; + } + + public void setKeyspace(String keyspace) { + this.keyspace = keyspace; + } + + public String getConsistencyLevel() { + return consistencyLevel; + } + + public void setConsistencyLevel(String consistencyLevel) { + this.consistencyLevel = consistencyLevel; + } + + public boolean isSslOptions() { + return sslOptions; + } + + public void setSslOptions(boolean sslOptions) { + this.sslOptions = sslOptions; + } + + public String getProtocolVersion() { + return protocolVersion; + } + + public void setProtocolVersion(String protocolVersion) { + this.protocolVersion = protocolVersion; + } + + public String getDataCenter() { + return dataCenter; + } + + public void setDataCenter(String dataCenter) { + this.dataCenter = dataCenter; + } + + public int getLocalPoolSize() { + return localPoolSize; + } + + public void setLocalPoolSize(int localPoolSize) { + this.localPoolSize = localPoolSize; + } + + public int getRemotePoolSize() { + return remotePoolSize; + } + + public void setRemotePoolSize(int remotePoolSize) { + this.remotePoolSize = remotePoolSize; + } + + public void validate() throws IllegalArgumentException { + if (host == null || host.isEmpty()) { + throw new IllegalArgumentException("Host is required"); + } + if (port == null || port.isEmpty()) { + throw new IllegalArgumentException("Port is required"); + } + if (username == null || username.isEmpty()) { + throw new IllegalArgumentException("Username is required"); + } + if (password == null || password.isEmpty()) { + throw new IllegalArgumentException("Password is required"); + } + if (keyspace == null || keyspace.isEmpty()) { + throw new IllegalArgumentException("Keyspace is required"); + } + } + + @Override + public String toString() { + return "CassandraConfig{" + + "host='" + host + '\'' + + ", port='" + port + '\'' + + ", username='" + username + '\'' + + ", password='" + password + '\'' + + ", keyspace='" + keyspace + '\'' + + ", consistencyLevel='" + consistencyLevel + '\'' + + ", sslOptions=" + sslOptions + + ", protocolVersion='" + protocolVersion + '\'' + + ", dataCenter='" + dataCenter + '\'' + + ", localPoolSize=" + localPoolSize + + ", remotePoolSize=" + remotePoolSize + + '}'; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof CassandraConfig)) { + return false; + } + CassandraConfig cassandraConfig = (CassandraConfig) o; + return sslOptions == cassandraConfig.sslOptions + && localPoolSize == cassandraConfig.localPoolSize + && remotePoolSize == cassandraConfig.remotePoolSize + && Objects.equals(host, cassandraConfig.host) + && Objects.equals(port, cassandraConfig.port) + && Objects.equals(username, cassandraConfig.username) + && Objects.equals(password, cassandraConfig.password) + && Objects.equals(keyspace, cassandraConfig.keyspace) + && Objects.equals(consistencyLevel, cassandraConfig.consistencyLevel) + && Objects.equals(protocolVersion, cassandraConfig.protocolVersion) + && Objects.equals(dataCenter, cassandraConfig.dataCenter); + } + + @Override + public int hashCode() { + return Objects.hash( + host, + port, + username, + password, + keyspace, + consistencyLevel, + sslOptions, + protocolVersion, + dataCenter, + localPoolSize, + remotePoolSize); + } +} \ No newline at end of file diff --git a/v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/migrations/cassandra/package-info.java b/v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/migrations/cassandra/package-info.java new file mode 100644 index 0000000000..823c50ef83 --- /dev/null +++ b/v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/migrations/cassandra/package-info.java @@ -0,0 +1 @@ +//package com.google.cloud.teleport.v2.spanner.migrations.cassandra; \ No newline at end of file diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dao/source/CassandraDao.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dao/source/CassandraDao.java new file mode 100644 index 0000000000..5be2b60fa8 --- /dev/null +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dao/source/CassandraDao.java @@ -0,0 +1,36 @@ +package com.google.cloud.teleport.v2.templates.dao.source; + +import com.datastax.oss.driver.api.core.CqlSession; +import com.datastax.oss.driver.api.core.cql.SimpleStatement; +import com.google.cloud.teleport.v2.templates.exceptions.ConnectionException; +import com.google.cloud.teleport.v2.templates.utils.connection.IConnectionHelper; + +public class CassandraDao implements IDao { + private final String cassandraUrl; + private final String cassandraUser; + private final IConnectionHelper connectionHelper; + + public CassandraDao(String cassandraUrl, String cassandraUser, IConnectionHelper connectionHelper) { + this.cassandraUrl = cassandraUrl; + this.cassandraUser = cassandraUser; + this.connectionHelper = connectionHelper; + } + + @Override + public void write(String cqlStatement) throws Exception { + CqlSession session = null; + + try { + session = connectionHelper.getConnection(this.cassandraUrl + "/" + this.cassandraUser); + if (session == null) { + throw new ConnectionException("Connection is null"); + } + SimpleStatement statement = SimpleStatement.newInstance(cqlStatement); + session.execute(statement); + } finally { + if (session != null) { + session.close(); + } + } + } +} \ No newline at end of file diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/models/ConnectionHelperRequest.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/models/ConnectionHelperRequest.java index ec19a5c20a..862dac28a2 100644 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/models/ConnectionHelperRequest.java +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/models/ConnectionHelperRequest.java @@ -15,6 +15,7 @@ */ package com.google.cloud.teleport.v2.templates.models; +import com.google.cloud.teleport.v2.spanner.migrations.cassandra.CassandraConfig; import com.google.cloud.teleport.v2.spanner.migrations.shard.Shard; import java.util.List; @@ -33,7 +34,8 @@ public class ConnectionHelperRequest { private List shards; private String properties; - private int maxConnections; + private final int maxConnections; + private CassandraConfig cassandraConfig; public List getShards() { return shards; @@ -47,9 +49,18 @@ public int getMaxConnections() { return maxConnections; } + public CassandraConfig getCassandraConfig() { + return cassandraConfig; + } + public ConnectionHelperRequest(List shards, String properties, int maxConnections) { this.shards = shards; this.properties = properties; this.maxConnections = maxConnections; } + + public ConnectionHelperRequest(CassandraConfig cassandraConfig, int maxConnections) { + this.cassandraConfig = cassandraConfig; + this.maxConnections = maxConnections; + } } diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/connection/CassandraConnectionHelper.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/connection/CassandraConnectionHelper.java new file mode 100644 index 0000000000..0f313c328e --- /dev/null +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/connection/CassandraConnectionHelper.java @@ -0,0 +1,70 @@ +package com.google.cloud.teleport.v2.templates.utils.connection; + +import com.datastax.oss.driver.api.core.CqlSession; +import com.datastax.oss.driver.api.core.CqlSessionBuilder; +import com.datastax.oss.driver.api.core.config.DriverConfigLoader; +import com.datastax.oss.driver.api.core.config.DriverOption; +import com.datastax.oss.driver.api.core.config.ProgrammaticDriverConfigLoaderBuilder; +import com.datastax.oss.driver.api.core.config.TypedDriverOption; +import com.google.cloud.teleport.v2.spanner.migrations.cassandra.CassandraConfig; +import com.google.cloud.teleport.v2.templates.exceptions.ConnectionException; +import com.google.cloud.teleport.v2.templates.models.ConnectionHelperRequest; +import java.net.InetSocketAddress; +import java.util.HashMap; +import java.util.Map; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class CassandraConnectionHelper implements IConnectionHelper { + + private static final Logger LOG = LoggerFactory.getLogger(CassandraConnectionHelper.class); + private static Map connectionPoolMap = null; + + @Override + public synchronized void init(ConnectionHelperRequest connectionHelperRequest) { + if (connectionPoolMap != null) { + return; + } + LOG.info("Initializing Cassandra connection pool with size: {}", connectionHelperRequest.getMaxConnections()); + connectionPoolMap = new HashMap<>(); + CassandraConfig cassandraConfig = connectionHelperRequest.getCassandraConfig(); + cassandraConfig.validate(); + + CqlSessionBuilder builder = CqlSession.builder() + .addContactPoint(new InetSocketAddress(cassandraConfig.getHost(), Integer.parseInt(cassandraConfig.getPort()))) + .withAuthCredentials(cassandraConfig.getUsername(), cassandraConfig.getPassword()) + .withKeyspace(cassandraConfig.getKeyspace()); + + ProgrammaticDriverConfigLoaderBuilder configLoaderBuilder = DriverConfigLoader.programmaticBuilder(); + configLoaderBuilder.withInt((DriverOption) TypedDriverOption.CONNECTION_POOL_LOCAL_SIZE, cassandraConfig.getLocalPoolSize()); + configLoaderBuilder.withInt((DriverOption) TypedDriverOption.CONNECTION_POOL_REMOTE_SIZE, cassandraConfig.getRemotePoolSize()); + builder.withConfigLoader(configLoaderBuilder.build()); + + CqlSession session = builder.build(); + String connectionKey = cassandraConfig.getHost() + ":" + cassandraConfig.getPort() + "/" + cassandraConfig.getUsername(); + connectionPoolMap.put(connectionKey, session); + } + + @Override + public CqlSession getConnection(String connectionRequestKey) throws ConnectionException { + try { + if (connectionPoolMap == null) { + LOG.warn("Connection pool not initialized"); + return null; + } + CqlSession session = connectionPoolMap.get(connectionRequestKey); + if (session == null) { + LOG.warn("Connection pool not found for source connection: {}", connectionRequestKey); + return null; + } + return session; + } catch (Exception e) { + throw new ConnectionException(e); + } + } + + // for unit testing + public void setConnectionPoolMap(Map inputMap) { + connectionPoolMap = inputMap; + } +} \ No newline at end of file From 4b61fea026d231fb63a1524781ae81def4d35735 Mon Sep 17 00:00:00 2001 From: Akash Thawait Date: Thu, 28 Nov 2024 14:25:48 +0530 Subject: [PATCH 10/13] resolved conflict changes --- v2/spanner-to-sourcedb/pom.xml | 6 + .../v2/templates/dao/package-info.java | 18 --- .../v2/templates/dao/source/IDao.java | 26 ---- .../v2/templates/dao/source/package-info.java | 18 --- .../templates/dao/spanner/package-info.java | 18 --- .../dao/source/CassandraDao.java | 3 +- .../v2/templates/dml/IDMLGenerator.java | 40 ------- .../v2/templates/dml/package-info.java | 18 --- .../templates/processor/SourceProcessor.java | 74 ------------ .../processor/SourceProcessorFactory.java | 112 ------------------ .../v2/templates/processor/package-info.java | 18 --- 11 files changed, 8 insertions(+), 343 deletions(-) delete mode 100644 v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dao/package-info.java delete mode 100644 v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dao/source/IDao.java delete mode 100644 v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dao/source/package-info.java delete mode 100644 v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dao/spanner/package-info.java rename v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/{ => dbutils}/dao/source/CassandraDao.java (90%) delete mode 100644 v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dml/IDMLGenerator.java delete mode 100644 v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dml/package-info.java delete mode 100644 v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/processor/SourceProcessor.java delete mode 100644 v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/processor/SourceProcessorFactory.java delete mode 100644 v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/processor/package-info.java diff --git a/v2/spanner-to-sourcedb/pom.xml b/v2/spanner-to-sourcedb/pom.xml index 841405fb9a..be16f83834 100644 --- a/v2/spanner-to-sourcedb/pom.xml +++ b/v2/spanner-to-sourcedb/pom.xml @@ -89,6 +89,12 @@ ${project.version} test + + com.datastax.oss + java-driver-core + 4.17.0 + compile + diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dao/package-info.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dao/package-info.java deleted file mode 100644 index 2a53019c17..0000000000 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dao/package-info.java +++ /dev/null @@ -1,18 +0,0 @@ -/* - * Copyright (C) 2024 Google Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -/** Package info for dao module. */ -package com.google.cloud.teleport.v2.templates.dao; diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dao/source/IDao.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dao/source/IDao.java deleted file mode 100644 index 13974c1f10..0000000000 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dao/source/IDao.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright (C) 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ -package com.google.cloud.teleport.v2.templates.dao.source; - -public interface IDao { - /** - * Executes a given write statement against the data source. - * - * @param statement The SQL or query statement. - * @throws Exception If there is an error executing the statement. - */ - void write(T statement) throws Exception; -} diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dao/source/package-info.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dao/source/package-info.java deleted file mode 100644 index dbd1d5f6d1..0000000000 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dao/source/package-info.java +++ /dev/null @@ -1,18 +0,0 @@ -/* - * Copyright (C) 2024 Google Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -/** Package info for source module. */ -package com.google.cloud.teleport.v2.templates.dao.source; diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dao/spanner/package-info.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dao/spanner/package-info.java deleted file mode 100644 index 9473df0800..0000000000 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dao/spanner/package-info.java +++ /dev/null @@ -1,18 +0,0 @@ -/* - * Copyright (C) 2024 Google Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -/** Package info for spanner module. */ -package com.google.cloud.teleport.v2.templates.dao.spanner; diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dao/source/CassandraDao.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dbutils/dao/source/CassandraDao.java similarity index 90% rename from v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dao/source/CassandraDao.java rename to v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dbutils/dao/source/CassandraDao.java index 5be2b60fa8..8ac95bec5f 100644 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dao/source/CassandraDao.java +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dbutils/dao/source/CassandraDao.java @@ -1,7 +1,8 @@ -package com.google.cloud.teleport.v2.templates.dao.source; +package com.google.cloud.teleport.v2.templates.dbutils.dao.source; import com.datastax.oss.driver.api.core.CqlSession; import com.datastax.oss.driver.api.core.cql.SimpleStatement; +import com.google.cloud.teleport.v2.templates.dao.source.IDao; import com.google.cloud.teleport.v2.templates.exceptions.ConnectionException; import com.google.cloud.teleport.v2.templates.utils.connection.IConnectionHelper; diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dml/IDMLGenerator.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dml/IDMLGenerator.java deleted file mode 100644 index 31923f6d6d..0000000000 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dml/IDMLGenerator.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright (C) 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ -package com.google.cloud.teleport.v2.templates.dml; - -import com.google.cloud.teleport.v2.templates.models.DMLGeneratorRequest; -import com.google.cloud.teleport.v2.templates.models.DMLGeneratorResponse; - -/** - * Interface for generating DML statements. - * - *

This interface provides a contract for implementing classes to define the logic for generating - * DML statements, such as INSERT, UPDATE, and DELETE, based on a provided {@link - * DMLGeneratorRequest}. - * - *

Classes implementing this interface should handle the construction of DML statements tailored - * to the specific requirements of the target database. - */ -public interface IDMLGenerator { - /** - * Generates a DML statement based on the provided {@link DMLGeneratorRequest}. - * - * @param dmlGeneratorRequest the request containing necessary information to construct the DML - * statement, including modification type, table schema, new values, and key values. - * @return a {@link DMLGeneratorResponse} object containing the generated DML statement. - */ - DMLGeneratorResponse getDMLStatement(DMLGeneratorRequest dmlGeneratorRequest); -} diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dml/package-info.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dml/package-info.java deleted file mode 100644 index c3444a47fc..0000000000 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dml/package-info.java +++ /dev/null @@ -1,18 +0,0 @@ -/* - * Copyright (C) 2024 Google Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -/** Package info for dml module. */ -package com.google.cloud.teleport.v2.templates.dml; diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/processor/SourceProcessor.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/processor/SourceProcessor.java deleted file mode 100644 index 344a73162e..0000000000 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/processor/SourceProcessor.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Copyright (C) 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ -package com.google.cloud.teleport.v2.templates.processor; - -import com.google.cloud.teleport.v2.templates.dao.source.IDao; -import com.google.cloud.teleport.v2.templates.dml.IDMLGenerator; -import java.util.HashMap; -import java.util.Map; - -/** - * Represents a processor responsible for managing source database operations. This class - * encapsulates a DML generator and a map of DAOs for interacting with source databases. - */ -public class SourceProcessor { - - private final IDMLGenerator dmlGenerator; - private final Map sourceDaoMap; - - /** - * Constructs a SourceProcessor with the specified DML generator and source DAO map. - * - * @param dmlGenerator the DML generator for the source - * @param sourceDaoMap the map of shard ID to DAO for the source - */ - private SourceProcessor(IDMLGenerator dmlGenerator, Map sourceDaoMap) { - this.dmlGenerator = dmlGenerator; - this.sourceDaoMap = sourceDaoMap; - } - - public IDMLGenerator getDmlGenerator() { - return dmlGenerator; - } - - public Map getSourceDaoMap() { - return sourceDaoMap; - } - - public static Builder builder() { - return new Builder(); - } - - /** Builder class for constructing instances of {@link SourceProcessor}. */ - public static class Builder { - private IDMLGenerator dmlGenerator; - private Map sourceDaoMap = new HashMap<>(); - - public Builder dmlGenerator(IDMLGenerator dmlGenerator) { - this.dmlGenerator = dmlGenerator; - return this; - } - - public Builder sourceDaoMap(Map sourceDaoMap) { - this.sourceDaoMap = sourceDaoMap; - return this; - } - - public SourceProcessor build() { - return new SourceProcessor(dmlGenerator, sourceDaoMap); - } - } -} diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/processor/SourceProcessorFactory.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/processor/SourceProcessorFactory.java deleted file mode 100644 index a754232863..0000000000 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/processor/SourceProcessorFactory.java +++ /dev/null @@ -1,112 +0,0 @@ -/* - * Copyright (C) 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ -package com.google.cloud.teleport.v2.templates.processor; - -import com.google.cloud.teleport.v2.spanner.migrations.shard.Shard; -import com.google.cloud.teleport.v2.templates.constants.Constants; -import com.google.cloud.teleport.v2.templates.dao.source.IDao; -import com.google.cloud.teleport.v2.templates.dao.source.JdbcDao; -import com.google.cloud.teleport.v2.templates.dml.IDMLGenerator; -import com.google.cloud.teleport.v2.templates.dml.MySQLDMLGenerator; -import com.google.cloud.teleport.v2.templates.exceptions.InvalidSourceException; -import com.google.cloud.teleport.v2.templates.models.ConnectionHelperRequest; -import com.google.cloud.teleport.v2.templates.utils.connection.IConnectionHelper; -import com.google.cloud.teleport.v2.templates.utils.connection.MySQLConnectionHelper; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.function.Function; - -public class SourceProcessorFactory { - private static Map dmlGeneratorMap = - Map.of(Constants.SOURCE_MYSQL, new MySQLDMLGenerator()); - - private static Map connectionHelperMap = - Map.of(Constants.SOURCE_MYSQL, new MySQLConnectionHelper()); - - private static Map driverMap = - Map.of(Constants.SOURCE_MYSQL, "com.mysql.cj.jdbc.Driver"); - - private static Map> connectionUrl = - Map.of( - Constants.SOURCE_MYSQL, - shard -> - "jdbc:mysql://" + shard.getHost() + ":" + shard.getPort() + "/" + shard.getDbName()); - - // for unit testing purposes - public static void setConnectionHelperMap(Map connectionHelper) { - connectionHelperMap = connectionHelper; - } - - /** - * Creates a SourceProcessor instance for the specified source type. - * - * @param source the type of the source database - * @param shards the list of shards for the source - * @param maxConnections the maximum number of connections - * @return a configured SourceProcessor instance - * @throws Exception if the source type is invalid - */ - public static SourceProcessor createSourceProcessor( - String source, List shards, int maxConnections) throws InvalidSourceException { - IDMLGenerator dmlGenerator = getDMLGenerator(source); - IConnectionHelper connectionHelper = initializeConnectionHelper(source, shards, maxConnections); - Map sourceDaoMap = createSourceDaoMap(source, shards, connectionHelper); - - return SourceProcessor.builder().dmlGenerator(dmlGenerator).sourceDaoMap(sourceDaoMap).build(); - } - - private static IDMLGenerator getDMLGenerator(String source) throws InvalidSourceException { - return Optional.ofNullable(dmlGeneratorMap.get(source)) - .orElseThrow( - () -> new InvalidSourceException("Invalid source type for DML generator: " + source)); - } - - private static IConnectionHelper getConnectionHelper(String source) - throws InvalidSourceException { - return Optional.ofNullable(connectionHelperMap.get(source)) - .orElseThrow( - () -> - new InvalidSourceException("Invalid source type for connection helper: " + source)); - } - - private static IConnectionHelper initializeConnectionHelper( - String source, List shards, int maxConnections) throws InvalidSourceException { - IConnectionHelper connectionHelper = getConnectionHelper(source); - connectionHelper.init(new ConnectionHelperRequest(shards, null, maxConnections)); - return connectionHelper; - } - - private static Map createSourceDaoMap( - String source, List shards, IConnectionHelper connectionHelper) - throws InvalidSourceException { - Function urlGenerator = - Optional.ofNullable(connectionUrl.get(source)) - .orElseThrow( - () -> - new InvalidSourceException( - "Invalid source type for URL generation: " + source)); - - Map sourceDaoMap = new HashMap<>(); - for (Shard shard : shards) { - String connectionUrl = urlGenerator.apply(shard); - IDao sqlDao = new JdbcDao(connectionUrl, shard.getUserName(), connectionHelper); - sourceDaoMap.put(shard.getLogicalShardId(), sqlDao); - } - return sourceDaoMap; - } -} diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/processor/package-info.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/processor/package-info.java deleted file mode 100644 index c9b5e85854..0000000000 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/processor/package-info.java +++ /dev/null @@ -1,18 +0,0 @@ -/* - * Copyright (C) 2024 Google Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -/** Package info for processor module. */ -package com.google.cloud.teleport.v2.templates.processor; From d03c1167fa9dfc2f8a1d88bded5cdfb9755a8d21 Mon Sep 17 00:00:00 2001 From: Akash Thawait Date: Thu, 28 Nov 2024 14:32:13 +0530 Subject: [PATCH 11/13] resolved conflict changes --- .../connection/JdbcConnectionHelper.java | 20 ------------ .../templates/dbutils/dao/source/JdbcDao.java | 7 ----- .../dbutils/dao/spanner/SpannerDao.java | 4 --- .../dbutils/dml/MySQLDMLGenerator.java | 4 --- .../exceptions/InvalidSourceException.java | 31 ------------------- .../processor/SourceProcessorFactoryTest.java | 1 - 6 files changed, 67 deletions(-) delete mode 100644 v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/exceptions/InvalidSourceException.java diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dbutils/connection/JdbcConnectionHelper.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dbutils/connection/JdbcConnectionHelper.java index 6ebba07cac..77db7c8f9c 100644 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dbutils/connection/JdbcConnectionHelper.java +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dbutils/connection/JdbcConnectionHelper.java @@ -13,11 +13,7 @@ * License for the specific language governing permissions and limitations under * the License. */ -<<<<<<<< HEAD:v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/connection/MySQLConnectionHelper.java -package com.google.cloud.teleport.v2.templates.utils.connection; -======== package com.google.cloud.teleport.v2.templates.dbutils.connection; ->>>>>>>> dev-repackaged:v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dbutils/connection/JdbcConnectionHelper.java import com.google.cloud.teleport.v2.spanner.migrations.shard.Shard; import com.google.cloud.teleport.v2.templates.exceptions.ConnectionException; @@ -34,21 +30,13 @@ import org.slf4j.LoggerFactory; /** This is a per Dataflow worker singleton that holds connection pool. */ -<<<<<<<< HEAD:v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/connection/MySQLConnectionHelper.java -public class MySQLConnectionHelper implements IConnectionHelper { - - private static final Logger LOG = LoggerFactory.getLogger(MySQLConnectionHelper.class); -======== public class JdbcConnectionHelper implements IConnectionHelper { private static final Logger LOG = LoggerFactory.getLogger(JdbcConnectionHelper.class); ->>>>>>>> dev-repackaged:v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dbutils/connection/JdbcConnectionHelper.java private static Map connectionPoolMap = null; private static final String MYSQL_JDBC_DRIVER = "com.mysql.cj.jdbc.Driver"; @Override -<<<<<<<< HEAD:v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/connection/MySQLConnectionHelper.java -======== public synchronized boolean isConnectionPoolInitialized() { if (connectionPoolMap != null) { return true; @@ -57,7 +45,6 @@ public synchronized boolean isConnectionPoolInitialized() { } @Override ->>>>>>>> dev-repackaged:v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dbutils/connection/JdbcConnectionHelper.java public synchronized void init(ConnectionHelperRequest connectionHelperRequest) { if (connectionPoolMap != null) { return; @@ -72,16 +59,9 @@ public synchronized void init(ConnectionHelperRequest connectionHelperRequest) { config.setJdbcUrl(sourceConnectionUrl); config.setUsername(shard.getUserName()); config.setPassword(shard.getPassword()); -<<<<<<<< HEAD:v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/connection/MySQLConnectionHelper.java - config.setDriverClassName(MYSQL_JDBC_DRIVER); - config.setMaximumPoolSize(connectionHelperRequest.getMaxConnections()); - config.setConnectionInitSql( - "SET SESSION net_read_timeout=1200"); // to avoid timeouts at network level layer -======== config.setDriverClassName(connectionHelperRequest.getDriver()); config.setMaximumPoolSize(connectionHelperRequest.getMaxConnections()); config.setConnectionInitSql(connectionHelperRequest.getConnectionInitQuery()); ->>>>>>>> dev-repackaged:v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dbutils/connection/JdbcConnectionHelper.java Properties jdbcProperties = new Properties(); if (connectionHelperRequest.getProperties() != null && !connectionHelperRequest.getProperties().isEmpty()) { diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dbutils/dao/source/JdbcDao.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dbutils/dao/source/JdbcDao.java index 7a8c5b7da6..82ace6e411 100644 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dbutils/dao/source/JdbcDao.java +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dbutils/dao/source/JdbcDao.java @@ -13,17 +13,10 @@ * License for the specific language governing permissions and limitations under * the License. */ -<<<<<<<< HEAD:v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dao/source/JdbcDao.java -package com.google.cloud.teleport.v2.templates.dao.source; - -import com.google.cloud.teleport.v2.templates.exceptions.ConnectionException; -import com.google.cloud.teleport.v2.templates.utils.connection.IConnectionHelper; -======== package com.google.cloud.teleport.v2.templates.dbutils.dao.source; import com.google.cloud.teleport.v2.templates.dbutils.connection.IConnectionHelper; import com.google.cloud.teleport.v2.templates.exceptions.ConnectionException; ->>>>>>>> dev-repackaged:v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dbutils/dao/source/JdbcDao.java import java.sql.Connection; import java.sql.SQLException; import java.sql.Statement; diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dbutils/dao/spanner/SpannerDao.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dbutils/dao/spanner/SpannerDao.java index 327b181b55..082caef085 100644 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dbutils/dao/spanner/SpannerDao.java +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dbutils/dao/spanner/SpannerDao.java @@ -13,11 +13,7 @@ * License for the specific language governing permissions and limitations under * the License. */ -<<<<<<<< HEAD:v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dao/spanner/SpannerDao.java -package com.google.cloud.teleport.v2.templates.dao.spanner; -======== package com.google.cloud.teleport.v2.templates.dbutils.dao.spanner; ->>>>>>>> dev-repackaged:v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dbutils/dao/spanner/SpannerDao.java import com.google.cloud.spanner.DatabaseClient; import com.google.cloud.spanner.Mutation; diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dbutils/dml/MySQLDMLGenerator.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dbutils/dml/MySQLDMLGenerator.java index 659d36c7d0..8774fedfa9 100644 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dbutils/dml/MySQLDMLGenerator.java +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dbutils/dml/MySQLDMLGenerator.java @@ -13,11 +13,7 @@ * License for the specific language governing permissions and limitations under * the License. */ -<<<<<<<< HEAD:v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dml/MySQLDMLGenerator.java -package com.google.cloud.teleport.v2.templates.dml; -======== package com.google.cloud.teleport.v2.templates.dbutils.dml; ->>>>>>>> dev-repackaged:v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dbutils/dml/MySQLDMLGenerator.java import com.google.cloud.teleport.v2.spanner.migrations.schema.ColumnPK; import com.google.cloud.teleport.v2.spanner.migrations.schema.SourceColumnDefinition; diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/exceptions/InvalidSourceException.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/exceptions/InvalidSourceException.java deleted file mode 100644 index 58e224ae79..0000000000 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/exceptions/InvalidSourceException.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (C) 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ -package com.google.cloud.teleport.v2.templates.exceptions; - -/** Exception in case of unsupported source database. */ -public class InvalidSourceException extends Exception { - public InvalidSourceException(Exception e) { - super(e); - } - - public InvalidSourceException(String message) { - super(message); - } - - public InvalidSourceException(String message, Exception e) { - super(message, e); - } -} diff --git a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/processor/SourceProcessorFactoryTest.java b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/processor/SourceProcessorFactoryTest.java index dd0a9870f4..93a318b0b7 100644 --- a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/processor/SourceProcessorFactoryTest.java +++ b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/processor/SourceProcessorFactoryTest.java @@ -22,7 +22,6 @@ import com.google.cloud.teleport.v2.templates.constants.Constants; import com.google.cloud.teleport.v2.templates.dao.source.JdbcDao; import com.google.cloud.teleport.v2.templates.dml.MySQLDMLGenerator; -import com.google.cloud.teleport.v2.templates.exceptions.InvalidSourceException; import com.google.cloud.teleport.v2.templates.utils.connection.MySQLConnectionHelper; import java.util.Arrays; import java.util.List; From 779a168561fcd46e0db290ddaf270b68548947d6 Mon Sep 17 00:00:00 2001 From: Akash Thawait Date: Thu, 28 Nov 2024 14:41:26 +0530 Subject: [PATCH 12/13] resolved conflict changes --- .../connection/CassandraConnectionHelper.java | 8 +++++- .../connection/JdbcConnectionHelper.java | 1 - .../models/ConnectionHelperRequest.java | 2 ++ .../templates/transforms/SourceWriterFn.java | 2 +- .../utils/connection/IConnectionHelper.java | 25 ------------------- .../utils/connection/package-info.java | 18 ------------- 6 files changed, 10 insertions(+), 46 deletions(-) rename v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/{utils => dbutils}/connection/CassandraConnectionHelper.java (93%) delete mode 100644 v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/connection/IConnectionHelper.java delete mode 100644 v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/connection/package-info.java diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/connection/CassandraConnectionHelper.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dbutils/connection/CassandraConnectionHelper.java similarity index 93% rename from v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/connection/CassandraConnectionHelper.java rename to v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dbutils/connection/CassandraConnectionHelper.java index 0f313c328e..e4f20f3e78 100644 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/connection/CassandraConnectionHelper.java +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dbutils/connection/CassandraConnectionHelper.java @@ -1,4 +1,4 @@ -package com.google.cloud.teleport.v2.templates.utils.connection; +package com.google.cloud.teleport.v2.templates.dbutils.connection; import com.datastax.oss.driver.api.core.CqlSession; import com.datastax.oss.driver.api.core.CqlSessionBuilder; @@ -12,6 +12,7 @@ import java.net.InetSocketAddress; import java.util.HashMap; import java.util.Map; +import com.google.cloud.teleport.v2.templates.dbutils.connection.IConnectionHelper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -63,6 +64,11 @@ public CqlSession getConnection(String connectionRequestKey) throws ConnectionEx } } + @Override + public boolean isConnectionPoolInitialized() { + return false; + } + // for unit testing public void setConnectionPoolMap(Map inputMap) { connectionPoolMap = inputMap; diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dbutils/connection/JdbcConnectionHelper.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dbutils/connection/JdbcConnectionHelper.java index 77db7c8f9c..5dfc014aba 100644 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dbutils/connection/JdbcConnectionHelper.java +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dbutils/connection/JdbcConnectionHelper.java @@ -34,7 +34,6 @@ public class JdbcConnectionHelper implements IConnectionHelper { private static final Logger LOG = LoggerFactory.getLogger(JdbcConnectionHelper.class); private static Map connectionPoolMap = null; - private static final String MYSQL_JDBC_DRIVER = "com.mysql.cj.jdbc.Driver"; @Override public synchronized boolean isConnectionPoolInitialized() { diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/models/ConnectionHelperRequest.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/models/ConnectionHelperRequest.java index 6f0cf9dfcc..13cfd7331a 100644 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/models/ConnectionHelperRequest.java +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/models/ConnectionHelperRequest.java @@ -55,6 +55,8 @@ public int getMaxConnections() { public CassandraConfig getCassandraConfig() { return cassandraConfig; + } + public String getDriver() { return driver; } diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/transforms/SourceWriterFn.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/transforms/SourceWriterFn.java index c00b311e4d..f4af6d9780 100644 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/transforms/SourceWriterFn.java +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/transforms/SourceWriterFn.java @@ -126,7 +126,7 @@ public void setSourceProcessor(SourceProcessor sourceProcessor) { /** Setup function connects to Cloud Spanner. */ @Setup - public void setup() { + public void setup() throws UnsupportedSourceException { mapper = new ObjectMapper(); mapper.enable(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS); sourceProcessor = diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/connection/IConnectionHelper.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/connection/IConnectionHelper.java deleted file mode 100644 index aeaf49b882..0000000000 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/connection/IConnectionHelper.java +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Copyright (C) 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ -package com.google.cloud.teleport.v2.templates.utils.connection; - -import com.google.cloud.teleport.v2.templates.exceptions.ConnectionException; -import com.google.cloud.teleport.v2.templates.models.ConnectionHelperRequest; - -public interface IConnectionHelper { - void init(ConnectionHelperRequest connectionHelperRequest); - - T getConnection(String connectionRequestKey) throws ConnectionException; -} diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/connection/package-info.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/connection/package-info.java deleted file mode 100644 index e5e22823ee..0000000000 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/connection/package-info.java +++ /dev/null @@ -1,18 +0,0 @@ -/* - * Copyright (C) 2024 Google Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -/** Package info for connection module. */ -package com.google.cloud.teleport.v2.templates.utils.connection; From 88ea160e6bd981e54408fb8bcc9b5d082696b489 Mon Sep 17 00:00:00 2001 From: Akash Thawait Date: Thu, 28 Nov 2024 14:53:28 +0530 Subject: [PATCH 13/13] resolved conflict changes --- .../v2/templates/dbutils/dao/JdbcDaoTest.java | 20 ----- .../templates/dbutils/dao/SpannerDaoTest.java | 8 -- .../dbutils/dml/MySQLDMLGeneratorTest.java | 10 --- .../processor/SourceProcessorFactoryTest.java | 84 ------------------- .../connection/MySQLConnectionHelperTest.java | 4 - 5 files changed, 126 deletions(-) delete mode 100644 v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/processor/SourceProcessorFactoryTest.java delete mode 100644 v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/utils/connection/MySQLConnectionHelperTest.java diff --git a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/dbutils/dao/JdbcDaoTest.java b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/dbutils/dao/JdbcDaoTest.java index efa869160c..dad21878c6 100644 --- a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/dbutils/dao/JdbcDaoTest.java +++ b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/dbutils/dao/JdbcDaoTest.java @@ -13,11 +13,7 @@ * License for the specific language governing permissions and limitations under * the License. */ -<<<<<<<< HEAD:v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/dao/JdbcDaoTest.java -package com.google.cloud.teleport.v2.templates.dao; -======== package com.google.cloud.teleport.v2.templates.dbutils.dao; ->>>>>>>> dev-repackaged:v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/dbutils/dao/JdbcDaoTest.java import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; @@ -25,15 +21,9 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -<<<<<<<< HEAD:v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/dao/JdbcDaoTest.java -import com.google.cloud.teleport.v2.templates.dao.source.JdbcDao; -import com.google.cloud.teleport.v2.templates.exceptions.ConnectionException; -import com.google.cloud.teleport.v2.templates.utils.connection.MySQLConnectionHelper; -======== import com.google.cloud.teleport.v2.templates.dbutils.connection.JdbcConnectionHelper; import com.google.cloud.teleport.v2.templates.dbutils.dao.source.JdbcDao; import com.google.cloud.teleport.v2.templates.exceptions.ConnectionException; ->>>>>>>> dev-repackaged:v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/dbutils/dao/JdbcDaoTest.java import com.zaxxer.hikari.HikariDataSource; import java.sql.Connection; import java.sql.Statement; @@ -64,11 +54,7 @@ public void doBeforeEachTest() throws java.sql.SQLException { @Test(expected = ConnectionException.class) public void testNullConnection() throws java.sql.SQLException, ConnectionException { -<<<<<<<< HEAD:v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/dao/JdbcDaoTest.java - JdbcDao sqlDao = new JdbcDao("url", "user", new MySQLConnectionHelper()); -======== JdbcDao sqlDao = new JdbcDao("url", "user", new JdbcConnectionHelper()); ->>>>>>>> dev-repackaged:v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/dbutils/dao/JdbcDaoTest.java sqlDao.write("sql"); } @@ -76,15 +62,9 @@ public void testNullConnection() throws java.sql.SQLException, ConnectionExcepti public void testSuccess() throws java.sql.SQLException, ConnectionException { Map connectionPoolMap = new HashMap<>(); connectionPoolMap.put("url/user", mockHikariDataSource); -<<<<<<<< HEAD:v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/dao/JdbcDaoTest.java - MySQLConnectionHelper mySQLConnectionHelper = new MySQLConnectionHelper(); - mySQLConnectionHelper.setConnectionPoolMap(connectionPoolMap); - JdbcDao sqlDao = new JdbcDao("url", "user", mySQLConnectionHelper); -======== JdbcConnectionHelper jdbcConnectionHelper = new JdbcConnectionHelper(); jdbcConnectionHelper.setConnectionPoolMap(connectionPoolMap); JdbcDao sqlDao = new JdbcDao("url", "user", jdbcConnectionHelper); ->>>>>>>> dev-repackaged:v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/dbutils/dao/JdbcDaoTest.java sqlDao.write("sql"); verify(mockStatement).executeUpdate(eq("sql")); } diff --git a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/dbutils/dao/SpannerDaoTest.java b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/dbutils/dao/SpannerDaoTest.java index ea8ab20c4b..882c379587 100644 --- a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/dbutils/dao/SpannerDaoTest.java +++ b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/dbutils/dao/SpannerDaoTest.java @@ -13,11 +13,7 @@ * License for the specific language governing permissions and limitations under * the License. */ -<<<<<<<< HEAD:v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/dao/SpannerDaoTest.java -package com.google.cloud.teleport.v2.templates.dao; -======== package com.google.cloud.teleport.v2.templates.dbutils.dao; ->>>>>>>> dev-repackaged:v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/dbutils/dao/SpannerDaoTest.java import static com.google.common.truth.Truth.assertThat; import static org.mockito.ArgumentMatchers.any; @@ -34,11 +30,7 @@ import com.google.cloud.spanner.ReadOnlyTransaction; import com.google.cloud.spanner.Struct; import com.google.cloud.teleport.v2.templates.constants.Constants; -<<<<<<<< HEAD:v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/dao/SpannerDaoTest.java -import com.google.cloud.teleport.v2.templates.dao.spanner.SpannerDao; -======== import com.google.cloud.teleport.v2.templates.dbutils.dao.spanner.SpannerDao; ->>>>>>>> dev-repackaged:v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/dbutils/dao/SpannerDaoTest.java import com.google.cloud.teleport.v2.templates.utils.ShadowTableRecord; import com.google.common.collect.ImmutableList; import org.apache.beam.sdk.io.gcp.spanner.SpannerAccessor; diff --git a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/dbutils/dml/MySQLDMLGeneratorTest.java b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/dbutils/dml/MySQLDMLGeneratorTest.java index 603488c9f3..b01d139a6d 100644 --- a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/dbutils/dml/MySQLDMLGeneratorTest.java +++ b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/dbutils/dml/MySQLDMLGeneratorTest.java @@ -13,11 +13,7 @@ * License for the specific language governing permissions and limitations under * the License. */ -<<<<<<<< HEAD:v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/dml/MySQLDMLGeneratorTest.java -package com.google.cloud.teleport.v2.templates.dml; -======== package com.google.cloud.teleport.v2.templates.dbutils.dml; ->>>>>>>> dev-repackaged:v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/dbutils/dml/MySQLDMLGeneratorTest.java import static org.junit.Assert.assertTrue; @@ -33,15 +29,9 @@ import com.google.cloud.teleport.v2.spanner.migrations.schema.SyntheticPKey; import com.google.cloud.teleport.v2.spanner.migrations.utils.SessionFileReader; import com.google.cloud.teleport.v2.templates.changestream.TrimmedShardedDataChangeRecord; -<<<<<<<< HEAD:v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/dml/MySQLDMLGeneratorTest.java -import com.google.cloud.teleport.v2.templates.models.DMLGeneratorRequest; -import com.google.cloud.teleport.v2.templates.models.DMLGeneratorResponse; -import com.google.cloud.teleport.v2.templates.utils.InputRecordProcessor; -======== import com.google.cloud.teleport.v2.templates.dbutils.processor.InputRecordProcessor; import com.google.cloud.teleport.v2.templates.models.DMLGeneratorRequest; import com.google.cloud.teleport.v2.templates.models.DMLGeneratorResponse; ->>>>>>>> dev-repackaged:v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/dbutils/dml/MySQLDMLGeneratorTest.java import com.google.gson.FieldNamingPolicy; import com.google.gson.GsonBuilder; import java.io.InputStream; diff --git a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/processor/SourceProcessorFactoryTest.java b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/processor/SourceProcessorFactoryTest.java deleted file mode 100644 index 93a318b0b7..0000000000 --- a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/processor/SourceProcessorFactoryTest.java +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Copyright (C) 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ -package com.google.cloud.teleport.v2.templates.processor; - -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.doNothing; - -import com.google.cloud.teleport.v2.spanner.migrations.shard.Shard; -import com.google.cloud.teleport.v2.templates.constants.Constants; -import com.google.cloud.teleport.v2.templates.dao.source.JdbcDao; -import com.google.cloud.teleport.v2.templates.dml.MySQLDMLGenerator; -import com.google.cloud.teleport.v2.templates.utils.connection.MySQLConnectionHelper; -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import org.junit.Assert; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.JUnit4; -import org.mockito.Mockito; - -@RunWith(JUnit4.class) -public class SourceProcessorFactoryTest { - @Test - public void testCreateSourceProcessor_validSource() throws Exception { - List shards = - Arrays.asList( - new Shard( - "shard1", - "localhost", - "3306", - "myuser", - "mypassword", - "mydatabase", - "mynamespace", - "projects/myproject/secrets/mysecret/versions/latest", - "")); - int maxConnections = 10; - MySQLConnectionHelper mockConnectionHelper = Mockito.mock(MySQLConnectionHelper.class); - doNothing().when(mockConnectionHelper).init(any()); - SourceProcessorFactory.setConnectionHelperMap( - Map.of(Constants.SOURCE_MYSQL, mockConnectionHelper)); - SourceProcessor processor = - SourceProcessorFactory.createSourceProcessor( - Constants.SOURCE_MYSQL, shards, maxConnections); - - Assert.assertNotNull(processor); - Assert.assertTrue(processor.getDmlGenerator() instanceof MySQLDMLGenerator); - Assert.assertEquals(1, processor.getSourceDaoMap().size()); - Assert.assertTrue(processor.getSourceDaoMap().get("shard1") instanceof JdbcDao); - } - - @Test(expected = InvalidSourceException.class) - public void testCreateSourceProcessor_invalidSource() throws Exception { - List shards = - Arrays.asList( - new Shard( - "shard1", - "localhost", - "3306", - "myuser", - "mypassword", - "mydatabase", - "mynamespace", - "projects/myproject/secrets/mysecret/versions/latest", - "")); - int maxConnections = 10; - - SourceProcessorFactory.createSourceProcessor("invalid_source", shards, maxConnections); - } -} diff --git a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/utils/connection/MySQLConnectionHelperTest.java b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/utils/connection/MySQLConnectionHelperTest.java deleted file mode 100644 index 16a3aa9895..0000000000 --- a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/utils/connection/MySQLConnectionHelperTest.java +++ /dev/null @@ -1,4 +0,0 @@ -package com.google.cloud.teleport.v2.templates.utils.connection; - -public class MySQLConnectionHelperTest { -}