diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerShardedMigrationWithMigrationShardIdColumnIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerShardedMigrationWithMigrationMySqlShardIdColumnIT.java similarity index 98% rename from v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerShardedMigrationWithMigrationShardIdColumnIT.java rename to v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerShardedMigrationWithMigrationMySqlShardIdColumnIT.java index 383d1469d8..09ef247041 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerShardedMigrationWithMigrationShardIdColumnIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerShardedMigrationWithMigrationMySqlShardIdColumnIT.java @@ -52,12 +52,12 @@ @Category({TemplateIntegrationTest.class, SkipDirectRunnerTest.class}) @TemplateIntegrationTest(DataStreamToSpanner.class) @RunWith(JUnit4.class) -public class DataStreamToSpannerShardedMigrationWithMigrationShardIdColumnIT +public class DataStreamToSpannerShardedMigrationWithMigrationMySqlShardIdColumnIT extends DataStreamToSpannerITBase { private static final Logger LOG = LoggerFactory.getLogger( - DataStreamToSpannerShardedMigrationWithMigrationShardIdColumnIT.class); + DataStreamToSpannerShardedMigrationWithMigrationMySqlShardIdColumnIT.class); private static final String TABLE = "Users"; private static final String MOVIE_TABLE = "Movie"; @@ -75,7 +75,7 @@ public class DataStreamToSpannerShardedMigrationWithMigrationShardIdColumnIT private static final String SPANNER_DDL_RESOURCE = "DataStreamToSpannerShardedMigrationWithMigrationShardIdColumnIT/spanner-schema.sql"; - private static HashSet + private static HashSet testInstances = new HashSet<>(); private static PipelineLauncher.LaunchInfo jobInfo1; private static PipelineLauncher.LaunchInfo jobInfo2; @@ -92,7 +92,7 @@ public class DataStreamToSpannerShardedMigrationWithMigrationShardIdColumnIT public void setUp() throws IOException, InterruptedException { // Prevent cleaning up of dataflow job after a test method is executed. skipBaseCleanup = true; - synchronized (DataStreamToSpannerShardedMigrationWithMigrationShardIdColumnIT.class) { + synchronized (DataStreamToSpannerShardedMigrationWithMigrationMySqlShardIdColumnIT.class) { testInstances.add(this); if (spannerResourceManager == null) { spannerResourceManager = setUpSpannerResourceManager(); @@ -150,7 +150,7 @@ public void setUp() throws IOException, InterruptedException { */ @AfterClass public static void cleanUp() throws IOException { - for (DataStreamToSpannerShardedMigrationWithMigrationShardIdColumnIT instance : testInstances) { + for (DataStreamToSpannerShardedMigrationWithMigrationMySqlShardIdColumnIT instance : testInstances) { instance.tearDownBase(); } ResourceManagerUtils.cleanResources(spannerResourceManager, pubsubResourceManager); diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerShardedMigrationWithoutMigrationShardIdColumnIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerShardedMigrationWithoutMigrationMySqlShardIdColumnIT.java similarity index 97% rename from v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerShardedMigrationWithoutMigrationShardIdColumnIT.java rename to v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerShardedMigrationWithoutMigrationMySqlShardIdColumnIT.java index b875c0e513..3ae8fab34f 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerShardedMigrationWithoutMigrationShardIdColumnIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DataStreamToSpannerShardedMigrationWithoutMigrationMySqlShardIdColumnIT.java @@ -51,17 +51,17 @@ @Category({TemplateIntegrationTest.class, SkipDirectRunnerTest.class}) @TemplateIntegrationTest(DataStreamToSpanner.class) @RunWith(JUnit4.class) -public class DataStreamToSpannerShardedMigrationWithoutMigrationShardIdColumnIT +public class DataStreamToSpannerShardedMigrationWithoutMigrationMySqlShardIdColumnIT extends DataStreamToSpannerITBase { private static final Logger LOG = LoggerFactory.getLogger( - DataStreamToSpannerShardedMigrationWithoutMigrationShardIdColumnIT.class); + DataStreamToSpannerShardedMigrationWithoutMigrationMySqlShardIdColumnIT.class); private static final String TABLE = "Users"; private static final String SPANNER_DDL_RESOURCE = "DataStreamToSpannerShardedMigrationWithoutMigrationShardIdColumnIT/spanner-schema.sql"; - private static HashSet + private static HashSet testInstances = new HashSet<>(); private static PipelineLauncher.LaunchInfo jobInfo1; private static PipelineLauncher.LaunchInfo jobInfo2; @@ -78,7 +78,7 @@ public class DataStreamToSpannerShardedMigrationWithoutMigrationShardIdColumnIT public void setUp() throws IOException { // Prevent cleaning up of dataflow job after a test method is executed. skipBaseCleanup = true; - synchronized (DataStreamToSpannerShardedMigrationWithoutMigrationShardIdColumnIT.class) { + synchronized (DataStreamToSpannerShardedMigrationWithoutMigrationMySqlShardIdColumnIT.class) { testInstances.add(this); if (spannerResourceManager == null) { spannerResourceManager = setUpSpannerResourceManager(); @@ -131,7 +131,7 @@ public void setUp() throws IOException { */ @AfterClass public static void cleanUp() throws IOException { - for (DataStreamToSpannerShardedMigrationWithoutMigrationShardIdColumnIT instance : + for (DataStreamToSpannerShardedMigrationWithoutMigrationMySqlShardIdColumnIT instance : testInstances) { instance.tearDownBase(); } diff --git a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DatastreamToSpannerSingleDFShardedMigrationIT.java b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DatastreamToSpannerSingleDFShardedMigrationIT.java index 28483263d7..22af60d383 100644 --- a/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DatastreamToSpannerSingleDFShardedMigrationIT.java +++ b/v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/DatastreamToSpannerSingleDFShardedMigrationIT.java @@ -87,7 +87,7 @@ public class DatastreamToSpannerSingleDFShardedMigrationIT extends DataStreamToS public void setUp() throws IOException, InterruptedException { // Prevent cleaning up of dataflow job after a test method is executed. skipBaseCleanup = true; - synchronized (DataStreamToSpannerShardedMigrationWithMigrationShardIdColumnIT.class) { + synchronized (DataStreamToSpannerShardedMigrationWithMigrationMySqlShardIdColumnIT.class) { testInstances.add(this); if (spannerResourceManager == null) { spannerResourceManager = setUpSpannerResourceManager(); diff --git a/v2/gcs-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/common/ProcessingContext.java b/v2/gcs-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/common/ProcessingContext.java index 5e5a3b2f54..4fd256b62f 100644 --- a/v2/gcs-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/common/ProcessingContext.java +++ b/v2/gcs-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/common/ProcessingContext.java @@ -16,7 +16,8 @@ package com.google.cloud.teleport.v2.templates.common; import com.google.cloud.teleport.v2.spanner.migrations.schema.Schema; -import com.google.cloud.teleport.v2.spanner.migrations.shard.Shard; +import com.google.cloud.teleport.v2.spanner.migrations.shard.IShard; + import java.io.Serializable; import java.util.Objects; import org.joda.time.Duration; @@ -24,7 +25,7 @@ /** Each worker task context. */ public class ProcessingContext implements Serializable { - private Shard shard; + private IShard iShard; private Schema schema; private String sourceDbTimezoneOffset; @@ -34,14 +35,14 @@ public class ProcessingContext implements Serializable { private String runId; public ProcessingContext( - Shard shard, + IShard iShard, Schema schema, String sourceDbTimezoneOffset, String startTimestamp, Duration windowDuration, String gcsPath, String runId) { - this.shard = shard; + this.iShard = iShard; this.schema = schema; this.sourceDbTimezoneOffset = sourceDbTimezoneOffset; this.startTimestamp = startTimestamp; @@ -50,8 +51,8 @@ public ProcessingContext( this.runId = runId; } - public Shard getShard() { - return shard; + public IShard getShard() { + return iShard; } public Schema getSchema() { @@ -85,8 +86,8 @@ public String getRunId() { @Override public String toString() { - return "{ Shard details :" - + shard.toString() + return "{ IShard details :" + + iShard.toString() + " sourceDbTimezoneOffset: " + sourceDbTimezoneOffset + " startTimestamp: " diff --git a/v2/gcs-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/constants/Constants.java b/v2/gcs-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/constants/Constants.java index 165c9ab489..d8b3458a4a 100644 --- a/v2/gcs-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/constants/Constants.java +++ b/v2/gcs-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/constants/Constants.java @@ -33,12 +33,12 @@ public class Constants { /** Run mode - resumeAll. */ public static final String RUN_MODE_RESUME_ALL = "resumeAll"; - /** Shard progress status - success. */ + /** IShard progress status - success. */ public static final String SHARD_PROGRESS_STATUS_SUCCESS = "SUCCESS"; - /** Shard progress status - error. */ + /** IShard progress status - error. */ public static final String SHARD_PROGRESS_STATUS_ERROR = "ERROR"; - /** Shard progress status - reprocess. */ + /** IShard progress status - reprocess. */ public static final String SHARD_PROGRESS_STATUS_REPROCESS = "REPROCESS"; } diff --git a/v2/gcs-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/processing/handler/GCSToSourceStreamingHandler.java b/v2/gcs-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/processing/handler/GCSToSourceStreamingHandler.java index f4f5a86714..d5ffc427d7 100644 --- a/v2/gcs-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/processing/handler/GCSToSourceStreamingHandler.java +++ b/v2/gcs-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/processing/handler/GCSToSourceStreamingHandler.java @@ -57,7 +57,7 @@ public static String process( List records = inputFileReader.getRecords(); Instant readEndTime = Instant.now(); LOG.info( - "Shard " + "IShard " + shardId + ": read " + records.size() @@ -82,7 +82,7 @@ public static String process( MySqlDao dao = new DaoFactory( connectString, - taskContext.getShard().getUserName(), + taskContext.getShard().getUser(), taskContext.getShard().getPassword()) .getMySqlDao(shardId); @@ -101,7 +101,7 @@ public static String process( markShardSuccess(taskContext, spannerDao, fileProcessedStartInterval); dao.cleanup(); LOG.info( - "Shard " + shardId + ": Successfully processed batch of " + records.size() + " records."); + "IShard " + shardId + ": Successfully processed batch of " + records.size() + " records."); } catch (Exception e) { Metrics.counter(GCSToSourceStreamingHandler.class, "shard_failed_" + shardId).inc(); markShardFailure(taskContext, spannerDao, fileProcessedStartInterval); diff --git a/v2/gcs-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/processing/handler/InputRecordProcessor.java b/v2/gcs-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/processing/handler/InputRecordProcessor.java index dca51f3b54..f2942c41bb 100644 --- a/v2/gcs-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/processing/handler/InputRecordProcessor.java +++ b/v2/gcs-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/processing/handler/InputRecordProcessor.java @@ -138,7 +138,7 @@ public static void processRecords( dao.batchWrite(dmlBatch); Instant daoEndTime = Instant.now(); LOG.info( - "Shard " + "MySqlShard " + shardId + ": Write to mysql for " + recordList.size() diff --git a/v2/gcs-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/transforms/GcsToSourceStreamer.java b/v2/gcs-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/transforms/GcsToSourceStreamer.java index 92ecb8f0d1..9ea0b17f07 100644 --- a/v2/gcs-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/transforms/GcsToSourceStreamer.java +++ b/v2/gcs-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/transforms/GcsToSourceStreamer.java @@ -182,7 +182,7 @@ public void onExpiry( @StateId("stopProcessing") ValueState stopProcessing) { String shardId = keyString.read(); LOG.info( - "Shard " + shardId + ": started timer processing for expiry time: " + context.timestamp()); + "IShard " + shardId + ": started timer processing for expiry time: " + context.timestamp()); ProcessingContext taskContext = processingContext.read(); Boolean failedShard = stopProcessing.read(); if (failedShard != null && failedShard) { diff --git a/v2/gcs-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/ProcessingContextGenerator.java b/v2/gcs-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/ProcessingContextGenerator.java index 546fb3d2e0..2c1492f1f0 100644 --- a/v2/gcs-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/ProcessingContextGenerator.java +++ b/v2/gcs-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/utils/ProcessingContextGenerator.java @@ -17,7 +17,7 @@ import com.google.cloud.teleport.v2.spanner.migrations.metadata.SpannerToGcsJobMetadata; import com.google.cloud.teleport.v2.spanner.migrations.schema.Schema; -import com.google.cloud.teleport.v2.spanner.migrations.shard.Shard; +import com.google.cloud.teleport.v2.spanner.migrations.shard.IShard; import com.google.cloud.teleport.v2.spanner.migrations.utils.SecretManagerAccessorImpl; import com.google.cloud.teleport.v2.spanner.migrations.utils.SessionFileReader; import com.google.cloud.teleport.v2.spanner.migrations.utils.ShardFileReader; @@ -66,7 +66,7 @@ public static Map getProcessingContextForGCS( Schema schema = SessionFileReader.read(sessionFilePath); ShardFileReader shardFileReader = new ShardFileReader(new SecretManagerAccessorImpl()); - List shards = shardFileReader.getOrderedShardDetails(sourceShardsFilePath); + List iShards = shardFileReader.getOrderedShardDetails(sourceShardsFilePath); ShardProgressTracker shardProgressTracker = new ShardProgressTracker( @@ -92,7 +92,7 @@ public static Map getProcessingContextForGCS( metadataDatabase, tableSuffix, runId, - shards, + iShards, schema, isMetadataDbPostgres); } else { @@ -105,7 +105,7 @@ public static Map getProcessingContextForGCS( metadataDatabase, tableSuffix, runId, - shards, + iShards, schema, shardProgressTracker, runMode, @@ -126,7 +126,7 @@ private static Map getProcessingContextForRegularMode String metadataDatabase, String tableSuffix, String runId, - List shards, + List iShards, Schema schema, boolean isMetadataDbPostgres) { @@ -157,18 +157,18 @@ private static Map getProcessingContextForRegularMode Duration duration = DurationUtils.parseDuration(windowDuration); - for (Shard shard : shards) { - LOG.info(" The sorted shard is: {}", shard); + for (IShard iShard : iShards) { + LOG.info(" The sorted IShards is: {}", iShard); ProcessingContext taskContext = new ProcessingContext( - shard, + iShard, schema, sourceDbTimezoneOffset, startTimestamp, duration, gcsInputDirectoryPath, runId); - response.put(shard.getLogicalShardId(), taskContext); + response.put(iShard.getLogicalShardId(), taskContext); } return response; @@ -182,7 +182,7 @@ private static Map getProcessingContextForReprocessOr String metadataDatabase, String tableSuffix, String runId, - List shards, + List iShards, Schema schema, ShardProgressTracker shardProgressTracker, String runMode, @@ -228,9 +228,9 @@ private static Map getProcessingContextForReprocessOr Duration duration = DurationUtils.parseDuration(windowDuration); - for (Shard shard : shards) { - LOG.info(" The sorted shard is: {}", shard); - ShardProgress shardProgress = shardProgressList.get(shard.getLogicalShardId()); + for (IShard iShard : iShards) { + LOG.info(" The sorted shards is: {}", iShard); + ShardProgress shardProgress = shardProgressList.get(iShard.getLogicalShardId()); String shardStartTime = null; if (shardProgress != null) { @@ -239,27 +239,27 @@ private static Map getProcessingContextForReprocessOr if ((runMode.equals(Constants.RUN_MODE_RESUME_SUCCESS) || runMode.equals(Constants.RUN_MODE_RESUME_ALL)) && shardProgress.getStatus().equals(Constants.SHARD_PROGRESS_STATUS_SUCCESS)) { - // Advance the start time by window duration for successful shards + // Advance the start time by window duration for successful Shards shardStartTimeInst = shardStartTimeInst.plus(duration); } shardStartTime = shardStartTimeInst.toString(); - LOG.info(" The startTime for shard {} is : {}", shard, shardStartTime); + LOG.info(" The startTime for shards {} is : {}", iShard, shardStartTime); } else { LOG.info( - " Skipping shard: {} as it does not qualify for given runMode {}.", shard, runMode); + " Skipping shards: {} as it does not qualify for given runMode {}.", iShard, runMode); continue; } ProcessingContext taskContext = new ProcessingContext( - shard, + iShard, schema, sourceDbTimezoneOffset, shardStartTime, duration, gcsInputDirectoryPath, runId); - response.put(shard.getLogicalShardId(), taskContext); + response.put(iShard.getLogicalShardId(), taskContext); } return response; } diff --git a/v2/gcs-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/GCSToSourceDbInterleaveIT.java b/v2/gcs-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/GCSToSourceDbInterleaveIT.java index 2608225042..ba4f7181bf 100644 --- a/v2/gcs-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/GCSToSourceDbInterleaveIT.java +++ b/v2/gcs-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/GCSToSourceDbInterleaveIT.java @@ -24,7 +24,7 @@ import com.google.cloud.spanner.Mutation; import com.google.cloud.teleport.metadata.SkipDirectRunnerTest; import com.google.cloud.teleport.metadata.TemplateIntegrationTest; -import com.google.cloud.teleport.v2.spanner.migrations.shard.Shard; +import com.google.cloud.teleport.v2.spanner.migrations.shard.MySqlShard; import com.google.common.io.Resources; import com.google.gson.Gson; import com.google.gson.JsonArray; @@ -410,19 +410,19 @@ private void launchReaderDataflowJob() throws IOException { private void createAndUploadShardConfigToGcs( GcsResourceManager gcsResourceManager, MySQLResourceManager jdbcResourceManager) throws IOException { - Shard shard = new Shard(); - shard.setLogicalShardId("Shard1"); - shard.setUser(jdbcResourceManager.getUsername()); - shard.setHost(jdbcResourceManager.getHost()); - shard.setPassword(jdbcResourceManager.getPassword()); - shard.setPort(String.valueOf(jdbcResourceManager.getPort())); - shard.setDbName(jdbcResourceManager.getDatabaseName()); - JsonObject jsObj = (JsonObject) new Gson().toJsonTree(shard).getAsJsonObject(); + MySqlShard mySqlShard = new MySqlShard(); + mySqlShard.setLogicalShardId("Shard1"); + mySqlShard.setUser(jdbcResourceManager.getUsername()); + mySqlShard.setHost(jdbcResourceManager.getHost()); + mySqlShard.setPassword(jdbcResourceManager.getPassword()); + mySqlShard.setPort(String.valueOf(jdbcResourceManager.getPort())); + mySqlShard.setDbName(jdbcResourceManager.getDatabaseName()); + JsonObject jsObj = (JsonObject) new Gson().toJsonTree(mySqlShard).getAsJsonObject(); jsObj.remove("secretManagerUri"); // remove field secretManagerUri JsonArray ja = new JsonArray(); ja.add(jsObj); String shardFileContents = ja.toString(); - LOG.info("Shard file contents: {}", shardFileContents); - gcsResourceManager.createArtifact("input/shard.json", shardFileContents); + LOG.info("MySqlShard file contents: {}", shardFileContents); + gcsResourceManager.createArtifact("input/mySqlShard.json", shardFileContents); } } diff --git a/v2/gcs-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/GCSToSourceDbInterleaveMultiShardIT.java b/v2/gcs-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/GCSToSourceDbInterleaveMultiMySqlShardIT.java similarity index 91% rename from v2/gcs-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/GCSToSourceDbInterleaveMultiShardIT.java rename to v2/gcs-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/GCSToSourceDbInterleaveMultiMySqlShardIT.java index 884d825fce..5daf079341 100644 --- a/v2/gcs-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/GCSToSourceDbInterleaveMultiShardIT.java +++ b/v2/gcs-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/GCSToSourceDbInterleaveMultiMySqlShardIT.java @@ -24,7 +24,7 @@ import com.google.cloud.spanner.Mutation; import com.google.cloud.teleport.metadata.SkipDirectRunnerTest; import com.google.cloud.teleport.metadata.TemplateIntegrationTest; -import com.google.cloud.teleport.v2.spanner.migrations.shard.Shard; +import com.google.cloud.teleport.v2.spanner.migrations.shard.MySqlShard; import com.google.common.io.Resources; import com.google.gson.Gson; import com.google.gson.JsonArray; @@ -62,10 +62,10 @@ @Category({TemplateIntegrationTest.class, SkipDirectRunnerTest.class}) @TemplateIntegrationTest(GCSToSourceDb.class) @RunWith(JUnit4.class) -public class GCSToSourceDbInterleaveMultiShardIT extends TemplateTestBase { +public class GCSToSourceDbInterleaveMultiMySqlShardIT extends TemplateTestBase { private static final Logger LOG = - LoggerFactory.getLogger(GCSToSourceDbInterleaveMultiShardIT.class); + LoggerFactory.getLogger(GCSToSourceDbInterleaveMultiMySqlShardIT.class); private static final String SPANNER_DDL_RESOURCE = "GCSToSourceDbInterleaveMultiShardIT/spanner-schema.sql"; @@ -74,7 +74,7 @@ public class GCSToSourceDbInterleaveMultiShardIT extends TemplateTestBase { private static final String MYSQL_DDL_RESOURCE = "GCSToSourceDbInterleaveMultiShardIT/mysql-schema.sql"; - private static HashSet testInstances = new HashSet<>(); + private static HashSet testInstances = new HashSet<>(); private static PipelineLauncher.LaunchInfo writerJobInfo; private static PipelineLauncher.LaunchInfo readerJobInfo; public static SpannerResourceManager spannerResourceManager; @@ -92,7 +92,7 @@ public class GCSToSourceDbInterleaveMultiShardIT extends TemplateTestBase { @Before public void setUp() throws IOException { skipBaseCleanup = true; - synchronized (GCSToSourceDbInterleaveMultiShardIT.class) { + synchronized (GCSToSourceDbInterleaveMultiMySqlShardIT.class) { testInstances.add(this); if (writerJobInfo == null) { spannerResourceManager = createSpannerDatabase(SPANNER_DDL_RESOURCE); @@ -124,7 +124,7 @@ public void setUp() throws IOException { */ @AfterClass public static void cleanUp() throws IOException { - for (GCSToSourceDbInterleaveMultiShardIT instance : testInstances) { + for (GCSToSourceDbInterleaveMultiMySqlShardIT instance : testInstances) { instance.tearDownBase(); } ResourceManagerUtils.cleanResources( @@ -445,30 +445,30 @@ private void launchReaderDataflowJob() throws IOException { } private void createAndUploadShardConfigToGcs() throws IOException { - Shard shard = new Shard(); - shard.setLogicalShardId("shardA"); - shard.setUser(jdbcResourceManagerShardA.getUsername()); - shard.setHost(jdbcResourceManagerShardA.getHost()); - shard.setPassword(jdbcResourceManagerShardA.getPassword()); - shard.setPort(String.valueOf(jdbcResourceManagerShardA.getPort())); - shard.setDbName(jdbcResourceManagerShardA.getDatabaseName()); - JsonObject jsObj = (JsonObject) new Gson().toJsonTree(shard).getAsJsonObject(); + MySqlShard mySqlShard = new MySqlShard(); + mySqlShard.setLogicalShardId("shardA"); + mySqlShard.setUser(jdbcResourceManagerShardA.getUsername()); + mySqlShard.setHost(jdbcResourceManagerShardA.getHost()); + mySqlShard.setPassword(jdbcResourceManagerShardA.getPassword()); + mySqlShard.setPort(String.valueOf(jdbcResourceManagerShardA.getPort())); + mySqlShard.setDbName(jdbcResourceManagerShardA.getDatabaseName()); + JsonObject jsObj = (JsonObject) new Gson().toJsonTree(mySqlShard).getAsJsonObject(); jsObj.remove("secretManagerUri"); // remove field secretManagerUri - Shard shardB = new Shard(); - shardB.setLogicalShardId("shardB"); - shardB.setUser(jdbcResourceManagerShardB.getUsername()); - shardB.setHost(jdbcResourceManagerShardB.getHost()); - shardB.setPassword(jdbcResourceManagerShardB.getPassword()); - shardB.setPort(String.valueOf(jdbcResourceManagerShardB.getPort())); - shardB.setDbName(jdbcResourceManagerShardB.getDatabaseName()); - JsonObject jsObjB = (JsonObject) new Gson().toJsonTree(shardB).getAsJsonObject(); + MySqlShard mySqlShardB = new MySqlShard(); + mySqlShardB.setLogicalShardId("mySqlShardB"); + mySqlShardB.setUser(jdbcResourceManagerShardB.getUsername()); + mySqlShardB.setHost(jdbcResourceManagerShardB.getHost()); + mySqlShardB.setPassword(jdbcResourceManagerShardB.getPassword()); + mySqlShardB.setPort(String.valueOf(jdbcResourceManagerShardB.getPort())); + mySqlShardB.setDbName(jdbcResourceManagerShardB.getDatabaseName()); + JsonObject jsObjB = (JsonObject) new Gson().toJsonTree(mySqlShardB).getAsJsonObject(); jsObjB.remove("secretManagerUri"); // remove field secretManagerUri JsonArray ja = new JsonArray(); ja.add(jsObj); ja.add(jsObjB); String shardFileContents = ja.toString(); - LOG.info("Shard file contents: {}", shardFileContents); - gcsResourceManager.createArtifact("input/shard.json", shardFileContents); + LOG.info("MySqlShard file contents: {}", shardFileContents); + gcsResourceManager.createArtifact("input/mySqlShard.json", shardFileContents); } } diff --git a/v2/gcs-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/GCSToSourceDbWithReaderIT.java b/v2/gcs-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/GCSToSourceDbWithReaderIT.java index 7ed57d09e5..bd40ed51cf 100644 --- a/v2/gcs-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/GCSToSourceDbWithReaderIT.java +++ b/v2/gcs-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/GCSToSourceDbWithReaderIT.java @@ -21,7 +21,7 @@ import com.google.cloud.spanner.Mutation; import com.google.cloud.teleport.metadata.SkipDirectRunnerTest; import com.google.cloud.teleport.metadata.TemplateIntegrationTest; -import com.google.cloud.teleport.v2.spanner.migrations.shard.Shard; +import com.google.cloud.teleport.v2.spanner.migrations.shard.MySqlShard; import com.google.common.io.Resources; import com.google.gson.Gson; import com.google.gson.JsonArray; @@ -240,19 +240,19 @@ private void launchReaderDataflowJob() throws IOException { private void createAndUploadShardConfigToGcs( GcsResourceManager gcsResourceManager, MySQLResourceManager jdbcResourceManager) throws IOException { - Shard shard = new Shard(); - shard.setLogicalShardId("Shard1"); - shard.setUser(jdbcResourceManager.getUsername()); - shard.setHost(jdbcResourceManager.getHost()); - shard.setPassword(jdbcResourceManager.getPassword()); - shard.setPort(String.valueOf(jdbcResourceManager.getPort())); - shard.setDbName(jdbcResourceManager.getDatabaseName()); - JsonObject jsObj = new Gson().toJsonTree(shard).getAsJsonObject(); + MySqlShard mySqlShard = new MySqlShard(); + mySqlShard.setLogicalShardId("Shard1"); + mySqlShard.setUser(jdbcResourceManager.getUsername()); + mySqlShard.setHost(jdbcResourceManager.getHost()); + mySqlShard.setPassword(jdbcResourceManager.getPassword()); + mySqlShard.setPort(String.valueOf(jdbcResourceManager.getPort())); + mySqlShard.setDbName(jdbcResourceManager.getDatabaseName()); + JsonObject jsObj = new Gson().toJsonTree(mySqlShard).getAsJsonObject(); jsObj.remove("secretManagerUri"); // remove field secretManagerUri JsonArray ja = new JsonArray(); ja.add(jsObj); String shardFileContents = ja.toString(); - LOG.info("Shard file contents: {}", shardFileContents); - gcsResourceManager.createArtifact("input/shard.json", shardFileContents); + LOG.info("MySqlShard file contents: {}", shardFileContents); + gcsResourceManager.createArtifact("input/mySqlShard.json", shardFileContents); } } diff --git a/v2/gcs-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/GCSToSourceDbWithoutReaderIT.java b/v2/gcs-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/GCSToSourceDbWithoutReaderIT.java index 28dfafc5d9..0ee28c99c9 100644 --- a/v2/gcs-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/GCSToSourceDbWithoutReaderIT.java +++ b/v2/gcs-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/GCSToSourceDbWithoutReaderIT.java @@ -21,7 +21,7 @@ import com.google.cloud.teleport.metadata.SkipDirectRunnerTest; import com.google.cloud.teleport.metadata.TemplateIntegrationTest; -import com.google.cloud.teleport.v2.spanner.migrations.shard.Shard; +import com.google.cloud.teleport.v2.spanner.migrations.shard.MySqlShard; import com.google.cloud.teleport.v2.spanner.migrations.transformation.CustomTransformation; import com.google.common.io.Resources; import com.google.gson.Gson; @@ -282,20 +282,20 @@ private void launchWriterDataflowJob(CustomTransformation customTransformation) private void createAndUploadShardConfigToGcs( GcsResourceManager gcsResourceManager, MySQLResourceManager jdbcResourceManager) { - Shard shard = new Shard(); - shard.setLogicalShardId("Shard1"); - shard.setUser(jdbcResourceManager.getUsername()); - shard.setHost(jdbcResourceManager.getHost()); - shard.setPassword(jdbcResourceManager.getPassword()); - shard.setPort(String.valueOf(jdbcResourceManager.getPort())); - shard.setDbName(jdbcResourceManager.getDatabaseName()); - JsonObject jsObj = new Gson().toJsonTree(shard).getAsJsonObject(); + MySqlShard mySqlShard = new MySqlShard(); + mySqlShard.setLogicalShardId("Shard1"); + mySqlShard.setUser(jdbcResourceManager.getUsername()); + mySqlShard.setHost(jdbcResourceManager.getHost()); + mySqlShard.setPassword(jdbcResourceManager.getPassword()); + mySqlShard.setPort(String.valueOf(jdbcResourceManager.getPort())); + mySqlShard.setDbName(jdbcResourceManager.getDatabaseName()); + JsonObject jsObj = new Gson().toJsonTree(mySqlShard).getAsJsonObject(); jsObj.remove("secretManagerUri"); // remove field secretManagerUri JsonArray ja = new JsonArray(); ja.add(jsObj); String shardFileContents = ja.toString(); - LOG.info("Shard file contents: {}", shardFileContents); - gcsResourceManager.createArtifact("input/shard.json", shardFileContents); + LOG.info("MySqlShard file contents: {}", shardFileContents); + gcsResourceManager.createArtifact("input/mySqlShard.json", shardFileContents); } public void createAndUploadJarToGcs(GcsResourceManager gcsResourceManager) diff --git a/v2/gcs-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/SpannerToJdbcLTBase.java b/v2/gcs-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/SpannerToJdbcLTBase.java index bafca12c76..115c5a09f0 100644 --- a/v2/gcs-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/SpannerToJdbcLTBase.java +++ b/v2/gcs-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/SpannerToJdbcLTBase.java @@ -15,7 +15,7 @@ */ package com.google.cloud.teleport.v2.templates; -import com.google.cloud.teleport.v2.spanner.migrations.shard.Shard; +import com.google.cloud.teleport.v2.spanner.migrations.shard.MySqlShard; import com.google.common.base.MoreObjects; import com.google.common.io.Resources; import com.google.gson.Gson; @@ -130,14 +130,14 @@ public void createAndUploadShardConfigToGcs( for (int i = 0; i < 1; ++i) { if (jdbcResourceManagers.get(i) instanceof MySQLResourceManager) { MySQLResourceManager resourceManager = (MySQLResourceManager) jdbcResourceManagers.get(i); - Shard shard = new Shard(); - shard.setLogicalShardId("Shard" + (i + 1)); - shard.setUser(jdbcResourceManagers.get(i).getUsername()); - shard.setHost(resourceManager.getHost()); - shard.setPassword(jdbcResourceManagers.get(i).getPassword()); - shard.setPort(String.valueOf(resourceManager.getPort())); - shard.setDbName(jdbcResourceManagers.get(i).getDatabaseName()); - JsonObject jsObj = (JsonObject) new Gson().toJsonTree(shard).getAsJsonObject(); + MySqlShard mySqlShard = new MySqlShard(); + mySqlShard.setLogicalShardId("MySqlShard" + (i + 1)); + mySqlShard.setUser(jdbcResourceManagers.get(i).getUsername()); + mySqlShard.setHost(resourceManager.getHost()); + mySqlShard.setPassword(jdbcResourceManagers.get(i).getPassword()); + mySqlShard.setPort(String.valueOf(resourceManager.getPort())); + mySqlShard.setDbName(jdbcResourceManagers.get(i).getDatabaseName()); + JsonObject jsObj = (JsonObject) new Gson().toJsonTree(mySqlShard).getAsJsonObject(); jsObj.remove("secretManagerUri"); // remove field secretManagerUri ja.add(jsObj); } else { @@ -146,7 +146,7 @@ public void createAndUploadShardConfigToGcs( } } String shardFileContents = ja.toString(); - LOG.info("Shard file contents: {}", shardFileContents); + LOG.info("MySqlShard file contents: {}", shardFileContents); gcsResourceManager.createArtifact("input/shard.json", shardFileContents); } diff --git a/v2/gcs-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/TimezoneIT.java b/v2/gcs-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/TimezoneIT.java index 3079d20a7b..5319b5bdcb 100644 --- a/v2/gcs-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/TimezoneIT.java +++ b/v2/gcs-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/TimezoneIT.java @@ -23,7 +23,7 @@ import com.google.cloud.spanner.Mutation; import com.google.cloud.teleport.metadata.SkipDirectRunnerTest; import com.google.cloud.teleport.metadata.TemplateIntegrationTest; -import com.google.cloud.teleport.v2.spanner.migrations.shard.Shard; +import com.google.cloud.teleport.v2.spanner.migrations.shard.MySqlShard; import com.google.common.io.Resources; import com.google.gson.Gson; import com.google.gson.JsonArray; @@ -272,19 +272,19 @@ private void launchReaderDataflowJob() throws IOException { private void createAndUploadShardConfigToGcs( GcsResourceManager gcsResourceManager, MySQLResourceManager jdbcResourceManager) throws IOException { - Shard shard = new Shard(); - shard.setLogicalShardId("Shard1"); - shard.setUser(jdbcResourceManager.getUsername()); - shard.setHost(jdbcResourceManager.getHost()); - shard.setPassword(jdbcResourceManager.getPassword()); - shard.setPort(String.valueOf(jdbcResourceManager.getPort())); - shard.setDbName(jdbcResourceManager.getDatabaseName()); - JsonObject jsObj = (JsonObject) new Gson().toJsonTree(shard).getAsJsonObject(); + MySqlShard mySqlShard = new MySqlShard(); + mySqlShard.setLogicalShardId("Shard1"); + mySqlShard.setUser(jdbcResourceManager.getUsername()); + mySqlShard.setHost(jdbcResourceManager.getHost()); + mySqlShard.setPassword(jdbcResourceManager.getPassword()); + mySqlShard.setPort(String.valueOf(jdbcResourceManager.getPort())); + mySqlShard.setDbName(jdbcResourceManager.getDatabaseName()); + JsonObject jsObj = (JsonObject) new Gson().toJsonTree(mySqlShard).getAsJsonObject(); jsObj.remove("secretManagerUri"); // remove field secretManagerUri JsonArray ja = new JsonArray(); ja.add(jsObj); String shardFileContents = ja.toString(); - LOG.info("Shard file contents: {}", shardFileContents); - gcsResourceManager.createArtifact("input/shard.json", shardFileContents); + LOG.info("MySqlShard file contents: {}", shardFileContents); + gcsResourceManager.createArtifact("input/mySqlShard.json", shardFileContents); } } diff --git a/v2/gcs-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/processing/handler/GCSToSourceStreamingHandlerTest.java b/v2/gcs-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/processing/handler/GCSToSourceStreamingHandlerTest.java index 1a8ca71e20..8976bd486b 100644 --- a/v2/gcs-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/processing/handler/GCSToSourceStreamingHandlerTest.java +++ b/v2/gcs-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/processing/handler/GCSToSourceStreamingHandlerTest.java @@ -26,7 +26,7 @@ import com.google.cloud.Timestamp; import com.google.cloud.storage.BlobInfo; import com.google.cloud.storage.Storage; -import com.google.cloud.teleport.v2.spanner.migrations.shard.Shard; +import com.google.cloud.teleport.v2.spanner.migrations.shard.MySqlShard; import com.google.cloud.teleport.v2.templates.common.ProcessingContext; import com.google.cloud.teleport.v2.templates.common.TrimmedShardedDataChangeRecord; import java.nio.charset.StandardCharsets; @@ -58,12 +58,12 @@ public void setup() { public void testWriteFilteredEventsToGcs_Success() { ProcessingContext taskContext = mock(ProcessingContext.class); Storage mockStorage = mock(Storage.class); - Shard shard = mock(Shard.class); + MySqlShard mySqlShard = mock(MySqlShard.class); when(taskContext.getGCSPath()).thenReturn(VALID_GCS_PATH); when(taskContext.getStartTimestamp()).thenReturn("2023-06-23T10:15:30Z"); when(taskContext.getWindowDuration()).thenReturn(org.joda.time.Duration.standardMinutes(10)); - when(shard.getLogicalShardId()).thenReturn("shard-123"); - when(taskContext.getShard()).thenReturn(shard); + when(mySqlShard.getLogicalShardId()).thenReturn("mySqlShard-123"); + when(taskContext.getShard()).thenReturn(mySqlShard); List mods = new ArrayList<>(); List filteredEvents = new ArrayList<>(); @@ -91,7 +91,7 @@ public void testWriteFilteredEventsToGcs_Success() { assertEquals(capturedBlobInfo.getBucket(), "my-bucket"); assertEquals( capturedBlobInfo.getName(), - "my-path/filteredEvents/shard-123/2023-06-23T10:15:30.000Z-2023-06-23T10:25:30.000Z-pane-0-last-0-of-1.txt"); + "my-path/filteredEvents/mySqlShard-123/2023-06-23T10:15:30.000Z-2023-06-23T10:25:30.000Z-pane-0-last-0-of-1.txt"); } @Test @@ -103,9 +103,9 @@ public void testWriteFilteredEventsToGcs_StorageException() { when(taskContext.getGCSPath()).thenReturn(VALID_GCS_PATH); when(taskContext.getStartTimestamp()).thenReturn("2023-06-23T10:15:30Z"); when(taskContext.getWindowDuration()).thenReturn(org.joda.time.Duration.standardMinutes(10)); - Shard shard = mock(Shard.class); - when(shard.getLogicalShardId()).thenReturn("shard-123"); - when(taskContext.getShard()).thenReturn(shard); + MySqlShard mySqlShard = mock(MySqlShard.class); + when(mySqlShard.getLogicalShardId()).thenReturn("mySqlShard-123"); + when(taskContext.getShard()).thenReturn(mySqlShard); List filteredEvents = new ArrayList<>(); diff --git a/v2/gcs-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/utils/ShardProgressTrackerTest.java b/v2/gcs-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/utils/MySqlShardProgressTrackerTest.java similarity index 99% rename from v2/gcs-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/utils/ShardProgressTrackerTest.java rename to v2/gcs-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/utils/MySqlShardProgressTrackerTest.java index 77648e6cc0..d3bde81a68 100644 --- a/v2/gcs-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/utils/ShardProgressTrackerTest.java +++ b/v2/gcs-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/utils/MySqlShardProgressTrackerTest.java @@ -36,7 +36,7 @@ import org.mockito.junit.MockitoRule; @RunWith(JUnit4.class) -public final class ShardProgressTrackerTest { +public final class MySqlShardProgressTrackerTest { @Rule public final MockitoRule mocktio = MockitoJUnit.rule(); @Mock private SpannerDao spannerDaoMock; diff --git a/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/source/reader/io/jdbc/iowrapper/config/JdbcIOWrapperConfig.java b/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/source/reader/io/jdbc/iowrapper/config/JdbcIOWrapperConfig.java index 47807677ce..84fcb66e76 100644 --- a/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/source/reader/io/jdbc/iowrapper/config/JdbcIOWrapperConfig.java +++ b/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/source/reader/io/jdbc/iowrapper/config/JdbcIOWrapperConfig.java @@ -63,7 +63,7 @@ public JdbcSchemaReference jdbcSourceSchemaReference() { /** Configured Partition Column. If unspecified for a table, it's auto-inferred. */ public abstract ImmutableMap> tableVsPartitionColumns(); - /** Shard ID. */ + /** shard ID. */ @Nullable public abstract String shardID(); diff --git a/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/templates/PipelineController.java b/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/templates/PipelineController.java index 637fa0ae23..b9366f104d 100644 --- a/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/templates/PipelineController.java +++ b/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/templates/PipelineController.java @@ -26,7 +26,7 @@ import com.google.cloud.teleport.v2.spanner.migrations.schema.ISchemaMapper; import com.google.cloud.teleport.v2.spanner.migrations.schema.IdentityMapper; import com.google.cloud.teleport.v2.spanner.migrations.schema.SessionBasedMapper; -import com.google.cloud.teleport.v2.spanner.migrations.shard.Shard; +import com.google.cloud.teleport.v2.spanner.migrations.shard.IShard; import com.google.cloud.teleport.v2.spanner.migrations.spanner.SpannerSchema; import com.google.common.annotations.VisibleForTesting; import java.util.HashMap; @@ -79,7 +79,7 @@ static PipelineResult executeSingleInstanceMigration( static PipelineResult executeShardedMigration( SourceDbToSpannerOptions options, Pipeline pipeline, - List shards, + List shards, SpannerConfig spannerConfig) { // TODO // Merge logical shards into 1 physical shard @@ -97,8 +97,8 @@ static PipelineResult executeShardedMigration( LOG.info( "running migration for shards: {}", - shards.stream().map(Shard::getHost).collect(Collectors.toList())); - for (Shard shard : shards) { + shards.stream().map(IShard::getHost).collect(Collectors.toList())); + for (IShard shard : shards) { for (Map.Entry entry : shard.getDbNameToLogicalShardIdMap().entrySet()) { // Read data from source String shardId = entry.getValue(); @@ -109,7 +109,7 @@ static PipelineResult executeShardedMigration( ShardedDbConfigContainer dbConfigContainer = new ShardedDbConfigContainer( - shard, sqlDialect, namespace, shardId, entry.getKey(), options); + shard, sqlDialect, namespace, shardId, entry.getKey(), options); setupLogicalDbMigration( options, pipeline, @@ -249,7 +249,7 @@ JdbcIOWrapperConfig getJDBCIOWrapperConfig( static class ShardedDbConfigContainer implements DbConfigContainer { - private Shard shard; + private IShard shard; private SQLDialect sqlDialect; @@ -262,7 +262,7 @@ static class ShardedDbConfigContainer implements DbConfigContainer { private SourceDbToSpannerOptions options; public ShardedDbConfigContainer( - Shard shard, + IShard shard, SQLDialect sqlDialect, String namespace, String shardId, @@ -285,7 +285,7 @@ public JdbcIOWrapperConfig getJDBCIOWrapperConfig( shard.getHost(), shard.getConnectionProperties(), Integer.parseInt(shard.getPort()), - shard.getUserName(), + shard.getUser(), shard.getPassword(), dbName, namespace, diff --git a/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/templates/SourceDbToSpanner.java b/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/templates/SourceDbToSpanner.java index bfde56e03a..1d1b88082c 100644 --- a/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/templates/SourceDbToSpanner.java +++ b/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/templates/SourceDbToSpanner.java @@ -19,7 +19,7 @@ import com.google.cloud.teleport.metadata.TemplateCategory; import com.google.cloud.teleport.v2.common.UncaughtExceptionLogger; import com.google.cloud.teleport.v2.options.SourceDbToSpannerOptions; -import com.google.cloud.teleport.v2.spanner.migrations.shard.Shard; +import com.google.cloud.teleport.v2.spanner.migrations.shard.IShard; import com.google.cloud.teleport.v2.spanner.migrations.utils.SecretManagerAccessorImpl; import com.google.cloud.teleport.v2.spanner.migrations.utils.ShardFileReader; import com.google.common.annotations.VisibleForTesting; @@ -103,7 +103,7 @@ static PipelineResult run(SourceDbToSpannerOptions options) { // Decide type and source of migration if (options.getSourceConfigURL().startsWith("gs://")) { - List shards = + List shards = new ShardFileReader(new SecretManagerAccessorImpl()) .readForwardMigrationShardingConfig(options.getSourceConfigURL()); return PipelineController.executeShardedMigration(options, pipeline, shards, spannerConfig); diff --git a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/MySQLSingleShardIT.java b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/MySQLSingleMySqlShardIT.java similarity index 98% rename from v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/MySQLSingleShardIT.java rename to v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/MySQLSingleMySqlShardIT.java index 0258c1bed4..c7b8c899e2 100644 --- a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/MySQLSingleShardIT.java +++ b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/MySQLSingleMySqlShardIT.java @@ -41,7 +41,7 @@ @Category({TemplateIntegrationTest.class, SkipDirectRunnerTest.class}) @TemplateIntegrationTest(SourceDbToSpanner.class) @RunWith(JUnit4.class) -public class MySQLSingleShardIT extends SourceDbToSpannerITBase { +public class MySQLSingleMySqlShardIT extends SourceDbToSpannerITBase { private static PipelineLauncher.LaunchInfo jobInfo; public static MySQLResourceManager mySQLResourceManager; diff --git a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/PipelineControllerTest.java b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/PipelineControllerTest.java index afc34c5096..21af6626ac 100644 --- a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/PipelineControllerTest.java +++ b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/PipelineControllerTest.java @@ -28,7 +28,7 @@ import com.google.cloud.teleport.v2.spanner.migrations.schema.ISchemaMapper; import com.google.cloud.teleport.v2.spanner.migrations.schema.IdentityMapper; import com.google.cloud.teleport.v2.spanner.migrations.schema.SessionBasedMapper; -import com.google.cloud.teleport.v2.spanner.migrations.shard.Shard; +import com.google.cloud.teleport.v2.spanner.migrations.shard.MySqlShard; import com.google.cloud.teleport.v2.templates.PipelineController.ShardedDbConfigContainer; import com.google.cloud.teleport.v2.templates.PipelineController.SingleInstanceDbConfigContainer; import com.google.common.io.Resources; @@ -242,12 +242,12 @@ public void shardedDbConfigContainerTest() { sourceDbToSpannerOptions.setPassword(testPassword); sourceDbToSpannerOptions.setTables("table1,table2"); - Shard shard = - new Shard("shard1", "localhost", "3306", "user", "password", null, null, null, null); + MySqlShard mySqlShard = + new MySqlShard("shard1", "localhost", "3306", "user", "password", null, null, null, null); ShardedDbConfigContainer dbConfigContainer = new ShardedDbConfigContainer( - shard, SQLDialect.MYSQL, null, "shard1", "testDB", sourceDbToSpannerOptions); + mySqlShard, SQLDialect.MYSQL, null, "shard1", "testDB", sourceDbToSpannerOptions); PCollection dummyPCollection = pipeline.apply(Create.of(1)); pipeline.run(); diff --git a/v2/sourcedb-to-spanner/terraform/samples/sharded-bulk-migration/README.md b/v2/sourcedb-to-spanner/terraform/samples/sharded-bulk-migration/README.md index 72104a5412..6fef8f9cb0 100644 --- a/v2/sourcedb-to-spanner/terraform/samples/sharded-bulk-migration/README.md +++ b/v2/sourcedb-to-spanner/terraform/samples/sharded-bulk-migration/README.md @@ -59,7 +59,7 @@ provides instructions to add these roles to an existing service account. ## Dataflow permissions The Dataflow service account needs to be provided the `roles/storage.objectAdmin` and `roles/spanner.databaseAdmin` -roles. Also ensure that the workers can access the source shards. +roles. Also ensure that the workers can access the source mySqlShards. ## Assumptions @@ -69,7 +69,7 @@ It makes the following assumptions - [SMT](https://googlecloudplatform.github.io/spanner-migration-tool/quickstart.html) > can be used for converting a MySQL schema to a Spanner compatible schema. -1. Dataflow can establish network connectivity with the MySQL source shards. +1. Dataflow can establish network connectivity with the MySQL source mySqlShards. 2. Appropriate permissions are added to the service account running Terraform to allow resource creation. 3. Appropriate permissions are provided to the service account running Dataflow to write to Spanner. 4. A GCS working directory has been created for the terraform template. The dataflow jobs will write the output @@ -79,8 +79,8 @@ It makes the following assumptions - 6. An SMT generated session file is provided containing the schema mapping information is created locally. This is mandatory for sharded migrations. Check out the FAQ on [how to generate a session file](#specifying-schema-overrides). -7. A sharding config containing source connection information of all the physical and logical shards is created locally. -8. The Source Schema should be the same across all shards. +7. A sharding config containing source connection information of all the physical and logical mySqlShards is created locally. +8. The Source Schema should be the same across all mySqlShards. Given these assumptions, it copies data from multiple source MySQL databases to the configured Spanner database. @@ -99,7 +99,7 @@ This sample contains the following files - to run this example. 6. `terraform_simple.tfvars` - This contains the minimal list of dummy inputs that need to be populated to run this example. -7. `shardingConfig.json` - This contains a sample sharding config that need to be populated with the source shards +7. `shardingConfig.json` - This contains a sample sharding config that need to be populated with the source mySqlShards connection details. ## Resources Created @@ -107,8 +107,8 @@ This sample contains the following files - Given these assumptions, it uses a supplied source database connection configuration and creates the following resources - -1. **Dataflow job(s)** - The Dataflow job(s) which reads from source shards and writes to - Spanner. One dataflow job can migrate multiple physical shards based on the `batch_size` parameter. +1. **Dataflow job(s)** - The Dataflow job(s) which reads from source mySqlShards and writes to + Spanner. One dataflow job can migrate multiple physical mySqlShards based on the `batch_size` parameter. 2. **GCS Session File** - The GCS object created by uploading the local session file. 3. **GCS Source Configs** - The GCS object(s) created by splitting the local sharding config into various batches, with 1 source config per dataflow job. @@ -158,7 +158,7 @@ dataflow_job_urls = [ ] ``` -**Note 1:** Each of the jobs will have a random name which will be migrating 1 or more physical shards. +**Note 1:** Each of the jobs will have a random name which will be migrating 1 or more physical mySqlShards. **Note 2:** Terraform, by default, creates at most 10 resources in parallel. If you want to run more than 10 dataflow jobs in parallel, run terraform using the `-parallelism=n` flag. However, more parallel dataflow jobs linearly increases @@ -192,11 +192,11 @@ to take over 10 mins: job logs. This likely due to private google access is not enabled for the subnetwork. Please enable private google access in your network. - **Job logs not present after "Discovering tables for DataSource":** This means dataflow is unable to access the mysql - shard. Please check your network configuration and credentials. + mySqlShard. Please check your network configuration and credentials. - If you are still getting a Timeout after this, this means the job graph construction is taking too long. You could potentially have too many tables allocated in a single Dataflow job. The recommendation is to set the `batch_size` - such that the total number of tables in a single job does not exceed `150`. This means if a logical shard has 15 - tables, and a physical shard has 2 logical shards, do not set the batch_size more than 5. + such that the total number of tables in a single job does not exceed `150`. This means if a logical mySqlShard has 15 + tables, and a physical mySqlShard has 2 logical mySqlShards, do not set the batch_size more than 5. ### Job graph is not loading/Custom counters not visible on Dataflow panel @@ -208,7 +208,7 @@ on [Cloud monitoring](https://cloud.google.com/dataflow/docs/guides/using-monito There can be multiple reasons for this. -- **Check source shard metrics:** Are memory/CPU limits being hit? Consider increasing the shard resources. +- **Check source mySqlShard metrics:** Are memory/CPU limits being hit? Consider increasing the mySqlShard resources. - **Check Spanner metrics:** Are memory/CPU limits being hit? Consider increasing the number of nodes. - **Check Dataflow metrics:** Are memory/CPU limits being hit? Dataflow should autoscale to required number of nodes. @@ -326,7 +326,7 @@ To specify a session file - This will automatically upload it to the working directory and configure it in the Dataflow job. -### Overriding Dataflow workers per shard/job +### Overriding Dataflow workers per mySqlShard/job Currently, per job configurations are not supported in this terraform template. diff --git a/v2/sourcedb-to-spanner/terraform/samples/sharded-bulk-migration/main.tf b/v2/sourcedb-to-spanner/terraform/samples/sharded-bulk-migration/main.tf index d13770549e..f1e557bfdf 100644 --- a/v2/sourcedb-to-spanner/terraform/samples/sharded-bulk-migration/main.tf +++ b/v2/sourcedb-to-spanner/terraform/samples/sharded-bulk-migration/main.tf @@ -1,5 +1,5 @@ locals { - # Generate individual source configs for each group of data shards based on batch size. + # Generate individual source configs for each group of data mySqlShards based on batch size. source_configs = [ for batch_start in range(0, length(var.data_shards), var.common_params.batch_size) : { configType : "dataflow", diff --git a/v2/sourcedb-to-spanner/terraform/samples/sharded-bulk-migration/terraform.tfvars b/v2/sourcedb-to-spanner/terraform/samples/sharded-bulk-migration/terraform.tfvars index b34212c436..7de6c7b5b3 100644 --- a/v2/sourcedb-to-spanner/terraform/samples/sharded-bulk-migration/terraform.tfvars +++ b/v2/sourcedb-to-spanner/terraform/samples/sharded-bulk-migration/terraform.tfvars @@ -25,16 +25,16 @@ common_params = { num_workers = 1 default_log_level = "INFO" - # This parameters decides the number of physical shards to migrate using a single dataflow job. + # This parameters decides the number of physical mySqlShards to migrate using a single dataflow job. # Set this in a way that restricts the total number of tables to 150 within a single job. - # Ex: if each physical shard has 2 logical shards, and each logical shard has 15 tables, + # Ex: if each physical mySqlShard has 2 logical mySqlShards, and each logical mySqlShard has 15 tables, # the batch size should not exceed 5. batch_size = 10 } data_shards = [ { - dataShardId = "data-shard1" + dataShardId = "data-mySqlShard1" host = "10.128.0.2" user = "user" password = "password" @@ -43,12 +43,12 @@ data_shards = [ { dbName = "tftest" databaseId = "logicaldb1" - refDataShardId = "data-shard1" + refDataShardId = "data-mySqlShard1" } ] }, { - dataShardId = "data-shard2" + dataShardId = "data-mySqlShard2" host = "10.128.0.3" user = "user" password = "password" @@ -57,7 +57,7 @@ data_shards = [ { dbName = "tftest" databaseId = "logicaldb2" - refDataShardId = "data-shard2" + refDataShardId = "data-mySqlShard2" } ] } diff --git a/v2/sourcedb-to-spanner/terraform/samples/sharded-bulk-migration/terraform_simple.tfvars b/v2/sourcedb-to-spanner/terraform/samples/sharded-bulk-migration/terraform_simple.tfvars index 771660f00d..79b5f40ac6 100644 --- a/v2/sourcedb-to-spanner/terraform/samples/sharded-bulk-migration/terraform_simple.tfvars +++ b/v2/sourcedb-to-spanner/terraform/samples/sharded-bulk-migration/terraform_simple.tfvars @@ -17,7 +17,7 @@ common_params = { data_shards = [ { - dataShardId = "data-shard1" + dataShardId = "data-mySqlShard1" host = "10.1.1.1" user = "username" password = "password" @@ -26,7 +26,7 @@ data_shards = [ { dbName = "db1" databaseId = "logicaldb1" - refDataShardId = "data-shard1" + refDataShardId = "data-mySqlShard1" } ] } diff --git a/v2/sourcedb-to-spanner/terraform/samples/sharded-bulk-migration/variables.tf b/v2/sourcedb-to-spanner/terraform/samples/sharded-bulk-migration/variables.tf index 719b9c0139..6e3df309e4 100644 --- a/v2/sourcedb-to-spanner/terraform/samples/sharded-bulk-migration/variables.tf +++ b/v2/sourcedb-to-spanner/terraform/samples/sharded-bulk-migration/variables.tf @@ -36,9 +36,9 @@ variable "common_params" { num_workers = optional(number) default_log_level = optional(string) - # This parameters decides the number of physical shards to migrate using a single dataflow job. + # This parameters decides the number of physical mySqlShards to migrate using a single dataflow job. # Set this in a way that restricts the total number of tables to 150 within a single job. - # Ex: if each physical shard has 2 logical shards, and each logical shard has 15 tables, + # Ex: if each physical mySqlShard has 2 logical mySqlShards, and each logical mySqlShard has 15 tables, # the batch size should not exceed 5. batch_size = optional(number, 1) }) diff --git a/v2/spanner-change-streams-to-sharded-file-sink/README.md b/v2/spanner-change-streams-to-sharded-file-sink/README.md index 861fa30c4c..80e08f6f11 100644 --- a/v2/spanner-change-streams-to-sharded-file-sink/README.md +++ b/v2/spanner-change-streams-to-sharded-file-sink/README.md @@ -1,7 +1,7 @@ # Spanner Changestream to Sharded File Sink Dataflow Template The [SpannerChangeStreamsToShardedFileSink](src/main/java/com/google/cloud/teleport/v2/templates/SpannerChangeStreamsToShardedFileSink.java) pipeline -ingests Spanner change stream data to GCS file sink sharded by logical shard id values. +ingests Spanner change stream data to GCS file sink sharded by logical mySqlShard id values. ## Getting Started diff --git a/v2/spanner-change-streams-to-sharded-file-sink/README_Spanner_Change_Streams_to_Sharded_File_Sink.md b/v2/spanner-change-streams-to-sharded-file-sink/README_Spanner_Change_Streams_to_Sharded_File_Sink.md index 76b2832750..83eada5e49 100644 --- a/v2/spanner-change-streams-to-sharded-file-sink/README_Spanner_Change_Streams_to_Sharded_File_Sink.md +++ b/v2/spanner-change-streams-to-sharded-file-sink/README_Spanner_Change_Streams_to_Sharded_File_Sink.md @@ -2,7 +2,7 @@ Spanner Change Streams to Sharded File Sink template --- Streaming pipeline. Ingests data from Spanner Change Streams, splits them into -shards and intervals , and writes them to a file sink. +mySqlShards and intervals , and writes them to a file sink. :memo: This is a Google-provided template! Please @@ -24,7 +24,7 @@ on [Metadata Annotations](https://github.com/GoogleCloudPlatform/DataflowTemplat * **metadataInstance** : This is the instance to store the metadata used by the connector to control the consumption of the change stream API data. * **metadataDatabase** : This is the database to store the metadata used by the connector to control the consumption of the change stream API data. * **gcsOutputDirectory** : The path and filename prefix for writing output files. Must end with a slash. DateTime formatting is used to parse directory path for date & time formatters. (Example: gs://your-bucket/your-path/). -* **sourceShardsFilePath** : Source shard details file path in Cloud Storage that contains connection profile of source shards. Atleast one shard information is expected. +* **sourceShardsFilePath** : Source mySqlShard details file path in Cloud Storage that contains connection profile of source mySqlShards. Atleast one mySqlShard information is expected. * **runIdentifier** : The identifier to distinguish between different runs of reverse replication flows. ### Optional parameters @@ -37,8 +37,8 @@ on [Metadata Annotations](https://github.com/GoogleCloudPlatform/DataflowTemplat * **metadataTableSuffix** : Suffix appended to the spanner_to_gcs_metadata and shard_file_create_progress metadata tables.Useful when doing multiple runs.Only alpha numeric and underscores are allowed. Defaults to empty. * **skipDirectoryName** : Records skipped from reverse replication are written to this directory. Default directory name is skip. * **runMode** : Regular starts from input start time, resume start from last processed time. Defaults to: regular. -* **shardingCustomJarPath** : Custom jar location in Cloud Storage that contains the customization logic for fetching shard id. Defaults to empty. -* **shardingCustomClassName** : Fully qualified class name having the custom shard id implementation. It is a mandatory field in case shardingCustomJarPath is specified. Defaults to empty. +* **shardingCustomJarPath** : Custom jar location in Cloud Storage that contains the customization logic for fetching mySqlShard id. Defaults to empty. +* **shardingCustomClassName** : Fully qualified class name having the custom mySqlShard id implementation. It is a mandatory field in case shardingCustomJarPath is specified. Defaults to empty. * **shardingCustomParameters** : String containing any custom parameters to be passed to the custom sharding class. Defaults to empty. diff --git a/v2/spanner-change-streams-to-sharded-file-sink/pom.xml b/v2/spanner-change-streams-to-sharded-file-sink/pom.xml index c875b60880..4c56f8d76a 100644 --- a/v2/spanner-change-streams-to-sharded-file-sink/pom.xml +++ b/v2/spanner-change-streams-to-sharded-file-sink/pom.xml @@ -45,7 +45,7 @@ com.google.cloud.teleport.v2 - spanner-custom-shard + spanner-custom-mySqlShard ${project.version} test diff --git a/v2/spanner-change-streams-to-sharded-file-sink/src/main/java/com/google/cloud/teleport/v2/templates/SpannerChangeStreamsToShardedFileSink.java b/v2/spanner-change-streams-to-sharded-file-sink/src/main/java/com/google/cloud/teleport/v2/templates/SpannerChangeStreamsToShardedFileSink.java index ed650d4727..554ce3c5b7 100644 --- a/v2/spanner-change-streams-to-sharded-file-sink/src/main/java/com/google/cloud/teleport/v2/templates/SpannerChangeStreamsToShardedFileSink.java +++ b/v2/spanner-change-streams-to-sharded-file-sink/src/main/java/com/google/cloud/teleport/v2/templates/SpannerChangeStreamsToShardedFileSink.java @@ -25,7 +25,7 @@ import com.google.cloud.teleport.v2.spanner.ddl.Ddl; import com.google.cloud.teleport.v2.spanner.migrations.metadata.SpannerToGcsJobMetadata; import com.google.cloud.teleport.v2.spanner.migrations.schema.Schema; -import com.google.cloud.teleport.v2.spanner.migrations.shard.Shard; +import com.google.cloud.teleport.v2.spanner.migrations.shard.MySqlShard; import com.google.cloud.teleport.v2.spanner.migrations.utils.SecretManagerAccessorImpl; import com.google.cloud.teleport.v2.spanner.migrations.utils.SessionFileReader; import com.google.cloud.teleport.v2.spanner.migrations.utils.ShardFileReader; @@ -353,13 +353,13 @@ public static PipelineResult run(Options options) { Pipeline pipeline = Pipeline.create(options); ShardFileReader shardFileReader = new ShardFileReader(new SecretManagerAccessorImpl()); - List shards = shardFileReader.getOrderedShardDetails(options.getSourceShardsFilePath()); - if (shards == null || shards.isEmpty()) { - throw new RuntimeException("The source shards file cannot be empty"); + List mySqlShards = shardFileReader.getOrderedShardDetails(options.getSourceShardsFilePath()); + if (mySqlShards == null || mySqlShards.isEmpty()) { + throw new RuntimeException("The source mySqlShards file cannot be empty"); } String shardingMode = Constants.SHARDING_MODE_SINGLE_SHARD; - if (shards.size() > 1) { + if (mySqlShards.size() > 1) { shardingMode = Constants.SHARDING_MODE_MULTI_SHARD; } @@ -424,7 +424,7 @@ public static PipelineResult run(Options options) { // updating per shard file creation progress FileCreationTracker fileCreationTracker = new FileCreationTracker(spannerDao, options.getRunIdentifier()); - fileCreationTracker.init(shards); + fileCreationTracker.init(mySqlShards); spannerDao.close(); @@ -443,7 +443,7 @@ public static PipelineResult run(Options options) { schema, ddl, shardingMode, - shards.get(0).getLogicalShardId(), + mySqlShards.get(0).getLogicalShardId(), options.getSkipDirectoryName(), options.getShardingCustomJarPath(), options.getShardingCustomClassName(), diff --git a/v2/spanner-change-streams-to-sharded-file-sink/src/main/java/com/google/cloud/teleport/v2/templates/utils/FileCreationTracker.java b/v2/spanner-change-streams-to-sharded-file-sink/src/main/java/com/google/cloud/teleport/v2/templates/utils/FileCreationTracker.java index e450f3d1b9..01e0724c6a 100644 --- a/v2/spanner-change-streams-to-sharded-file-sink/src/main/java/com/google/cloud/teleport/v2/templates/utils/FileCreationTracker.java +++ b/v2/spanner-change-streams-to-sharded-file-sink/src/main/java/com/google/cloud/teleport/v2/templates/utils/FileCreationTracker.java @@ -16,7 +16,7 @@ package com.google.cloud.teleport.v2.templates.utils; import com.google.cloud.spanner.SpannerException; -import com.google.cloud.teleport.v2.spanner.migrations.shard.Shard; +import com.google.cloud.teleport.v2.spanner.migrations.shard.MySqlShard; import java.util.List; import org.apache.beam.sdk.io.gcp.spanner.SpannerConfig; @@ -37,8 +37,8 @@ public FileCreationTracker(SpannerDao spannerDao, String runId) { this.runId = runId; } - public void init(List shards) { - spannerDao.initShardProgress(shards, runId); + public void init(List mySqlShards) { + spannerDao.initShardProgress(mySqlShards, runId); } public void updateProgress(String shard, String endTime) { diff --git a/v2/spanner-change-streams-to-sharded-file-sink/src/main/java/com/google/cloud/teleport/v2/templates/utils/SpannerDao.java b/v2/spanner-change-streams-to-sharded-file-sink/src/main/java/com/google/cloud/teleport/v2/templates/utils/SpannerDao.java index 990003c6bd..e781e3207c 100644 --- a/v2/spanner-change-streams-to-sharded-file-sink/src/main/java/com/google/cloud/teleport/v2/templates/utils/SpannerDao.java +++ b/v2/spanner-change-streams-to-sharded-file-sink/src/main/java/com/google/cloud/teleport/v2/templates/utils/SpannerDao.java @@ -25,7 +25,7 @@ import com.google.cloud.spanner.Statement; import com.google.cloud.spanner.TransactionRunner.TransactionCallable; import com.google.cloud.teleport.v2.spanner.migrations.metadata.SpannerToGcsJobMetadata; -import com.google.cloud.teleport.v2.spanner.migrations.shard.Shard; +import com.google.cloud.teleport.v2.spanner.migrations.shard.MySqlShard; import com.google.spanner.admin.database.v1.UpdateDatabaseDdlMetadata; import java.util.ArrayList; import java.util.Arrays; @@ -253,19 +253,19 @@ private void checkAndCreateDataSeenTable() { } } - public void initShardProgress(List shards, String runId) { + public void initShardProgress(List mySqlShards, String runId) { checkAndCreateDataSeenTable(); checkAndCreateProgressTable(); List mutations = new ArrayList<>(); Timestamp epochTimestamp = Timestamp.parseTimestamp("1970-01-01T12:00:00Z"); - for (Shard shard : shards) { + for (MySqlShard mySqlShard : mySqlShards) { mutations.add( Mutation.newInsertOrUpdateBuilder(shardFileCreateProgressTableName) .set("run_id") .to(runId) - .set("shard") - .to(shard.getLogicalShardId()) + .set("mySqlShard") + .to(mySqlShard.getLogicalShardId()) .set("created_upto") .to(epochTimestamp) .build()); diff --git a/v2/spanner-change-streams-to-sharded-file-sink/src/test/java/com/google/cloud/teleport/v2/templates/SpannerChangeStreamToGcsCustomShardIT.java b/v2/spanner-change-streams-to-sharded-file-sink/src/test/java/com/google/cloud/teleport/v2/templates/SpannerChangeStreamToGcsCustomMySqlShardIT.java similarity index 90% rename from v2/spanner-change-streams-to-sharded-file-sink/src/test/java/com/google/cloud/teleport/v2/templates/SpannerChangeStreamToGcsCustomShardIT.java rename to v2/spanner-change-streams-to-sharded-file-sink/src/test/java/com/google/cloud/teleport/v2/templates/SpannerChangeStreamToGcsCustomMySqlShardIT.java index 3c3278266c..b687ace82b 100644 --- a/v2/spanner-change-streams-to-sharded-file-sink/src/test/java/com/google/cloud/teleport/v2/templates/SpannerChangeStreamToGcsCustomShardIT.java +++ b/v2/spanner-change-streams-to-sharded-file-sink/src/test/java/com/google/cloud/teleport/v2/templates/SpannerChangeStreamToGcsCustomMySqlShardIT.java @@ -21,7 +21,7 @@ import com.google.cloud.spanner.Mutation; import com.google.cloud.teleport.metadata.SkipDirectRunnerTest; import com.google.cloud.teleport.metadata.TemplateIntegrationTest; -import com.google.cloud.teleport.v2.spanner.migrations.shard.Shard; +import com.google.cloud.teleport.v2.spanner.migrations.shard.MySqlShard; import com.google.gson.Gson; import com.google.gson.JsonArray; import com.google.gson.JsonObject; @@ -53,10 +53,10 @@ @Category({TemplateIntegrationTest.class, SkipDirectRunnerTest.class}) @TemplateIntegrationTest(SpannerChangeStreamsToShardedFileSink.class) @RunWith(JUnit4.class) -public class SpannerChangeStreamToGcsCustomShardIT extends SpannerChangeStreamToGcsITBase { +public class SpannerChangeStreamToGcsCustomMySqlShardIT extends SpannerChangeStreamToGcsITBase { private static final Logger LOG = - LoggerFactory.getLogger(SpannerChangeStreamToGcsCustomShardIT.class); - private static HashSet testInstances = new HashSet<>(); + LoggerFactory.getLogger(SpannerChangeStreamToGcsCustomMySqlShardIT.class); + private static HashSet testInstances = new HashSet<>(); private static GcsResourceManager gcsResourceManager; private static SpannerResourceManager spannerResourceManager; @@ -82,7 +82,7 @@ public class SpannerChangeStreamToGcsCustomShardIT extends SpannerChangeStreamTo @Before public void setUp() throws IOException, InterruptedException { skipBaseCleanup = true; - synchronized (SpannerChangeStreamToGcsCustomShardIT.class) { + synchronized (SpannerChangeStreamToGcsCustomMySqlShardIT.class) { testInstances.add(this); if (jobInfo == null) { gcsResourceManager = createGcsResourceManager(getClass().getSimpleName()); @@ -111,7 +111,7 @@ public void setUp() throws IOException, InterruptedException { @AfterClass public static void cleanUp() throws IOException { - for (SpannerChangeStreamToGcsCustomShardIT instance : testInstances) { + for (SpannerChangeStreamToGcsCustomMySqlShardIT instance : testInstances) { instance.tearDownBase(); } ResourceManagerUtils.cleanResources( @@ -206,18 +206,18 @@ private void createAndUploadShardConfigToGcs() throws IOException { JsonArray ja = new JsonArray(); for (String shardName : shardNames) { - Shard shard = new Shard(); - shard.setLogicalShardId(shardName); - shard.setUser("dummy"); - shard.setHost("dummy"); - shard.setPassword("dummy"); - shard.setPort("3306"); - JsonObject jsObj = (JsonObject) new Gson().toJsonTree(shard).getAsJsonObject(); + MySqlShard mySqlShard = new MySqlShard(); + mySqlShard.setLogicalShardId(shardName); + mySqlShard.setUser("dummy"); + mySqlShard.setHost("dummy"); + mySqlShard.setPassword("dummy"); + mySqlShard.setPort("3306"); + JsonObject jsObj = (JsonObject) new Gson().toJsonTree(mySqlShard).getAsJsonObject(); ja.add(jsObj); } String shardFileContents = ja.toString(); - LOG.info("Shard file contents: {}", shardFileContents); + LOG.info("MySqlShard file contents: {}", shardFileContents); // -DartifactBucket has the bucket name gcsResourceManager.createArtifact("input/shard.json", shardFileContents); } diff --git a/v2/spanner-change-streams-to-sharded-file-sink/src/test/java/com/google/cloud/teleport/v2/templates/SpannerChangeStreamToGcsMultiShardIT.java b/v2/spanner-change-streams-to-sharded-file-sink/src/test/java/com/google/cloud/teleport/v2/templates/SpannerChangeStreamToGcsMultiMySqlShardIT.java similarity index 93% rename from v2/spanner-change-streams-to-sharded-file-sink/src/test/java/com/google/cloud/teleport/v2/templates/SpannerChangeStreamToGcsMultiShardIT.java rename to v2/spanner-change-streams-to-sharded-file-sink/src/test/java/com/google/cloud/teleport/v2/templates/SpannerChangeStreamToGcsMultiMySqlShardIT.java index 2e9d52d4f4..dd071a6fb5 100644 --- a/v2/spanner-change-streams-to-sharded-file-sink/src/test/java/com/google/cloud/teleport/v2/templates/SpannerChangeStreamToGcsMultiShardIT.java +++ b/v2/spanner-change-streams-to-sharded-file-sink/src/test/java/com/google/cloud/teleport/v2/templates/SpannerChangeStreamToGcsMultiMySqlShardIT.java @@ -23,7 +23,7 @@ import com.google.cloud.spanner.TransactionRunner.TransactionCallable; import com.google.cloud.teleport.metadata.SkipDirectRunnerTest; import com.google.cloud.teleport.metadata.TemplateIntegrationTest; -import com.google.cloud.teleport.v2.spanner.migrations.shard.Shard; +import com.google.cloud.teleport.v2.spanner.migrations.shard.MySqlShard; import com.google.gson.Gson; import com.google.gson.JsonArray; import com.google.gson.JsonObject; @@ -57,12 +57,12 @@ @Category({TemplateIntegrationTest.class, SkipDirectRunnerTest.class}) @TemplateIntegrationTest(SpannerChangeStreamsToShardedFileSink.class) @RunWith(JUnit4.class) -public class SpannerChangeStreamToGcsMultiShardIT extends SpannerChangeStreamToGcsITBase { +public class SpannerChangeStreamToGcsMultiMySqlShardIT extends SpannerChangeStreamToGcsITBase { private static final Logger LOG = - LoggerFactory.getLogger(SpannerChangeStreamToGcsMultiShardIT.class); + LoggerFactory.getLogger(SpannerChangeStreamToGcsMultiMySqlShardIT.class); private static SpannerResourceManager spannerResourceManager; private static SpannerResourceManager spannerMetadataResourceManager; - private static HashSet testInstances = new HashSet<>(); + private static HashSet testInstances = new HashSet<>(); private static final String spannerDdl = "SpannerChangeStreamToGcsMultiShardIT/spanner-schema.sql"; private static final String sessionFileResourceName = @@ -85,7 +85,7 @@ public class SpannerChangeStreamToGcsMultiShardIT extends SpannerChangeStreamToG @Before public void setUp() throws IOException { skipBaseCleanup = true; - synchronized (SpannerChangeStreamToGcsMultiShardIT.class) { + synchronized (SpannerChangeStreamToGcsMultiMySqlShardIT.class) { testInstances.add(this); if (jobInfo == null) { gcsResourceManager = createGcsResourceManager(getClass().getSimpleName()); @@ -113,7 +113,7 @@ public void setUp() throws IOException { @AfterClass public static void cleanUp() throws IOException { - for (SpannerChangeStreamToGcsMultiShardIT instance : testInstances) { + for (SpannerChangeStreamToGcsMultiMySqlShardIT instance : testInstances) { instance.tearDownBase(); } ResourceManagerUtils.cleanResources( @@ -127,18 +127,18 @@ private void createAndUploadShardConfigToGcs() throws IOException { JsonArray ja = new JsonArray(); for (String shardName : shardNames) { - Shard shard = new Shard(); - shard.setLogicalShardId(shardName); - shard.setUser("dummy"); - shard.setHost("dummy"); - shard.setPassword("dummy"); - shard.setPort("3306"); - JsonObject jsObj = (JsonObject) new Gson().toJsonTree(shard).getAsJsonObject(); + MySqlShard mySqlShard = new MySqlShard(); + mySqlShard.setLogicalShardId(shardName); + mySqlShard.setUser("dummy"); + mySqlShard.setHost("dummy"); + mySqlShard.setPassword("dummy"); + mySqlShard.setPort("3306"); + JsonObject jsObj = (JsonObject) new Gson().toJsonTree(mySqlShard).getAsJsonObject(); ja.add(jsObj); } String shardFileContents = ja.toString(); - LOG.info("Shard file contents: {}", shardFileContents); + LOG.info("MySqlShard file contents: {}", shardFileContents); // -DartifactBucket has the bucket name gcsResourceManager.createArtifact("input/shard.json", shardFileContents); } diff --git a/v2/spanner-change-streams-to-sharded-file-sink/src/test/java/com/google/cloud/teleport/v2/templates/SpannerChangeStreamToGcsSingleShardIT.java b/v2/spanner-change-streams-to-sharded-file-sink/src/test/java/com/google/cloud/teleport/v2/templates/SpannerChangeStreamToGcsSingleMySqlShardIT.java similarity index 92% rename from v2/spanner-change-streams-to-sharded-file-sink/src/test/java/com/google/cloud/teleport/v2/templates/SpannerChangeStreamToGcsSingleShardIT.java rename to v2/spanner-change-streams-to-sharded-file-sink/src/test/java/com/google/cloud/teleport/v2/templates/SpannerChangeStreamToGcsSingleMySqlShardIT.java index 07b4e0459d..d5e4564b18 100644 --- a/v2/spanner-change-streams-to-sharded-file-sink/src/test/java/com/google/cloud/teleport/v2/templates/SpannerChangeStreamToGcsSingleShardIT.java +++ b/v2/spanner-change-streams-to-sharded-file-sink/src/test/java/com/google/cloud/teleport/v2/templates/SpannerChangeStreamToGcsSingleMySqlShardIT.java @@ -21,7 +21,7 @@ import com.google.cloud.spanner.Mutation; import com.google.cloud.teleport.metadata.SkipDirectRunnerTest; import com.google.cloud.teleport.metadata.TemplateIntegrationTest; -import com.google.cloud.teleport.v2.spanner.migrations.shard.Shard; +import com.google.cloud.teleport.v2.spanner.migrations.shard.MySqlShard; import com.google.gson.Gson; import com.google.gson.JsonArray; import com.google.gson.JsonObject; @@ -53,12 +53,12 @@ @Category({TemplateIntegrationTest.class, SkipDirectRunnerTest.class}) @TemplateIntegrationTest(SpannerChangeStreamsToShardedFileSink.class) @RunWith(JUnit4.class) -public class SpannerChangeStreamToGcsSingleShardIT extends SpannerChangeStreamToGcsITBase { +public class SpannerChangeStreamToGcsSingleMySqlShardIT extends SpannerChangeStreamToGcsITBase { private static final Logger LOG = - LoggerFactory.getLogger(SpannerChangeStreamToGcsSingleShardIT.class); + LoggerFactory.getLogger(SpannerChangeStreamToGcsSingleMySqlShardIT.class); private static SpannerResourceManager spannerResourceManager; private static SpannerResourceManager spannerMetadataResourceManager; - private static HashSet testInstances = new HashSet<>(); + private static HashSet testInstances = new HashSet<>(); private static final String spannerDdl = "SpannerChangeStreamToGcsSingleShardIT/spanner-schema.sql"; private static PipelineLauncher.LaunchInfo jobInfo; @@ -79,7 +79,7 @@ public class SpannerChangeStreamToGcsSingleShardIT extends SpannerChangeStreamTo @Before public void setUp() throws IOException { skipBaseCleanup = true; - synchronized (SpannerChangeStreamToGcsSingleShardIT.class) { + synchronized (SpannerChangeStreamToGcsSingleMySqlShardIT.class) { testInstances.add(this); if (jobInfo == null) { gcsResourceManager = createGcsResourceManager(getClass().getSimpleName()); @@ -107,7 +107,7 @@ public void setUp() throws IOException { @AfterClass public static void cleanUp() throws IOException { - for (SpannerChangeStreamToGcsSingleShardIT instance : testInstances) { + for (SpannerChangeStreamToGcsSingleMySqlShardIT instance : testInstances) { instance.tearDownBase(); } ResourceManagerUtils.cleanResources( @@ -116,19 +116,19 @@ public static void cleanUp() throws IOException { private void createAndUploadShardConfigToGcs() throws IOException { JsonArray ja = new JsonArray(); - Shard shard = new Shard(); - shard.setLogicalShardId("testShardA"); - shard.setUser("dummy"); - shard.setHost("dummy"); - shard.setPassword("dummy"); - shard.setPort("3306"); - JsonObject jsObj = (JsonObject) new Gson().toJsonTree(shard).getAsJsonObject(); + MySqlShard mySqlShard = new MySqlShard(); + mySqlShard.setLogicalShardId("testShardA"); + mySqlShard.setUser("dummy"); + mySqlShard.setHost("dummy"); + mySqlShard.setPassword("dummy"); + mySqlShard.setPort("3306"); + JsonObject jsObj = (JsonObject) new Gson().toJsonTree(mySqlShard).getAsJsonObject(); ja.add(jsObj); String shardFileContents = ja.toString(); - LOG.info("Shard file contents: {}", shardFileContents); + LOG.info("MySqlShard file contents: {}", shardFileContents); // -DartifactBucket has the bucket name - gcsResourceManager.createArtifact("input/shard.json", shardFileContents); + gcsResourceManager.createArtifact("input/mySqlShard.json", shardFileContents); } @Test diff --git a/v2/spanner-change-streams-to-sharded-file-sink/src/test/java/com/google/cloud/teleport/v2/templates/transforms/AssignShardIdFnTest.java b/v2/spanner-change-streams-to-sharded-file-sink/src/test/java/com/google/cloud/teleport/v2/templates/transforms/AssignMySqlShardIdFnTest.java similarity index 99% rename from v2/spanner-change-streams-to-sharded-file-sink/src/test/java/com/google/cloud/teleport/v2/templates/transforms/AssignShardIdFnTest.java rename to v2/spanner-change-streams-to-sharded-file-sink/src/test/java/com/google/cloud/teleport/v2/templates/transforms/AssignMySqlShardIdFnTest.java index 787d0285b1..743a769a02 100644 --- a/v2/spanner-change-streams-to-sharded-file-sink/src/test/java/com/google/cloud/teleport/v2/templates/transforms/AssignShardIdFnTest.java +++ b/v2/spanner-change-streams-to-sharded-file-sink/src/test/java/com/google/cloud/teleport/v2/templates/transforms/AssignMySqlShardIdFnTest.java @@ -62,9 +62,9 @@ import org.mockito.junit.MockitoJUnit; import org.mockito.junit.MockitoRule; -/** Tests for AssignShardIdFnTest class. */ +/** Tests for AssignMySqlShardIdFnTest class. */ @FixMethodOrder(MethodSorters.NAME_ASCENDING) -public class AssignShardIdFnTest { +public class AssignMySqlShardIdFnTest { @Rule public final transient TestPipeline pipeline = TestPipeline.create(); @Rule public final MockitoRule mocktio = MockitoJUnit.rule(); diff --git a/v2/spanner-change-streams-to-sharded-file-sink/terraform/Spanner_Change_Streams_to_Sharded_File_Sink/dataflow_job.tf b/v2/spanner-change-streams-to-sharded-file-sink/terraform/Spanner_Change_Streams_to_Sharded_File_Sink/dataflow_job.tf index e63faf45ec..4c23262ed8 100644 --- a/v2/spanner-change-streams-to-sharded-file-sink/terraform/Spanner_Change_Streams_to_Sharded_File_Sink/dataflow_job.tf +++ b/v2/spanner-change-streams-to-sharded-file-sink/terraform/Spanner_Change_Streams_to_Sharded_File_Sink/dataflow_job.tf @@ -107,7 +107,7 @@ variable "filtrationMode" { variable "sourceShardsFilePath" { type = string - description = "Source shard details file path in Cloud Storage that contains connection profile of source shards. Atleast one shard information is expected." + description = "Source mySqlShard details file path in Cloud Storage that contains connection profile of source mySqlShards. Atleast one mySqlShard information is expected." } @@ -137,13 +137,13 @@ variable "runMode" { variable "shardingCustomJarPath" { type = string - description = "Custom jar location in Cloud Storage that contains the customization logic for fetching shard id. Defaults to empty." + description = "Custom jar location in Cloud Storage that contains the customization logic for fetching mySqlShard id. Defaults to empty." default = null } variable "shardingCustomClassName" { type = string - description = "Fully qualified class name having the custom shard id implementation. It is a mandatory field in case shardingCustomJarPath is specified. Defaults to empty." + description = "Fully qualified class name having the custom mySqlShard id implementation. It is a mandatory field in case shardingCustomJarPath is specified. Defaults to empty." default = null } diff --git a/v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/migrations/shard/CassandraShard.java b/v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/migrations/shard/CassandraShard.java new file mode 100644 index 0000000000..6c40f4397e --- /dev/null +++ b/v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/migrations/shard/CassandraShard.java @@ -0,0 +1,276 @@ +package com.google.cloud.teleport.v2.spanner.migrations.shard; + +import java.util.Map; +import java.util.Objects; + +public class CassandraShard implements IShard { + private String logicalShardId; + private String host; + private String port; + private String username; + private String password; + private String keyspace; + private String consistencyLevel = "LOCAL_QUORUM"; + private boolean sslOptions = false; + private String protocolVersion = "v5"; + private String dataCenter = "datacenter1"; + private int localPoolSize = 1024; + private int remotePoolSize = 256; + + @Override + public String getLogicalShardId() { + return logicalShardId; + } + + @Override + public String getHost() { + return host; + } + + @Override + public String getPort() { + return port; + } + + @Override + public String getUser() { + return username; + } + + @Override + public String getPassword() { + return password; + } + + @Override + public String getDbName() { + throw new IllegalArgumentException("Not Supported in Cassandra Shard"); + } + + @Override + public String getNamespace() { + throw new IllegalArgumentException("Not Supported in Cassandra Shard"); + } + + @Override + public String getSecretManagerUri() { + throw new IllegalArgumentException("Not Supported in Cassandra Shard"); + } + + @Override + public String getConnectionProperties() { + throw new IllegalArgumentException("Not Supported in Cassandra Shard"); + } + + @Override + public Map getDbNameToLogicalShardIdMap() { + throw new IllegalArgumentException("Not Supported in Cassandra Shard"); + } + + @Override + public String getKeySpaceName() { + return keyspace; + } + + @Override + public String getConsistencyLevel() { + return consistencyLevel; + } + + @Override + public Boolean getSSLOptions() { + return sslOptions; + } + + @Override + public String getProtocolVersion() { + return protocolVersion; + } + + @Override + public String getDataCenter() { + return dataCenter; + } + + @Override + public Integer getLocalPoolSize() { + return localPoolSize; + } + + @Override + public Integer getRemotePoolSize() { + return remotePoolSize; + } + + @Override + public void setLogicalShardId(String logicalShardId) { + this.logicalShardId = logicalShardId; + } + + @Override + public void setHost(String host) { + this.host = host; + } + + @Override + public void setPort(String port) { + this.port = port; + } + + @Override + public void setUser(String user) { + this.username = user; + } + + @Override + public void setPassword(String password) { + this.password = password; + } + + @Override + public void setDbName(String dbName) { + throw new IllegalArgumentException("Not Supported in Cassandra Shard"); + } + + @Override + public void setNamespace(String namespace) { + throw new IllegalArgumentException("Not Supported in Cassandra Shard"); + } + + @Override + public void setSecretManagerUri(String secretManagerUri) { + throw new IllegalArgumentException("Not Supported in Cassandra Shard"); + } + + @Override + public void setConnectionProperties(String connectionProperties) { + throw new IllegalArgumentException("Not Supported in Cassandra Shard"); + } + + @Override + public void setDbNameToLogicalShardIdMap(Map dbNameToLogicalShardIdMap) { + throw new IllegalArgumentException("Not Supported in Cassandra Shard"); + } + + @Override + public void setKeySpaceName(String keySpaceName) { + + } + + @Override + public void setConsistencyLevel(String consistencyLevel) { + this.consistencyLevel = consistencyLevel; + } + + @Override + public void setSslOptions(boolean sslOptions) { + this.sslOptions = sslOptions; + } + + @Override + public void setProtocolVersion(String protocolVersion) { + this.protocolVersion = protocolVersion; + } + + @Override + public void setDataCenter(String dataCenter) { + this.dataCenter = dataCenter; + } + + @Override + public void setLocalPoolSize(int localPoolSize) { + this.localPoolSize = localPoolSize; + } + + @Override + public void setRemotePoolSize(int remotePoolSize) { + this.remotePoolSize = remotePoolSize; + } + + public void validate() throws IllegalArgumentException { + if (host == null || host.isEmpty()) { + throw new IllegalArgumentException("Host is required"); + } + if (port == null || port.isEmpty()) { + throw new IllegalArgumentException("Port is required"); + } + if (username == null || username.isEmpty()) { + throw new IllegalArgumentException("Username is required"); + } + if (password == null || password.isEmpty()) { + throw new IllegalArgumentException("Password is required"); + } + if (keyspace == null || keyspace.isEmpty()) { + throw new IllegalArgumentException("Keyspace is required"); + } + } + + @Override + public String toString() { + return "CassandraShard{" + + "logicalShardId='" + + logicalShardId + + '\'' + + ", host='" + + host + + '\'' + + ", port='" + + port + + '\'' + + ", user='" + + username + + '\'' + + ", keySpaceName='" + + keyspace + + '\'' + + ", datacenter='" + + dataCenter + + '\'' + + ", consistencyLevel='" + + consistencyLevel + + '\'' + + ", protocolVersion=" + + protocolVersion + + '}'; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof CassandraShard)) { + return false; + } + CassandraShard cassandraShard = (CassandraShard) o; + return Objects.equals(logicalShardId, cassandraShard.logicalShardId) + && Objects.equals(host, cassandraShard.host) + && Objects.equals(port, cassandraShard.port) + && Objects.equals(username, cassandraShard.username) + && Objects.equals(password, cassandraShard.password) + && Objects.equals(keyspace, cassandraShard.keyspace) + && Objects.equals(dataCenter, cassandraShard.dataCenter) + && Objects.equals(consistencyLevel, cassandraShard.consistencyLevel) + && Objects.equals(protocolVersion, cassandraShard.protocolVersion) + && Objects.equals(sslOptions, cassandraShard.sslOptions) + && Objects.equals(localPoolSize, cassandraShard.localPoolSize) + && Objects.equals(remotePoolSize, cassandraShard.remotePoolSize); + } + + @Override + public int hashCode() { + return Objects.hash( + logicalShardId, + host, + port, + username, + password, + keyspace, + dataCenter, + consistencyLevel, + protocolVersion, + sslOptions, + localPoolSize, + remotePoolSize); + } +} diff --git a/v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/migrations/shard/IShard.java b/v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/migrations/shard/IShard.java new file mode 100644 index 0000000000..06c19d472f --- /dev/null +++ b/v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/migrations/shard/IShard.java @@ -0,0 +1,50 @@ +package com.google.cloud.teleport.v2.spanner.migrations.shard; + +import java.io.Serializable; +import java.util.Map; + +public interface IShard extends Serializable { + // Getter methods + String getLogicalShardId(); + String getHost(); + String getPort(); + String getUser(); + String getPassword(); + String getDbName(); + String getNamespace(); + String getSecretManagerUri(); + String getConnectionProperties(); + Map getDbNameToLogicalShardIdMap(); + + // Cassandra Configuration + String getKeySpaceName(); + String getConsistencyLevel(); + Boolean getSSLOptions(); + String getProtocolVersion(); + String getDataCenter(); + Integer getLocalPoolSize(); + Integer getRemotePoolSize(); + + + // Setter methods + void setLogicalShardId(String logicalShardId); + void setHost(String host); + void setPort(String port); + void setUser(String user); + void setPassword(String password); + void setDbName(String dbName); + void setNamespace(String namespace); + void setSecretManagerUri(String secretManagerUri); + void setConnectionProperties(String connectionProperties); + void setDbNameToLogicalShardIdMap(Map dbNameToLogicalShardIdMap); + + // cassandra configuration + void setKeySpaceName(String keySpaceName); + void setConsistencyLevel(String consistencyLevel); + void setSslOptions(boolean sslOptions); + void setProtocolVersion(String protocolVersion); + void setDataCenter(String dataCenter); + void setLocalPoolSize(int localPoolSize); + void setRemotePoolSize(int remotePoolSize); + +} diff --git a/v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/migrations/shard/MySqlShard.java b/v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/migrations/shard/MySqlShard.java new file mode 100644 index 0000000000..59011f8d89 --- /dev/null +++ b/v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/migrations/shard/MySqlShard.java @@ -0,0 +1,294 @@ +/* + * Copyright (C) 2023 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.spanner.migrations.shard; + +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; + +public class MySqlShard implements IShard { + + private String logicalShardId; + private String host; + private String port; + private String user; + private String password; + private String dbName; + private String namespace; + private String secretManagerUri; + private String connectionProperties; + + private Map dbNameToLogicalShardIdMap = new HashMap<>(); + + public MySqlShard( + String logicalShardId, + String host, + String port, + String user, + String password, + String dbName, + String namespace, + String secretManagerUri, + String connectionProperties) { + this.logicalShardId = logicalShardId; + this.host = host; + this.port = port; + this.user = user; + this.password = password; + this.dbName = dbName; + this.namespace = namespace; + this.secretManagerUri = secretManagerUri; + this.connectionProperties = connectionProperties; + } + + public MySqlShard() { + } + + @Override + public String getLogicalShardId() { + return logicalShardId; + } + + @Override + public String getHost() { + return host; + } + + @Override + public String getPort() { + return port; + } + + @Override + public String getUser() { + return user; + } + + @Override + public String getPassword() { + return password; + } + + @Override + public String getDbName() { + return dbName; + } + + @Override + public String getNamespace() { + return namespace; + } + + @Override + public String getSecretManagerUri() { + return secretManagerUri; + } + + @Override + public String getConnectionProperties() { + return connectionProperties; + } + + @Override + public Map getDbNameToLogicalShardIdMap() { + return dbNameToLogicalShardIdMap; + } + + @Override + public String getKeySpaceName() { + throw new IllegalArgumentException("Not Supported in Mysql Shard"); + } + + @Override + public String getConsistencyLevel() { + throw new IllegalArgumentException("Not Supported in Mysql Shard"); + } + + @Override + public Boolean getSSLOptions() { + throw new IllegalArgumentException("Not Supported in Mysql Shard"); + } + + @Override + public String getProtocolVersion() { + throw new IllegalArgumentException("Not Supported in Mysql Shard"); + } + + @Override + public String getDataCenter() { + throw new IllegalArgumentException("Not Supported in Mysql Shard"); + } + + @Override + public Integer getLocalPoolSize() { + throw new IllegalArgumentException("Not Supported in Mysql Shard"); + } + + @Override + public Integer getRemotePoolSize() { + throw new IllegalArgumentException("Not Supported in Mysql Shard"); + } + + @Override + public void setLogicalShardId(String logicalShardId) { + this.logicalShardId = logicalShardId; + } + + @Override + public void setHost(String host) { + this.host = host; + } + + @Override + public void setPort(String port) { + this.port = port; + } + + @Override + public void setUser(String user) { + this.user = user; + } + + @Override + public void setPassword(String password) { + this.password = password; + } + + @Override + public void setDbName(String dbName) { + this.dbName = dbName; + } + + @Override + public void setNamespace(String namespace) { + this.namespace = namespace; + } + + @Override + public void setSecretManagerUri(String secretManagerUri) { + this.secretManagerUri = secretManagerUri; + } + + @Override + public void setConnectionProperties(String connectionProperties) { + this.connectionProperties = connectionProperties; + } + + @Override + public void setDbNameToLogicalShardIdMap(Map dbNameToLogicalShardIdMap) { + this.dbNameToLogicalShardIdMap = dbNameToLogicalShardIdMap; + } + + @Override + public void setKeySpaceName(String keySpaceName) { + throw new IllegalArgumentException("Not Supported in Mysql Shard"); + } + + @Override + public void setConsistencyLevel(String consistencyLevel) { + throw new IllegalArgumentException("Not Supported in Mysql Shard"); + } + + @Override + public void setSslOptions(boolean sslOptions) { + throw new IllegalArgumentException("Not Supported in Mysql Shard"); + } + + @Override + public void setProtocolVersion(String protocolVersion) { + throw new IllegalArgumentException("Not Supported in Mysql Shard"); + } + + @Override + public void setDataCenter(String dataCenter) { + throw new IllegalArgumentException("Not Supported in Mysql Shard"); + } + + @Override + public void setLocalPoolSize(int localPoolSize) { + throw new IllegalArgumentException("Not Supported in Mysql Shard"); + } + + @Override + public void setRemotePoolSize(int remotePoolSize) { + throw new IllegalArgumentException("Not Supported in Mysql Shard"); + } + + @Override + public String toString() { + return "MySqlShard{" + + "logicalShardId='" + + logicalShardId + + '\'' + + ", host='" + + host + + '\'' + + ", port='" + + port + + '\'' + + ", user='" + + user + + '\'' + + ", dbName='" + + dbName + + '\'' + + ", namespace='" + + namespace + + '\'' + + ", connectionProperties='" + + connectionProperties + + '\'' + + ", dbNameToLogicalShardIdMap=" + + dbNameToLogicalShardIdMap + + '}'; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof MySqlShard)) { + return false; + } + MySqlShard mySqlShard = (MySqlShard) o; + return Objects.equals(logicalShardId, mySqlShard.logicalShardId) + && Objects.equals(host, mySqlShard.host) + && Objects.equals(port, mySqlShard.port) + && Objects.equals(user, mySqlShard.user) + && Objects.equals(password, mySqlShard.password) + && Objects.equals(dbName, mySqlShard.dbName) + && Objects.equals(namespace, mySqlShard.namespace) + && Objects.equals(connectionProperties, mySqlShard.connectionProperties) + && Objects.equals(secretManagerUri, mySqlShard.secretManagerUri) + && Objects.equals(dbNameToLogicalShardIdMap, mySqlShard.dbNameToLogicalShardIdMap); + } + + @Override + public int hashCode() { + return Objects.hash( + logicalShardId, + host, + port, + user, + password, + dbName, + namespace, + connectionProperties, + secretManagerUri, + dbNameToLogicalShardIdMap); + } +} diff --git a/v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/migrations/shard/Shard.java b/v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/migrations/shard/Shard.java deleted file mode 100644 index 9bfc70a9a9..0000000000 --- a/v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/migrations/shard/Shard.java +++ /dev/null @@ -1,200 +0,0 @@ -/* - * Copyright (C) 2023 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ -package com.google.cloud.teleport.v2.spanner.migrations.shard; - -import java.io.Serializable; -import java.util.HashMap; -import java.util.Map; -import java.util.Objects; - -public class Shard implements Serializable { - - private String logicalShardId; - private String host; - private String port; - private String user; - private String password; - private String dbName; - private String namespace; - private String secretManagerUri; - private String connectionProperties; - - private Map dbNameToLogicalShardIdMap = new HashMap<>(); - - public Shard( - String logicalShardId, - String host, - String port, - String user, - String password, - String dbName, - String namespace, - String secretManagerUri, - String connectionProperties) { - this.logicalShardId = logicalShardId; - this.host = host; - this.port = port; - this.user = user; - this.password = password; - this.dbName = dbName; - this.namespace = namespace; - this.secretManagerUri = secretManagerUri; - this.connectionProperties = connectionProperties; - } - - public Shard() {} - - public String getLogicalShardId() { - return logicalShardId; - } - - public void setLogicalShardId(String input) { - this.logicalShardId = input; - } - - public String getUserName() { - return user; - } - - public void setUser(String input) { - this.user = input; - } - - public String getPassword() { - return password; - } - - public void setPassword(String input) { - this.password = input; - } - - public String getHost() { - return host; - } - - public void setHost(String input) { - this.host = input; - } - - public String getPort() { - return port; - } - - public void setPort(String input) { - this.port = input; - } - - public String getDbName() { - return dbName; - } - - public void setDbName(String input) { - this.dbName = input; - } - - public String getNamespace() { - return namespace; - } - - public void setNamespace(String namespace) { - this.namespace = namespace; - } - - public String getSecretManagerUri() { - return secretManagerUri; - } - - public void setSecretManagerUri(String input) { - this.secretManagerUri = input; - } - - public String getConnectionProperties() { - return connectionProperties; - } - - public void setConnectionProperties(String input) { - this.connectionProperties = input; - } - - public Map getDbNameToLogicalShardIdMap() { - return dbNameToLogicalShardIdMap; - } - - @Override - public String toString() { - return "Shard{" - + "logicalShardId='" - + logicalShardId - + '\'' - + ", host='" - + host - + '\'' - + ", port='" - + port - + '\'' - + ", user='" - + user - + '\'' - + ", dbName='" - + dbName - + '\'' - + ", namespace='" - + namespace - + '\'' - + ", connectionProperties='" - + connectionProperties - + '\'' - + ", dbNameToLogicalShardIdMap=" - + dbNameToLogicalShardIdMap - + '}'; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (!(o instanceof Shard)) { - return false; - } - Shard shard = (Shard) o; - return Objects.equals(logicalShardId, shard.logicalShardId) - && Objects.equals(host, shard.host) - && Objects.equals(port, shard.port) - && Objects.equals(user, shard.user) - && Objects.equals(password, shard.password) - && Objects.equals(dbName, shard.dbName) - && Objects.equals(namespace, shard.namespace) - && Objects.equals(connectionProperties, shard.connectionProperties) - && Objects.equals(secretManagerUri, shard.secretManagerUri) - && Objects.equals(dbNameToLogicalShardIdMap, shard.dbNameToLogicalShardIdMap); - } - - @Override - public int hashCode() { - return Objects.hash( - logicalShardId, - host, - port, - user, - password, - dbName, - namespace, - connectionProperties, - secretManagerUri, - dbNameToLogicalShardIdMap); - } -} diff --git a/v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/migrations/utils/CassandraConfigFileReader.java b/v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/migrations/utils/CassandraConfigFileReader.java new file mode 100644 index 0000000000..4b0d8906d7 --- /dev/null +++ b/v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/migrations/utils/CassandraConfigFileReader.java @@ -0,0 +1,48 @@ +package com.google.cloud.teleport.v2.spanner.migrations.utils; + +import com.google.cloud.teleport.v2.spanner.migrations.shard.CassandraShard; +import com.google.cloud.teleport.v2.spanner.migrations.shard.IShard; +import com.google.gson.FieldNamingPolicy; +import com.google.gson.GsonBuilder; +import org.apache.beam.sdk.io.FileSystems; +import org.apache.commons.io.IOUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.channels.Channels; +import java.nio.charset.StandardCharsets; +import java.util.Collections; +import java.util.List; + +/** Class to read the Cassandra configuration file in GCS and convert it into a CassandraConfig object. */ +public class CassandraConfigFileReader { + + private static final Logger LOG = LoggerFactory.getLogger(CassandraConfigFileReader.class); + + public List getCassandraShard(String cassandraConfigFilePath) { + try (InputStream stream = + Channels.newInputStream( + FileSystems.open(FileSystems.matchNewResource(cassandraConfigFilePath, false)))) { + + String result = IOUtils.toString(stream, StandardCharsets.UTF_8); + IShard iShard = + new GsonBuilder() + .setFieldNamingPolicy(FieldNamingPolicy.IDENTITY) + .create() + .fromJson(result, CassandraShard.class); + + LOG.info("The Cassandra config is: {}", iShard); + return Collections.singletonList(iShard); + + } catch (IOException e) { + LOG.error( + "Failed to read Cassandra config file. Make sure it is ASCII or UTF-8 encoded and contains a well-formed JSON string.", + e); + throw new RuntimeException( + "Failed to read Cassandra config file. Make sure it is ASCII or UTF-8 encoded and contains a well-formed JSON string.", + e); + } + } +} \ No newline at end of file diff --git a/v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/migrations/utils/ShardFileReader.java b/v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/migrations/utils/ShardFileReader.java index 080d525101..c0f3833e78 100644 --- a/v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/migrations/utils/ShardFileReader.java +++ b/v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/migrations/utils/ShardFileReader.java @@ -15,7 +15,8 @@ */ package com.google.cloud.teleport.v2.spanner.migrations.utils; -import com.google.cloud.teleport.v2.spanner.migrations.shard.Shard; +import com.google.cloud.teleport.v2.spanner.migrations.shard.IShard; +import com.google.cloud.teleport.v2.spanner.migrations.shard.MySqlShard; import com.google.gson.FieldNamingPolicy; import com.google.gson.GsonBuilder; import com.google.gson.ToNumberPolicy; @@ -53,47 +54,47 @@ public ShardFileReader(ISecretManagerAccessor secretManagerAccessor) { this.secretManagerAccessor = secretManagerAccessor; } - public List getOrderedShardDetails(String sourceShardsFilePath) { + public List getOrderedShardDetails(String sourceShardsFilePath) { try (InputStream stream = Channels.newInputStream( FileSystems.open(FileSystems.matchNewResource(sourceShardsFilePath, false)))) { String result = IOUtils.toString(stream, StandardCharsets.UTF_8); - Type listOfShardObject = new TypeToken>() {}.getType(); - List shardList = + Type listOfShardObject = new TypeToken>() {}.getType(); + List mySqlShardList = new GsonBuilder() .setFieldNamingPolicy(FieldNamingPolicy.IDENTITY) .create() .fromJson(result, listOfShardObject); - for (Shard shard : shardList) { - LOG.info(" The shard is: {} ", shard); + for (IShard iShards : mySqlShardList) { + LOG.info(" The mySqlShard is: {} ", iShards); String password = resolvePassword( sourceShardsFilePath, - shard.getSecretManagerUri(), - shard.getLogicalShardId(), - shard.getPassword()); + iShards.getSecretManagerUri(), + iShards.getLogicalShardId(), + iShards.getPassword()); if (password == null || password.isEmpty()) { throw new RuntimeException( - "Neither password nor secretManagerUri was found in the shard file " + "Neither password nor secretManagerUri was found in the mySqlShard file " + sourceShardsFilePath - + " for shard " - + shard.getLogicalShardId()); + + " for mySqlShard " + + iShards.getLogicalShardId()); } - shard.setPassword(password); + iShards.setPassword(password); } Collections.sort( - shardList, - new Comparator() { - public int compare(Shard s1, Shard s2) { + mySqlShardList, + new Comparator() { + public int compare(IShard s1, IShard s2) { return s1.getLogicalShardId().compareTo(s2.getLogicalShardId()); } }); - return shardList; + return mySqlShardList; } catch (IOException e) { LOG.error( @@ -162,7 +163,7 @@ private String resolvePassword( * @param sourceShardsFilePath * @return */ - public List readForwardMigrationShardingConfig(String sourceShardsFilePath) { + public List readForwardMigrationShardingConfig(String sourceShardsFilePath) { String jsonString = null; try { InputStream stream = @@ -190,7 +191,7 @@ public List readForwardMigrationShardingConfig(String sourceShardsFilePat .create() .fromJson(jsonString, shardConfiguration); - List shardList = new ArrayList<>(); + List mySqlShardList = new ArrayList<>(); List dataShards = (List) (((Map) shardConfigMap.getOrDefault("shardConfigurationBulk", new HashMap<>())) @@ -214,7 +215,7 @@ public List readForwardMigrationShardingConfig(String sourceShardsFilePat if (password == null || password.isEmpty()) { LOG.warn("could not fetch password for host: {}", host); throw new RuntimeException( - "Neither password nor secretManagerUri was found in the shard file " + "Neither password nor secretManagerUri was found in the mySqlShard file " + sourceShardsFilePath + " for host " + host); @@ -222,8 +223,8 @@ public List readForwardMigrationShardingConfig(String sourceShardsFilePat String namespace = Optional.ofNullable(dataShard.get("namespace")).map(Object::toString).orElse(null); - Shard shard = - new Shard( + IShard mySqlShard = + new MySqlShard( "", host, dataShard.getOrDefault("port", 0).toString(), @@ -235,12 +236,12 @@ public List readForwardMigrationShardingConfig(String sourceShardsFilePat dataShard.getOrDefault("connectionProperties", "").toString()); for (Map database : databases) { - shard + mySqlShard .getDbNameToLogicalShardIdMap() .put((String) (database.get("dbName")), (String) (database.get("databaseId"))); } - shardList.add(shard); + mySqlShardList.add(mySqlShard); } - return shardList; + return mySqlShardList; } } diff --git a/v2/spanner-common/src/test/java/com/google/cloud/teleport/v2/spanner/migrations/avro/GenericRecordTypeConvertorTest.java b/v2/spanner-common/src/test/java/com/google/cloud/teleport/v2/spanner/migrations/avro/GenericRecordTypeConvertorTest.java index 43f2a068ad..cd3a2ae823 100644 --- a/v2/spanner-common/src/test/java/com/google/cloud/teleport/v2/spanner/migrations/avro/GenericRecordTypeConvertorTest.java +++ b/v2/spanner-common/src/test/java/com/google/cloud/teleport/v2/spanner/migrations/avro/GenericRecordTypeConvertorTest.java @@ -635,7 +635,7 @@ public void transformChangeEventTest_ShardIdPopulation() throws InvalidTransform // Null shard id case, shard id population should be skipped. genericRecordTypeConvertor = new GenericRecordTypeConvertor(shardedMapper, "", null, null); actual = genericRecordTypeConvertor.transformChangeEvent(genericRecord, "people"); - // Shard id should not be present. + // MySqlShard id should not be present. assertEquals(Map.of("new_name", Value.string("name1")), actual); } diff --git a/v2/spanner-common/src/test/java/com/google/cloud/teleport/v2/spanner/migrations/utils/ShardFileReaderTest.java b/v2/spanner-common/src/test/java/com/google/cloud/teleport/v2/spanner/migrations/utils/MySqlShardFileReaderTest.java similarity index 69% rename from v2/spanner-common/src/test/java/com/google/cloud/teleport/v2/spanner/migrations/utils/ShardFileReaderTest.java rename to v2/spanner-common/src/test/java/com/google/cloud/teleport/v2/spanner/migrations/utils/MySqlShardFileReaderTest.java index afe07bd695..e725341dfd 100644 --- a/v2/spanner-common/src/test/java/com/google/cloud/teleport/v2/spanner/migrations/utils/ShardFileReaderTest.java +++ b/v2/spanner-common/src/test/java/com/google/cloud/teleport/v2/spanner/migrations/utils/MySqlShardFileReaderTest.java @@ -21,7 +21,8 @@ import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.when; -import com.google.cloud.teleport.v2.spanner.migrations.shard.Shard; +import com.google.cloud.teleport.v2.spanner.migrations.shard.MySqlShard; + import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -34,21 +35,21 @@ import org.mockito.junit.MockitoRule; @RunWith(JUnit4.class) -public final class ShardFileReaderTest { +public final class MySqlShardFileReaderTest { @Rule public final MockitoRule mocktio = MockitoJUnit.rule(); @Mock private ISecretManagerAccessor secretManagerAccessorMockImpl; @Test public void shardFileReading() { ShardFileReader shardFileReader = new ShardFileReader(new SecretManagerAccessorImpl()); - List shards = shardFileReader.getOrderedShardDetails("src/test/resources/shard.json"); - List expectedShards = + List mySqlShards = shardFileReader.getOrderedShardDetails("src/test/resources/shard.json"); + List expectedMySqlShards = Arrays.asList( - new Shard( + new MySqlShard( "shardA", "hostShardA", "3306", "test", "test", "test", "namespaceA", null, null), - new Shard("shardB", "hostShardB", "3306", "test", "test", "test", null, null, null)); + new MySqlShard("shardB", "hostShardB", "3306", "test", "test", "test", null, null, null)); - assertEquals(shards, expectedShards); + assertEquals(mySqlShards, expectedMySqlShards); } @Test @@ -73,11 +74,11 @@ public void shardFileReadingWithSecret() { .thenReturn("secretC"); ShardFileReader shardFileReader = new ShardFileReader(secretManagerAccessorMockImpl); - List shards = + List mySqlShards = shardFileReader.getOrderedShardDetails("src/test/resources/shard-with-secret.json"); - List expectedShards = + List expectedMySqlShards = Arrays.asList( - new Shard( + new MySqlShard( "shardA", "hostShardA", "3306", @@ -87,7 +88,7 @@ public void shardFileReadingWithSecret() { "namespaceA", "projects/123/secrets/secretA/versions/latest", null), - new Shard( + new MySqlShard( "shardB", "hostShardB", "3306", @@ -97,7 +98,7 @@ public void shardFileReadingWithSecret() { null, "projects/123/secrets/secretB", null), - new Shard( + new MySqlShard( "shardC", "hostShardC", "3306", @@ -107,9 +108,9 @@ public void shardFileReadingWithSecret() { "namespaceC", "projects/123/secrets/secretC/", null), - new Shard("shardD", "hostShardD", "3306", "test", "test", "test", null, null, null)); + new MySqlShard("shardD", "hostShardD", "3306", "test", "test", "test", null, null, null)); - assertEquals(shards, expectedShards); + assertEquals(mySqlShards, expectedMySqlShards); } @Test @@ -147,11 +148,11 @@ public void readBulkMigrationShardFile() { String testConnectionProperties = "useUnicode=yes&characterEncoding=UTF-8&zeroDateTimeBehavior=convertToNull"; ShardFileReader shardFileReader = new ShardFileReader(new SecretManagerAccessorImpl()); - List shards = + List mySqlShards = shardFileReader.readForwardMigrationShardingConfig( - "src/test/resources/bulk-migration-shards.json"); - Shard shard1 = - new Shard( + "src/test/resources/bulk-migration-mySqlShards.json"); + MySqlShard mySqlShard1 = + new MySqlShard( "", "1.1.1.1", "3306", @@ -161,23 +162,23 @@ public void readBulkMigrationShardFile() { "namespace1", null, testConnectionProperties); - shard1.getDbNameToLogicalShardIdMap().put("person1", "1-1-1-1-person"); - shard1.getDbNameToLogicalShardIdMap().put("person2", "1-1-1-1-person2"); - Shard shard2 = new Shard("", "1.1.1.2", "3306", "test1", "pass1", "", null, null, ""); - shard2.getDbNameToLogicalShardIdMap().put("person1", "1-1-1-2-person"); - shard2.getDbNameToLogicalShardIdMap().put("person20", "1-1-1-2-person2"); - List expectedShards = new ArrayList<>(Arrays.asList(shard1, shard2)); - - assertEquals(expectedShards, shards); - assertEquals(shard1.toString().contains(testConnectionProperties), true); - assertEquals(shard1.getConnectionProperties(), testConnectionProperties); - var originalHarshCode = shard1.hashCode(); - shard1.setConnectionProperties(""); - assertNotEquals(originalHarshCode, shard1.hashCode()); + mySqlShard1.getDbNameToLogicalShardIdMap().put("person1", "1-1-1-1-person"); + mySqlShard1.getDbNameToLogicalShardIdMap().put("person2", "1-1-1-1-person2"); + MySqlShard mySqlShard2 = new MySqlShard("", "1.1.1.2", "3306", "test1", "pass1", "", null, null, ""); + mySqlShard2.getDbNameToLogicalShardIdMap().put("person1", "1-1-1-2-person"); + mySqlShard2.getDbNameToLogicalShardIdMap().put("person20", "1-1-1-2-person2"); + List expectedMySqlShards = new ArrayList<>(Arrays.asList(mySqlShard1, mySqlShard2)); + + assertEquals(expectedMySqlShards, mySqlShards); + assertEquals(mySqlShard1.toString().contains(testConnectionProperties), true); + assertEquals(mySqlShard1.getConnectionProperties(), testConnectionProperties); + var originalHarshCode = mySqlShard1.hashCode(); + mySqlShard1.setConnectionProperties(""); + assertNotEquals(originalHarshCode, mySqlShard1.hashCode()); // Cover the equality override. - assertEquals(shard1, shard1); - assertNotEquals(shard1, ""); - assertNotEquals(shard1, shards.get(0)); + assertEquals(mySqlShard1, mySqlShard1); + assertNotEquals(mySqlShard1, ""); + assertNotEquals(mySqlShard1, mySqlShards.get(0)); } @Test @@ -187,11 +188,11 @@ public void readBulkMigrationShardFileWithSecrets() { when(secretManagerAccessorMockImpl.getSecret("projects/123/secrets/secretB/versions/latest")) .thenReturn("secretB"); ShardFileReader shardFileReader = new ShardFileReader(secretManagerAccessorMockImpl); - List shards = + List mySqlShards = shardFileReader.readForwardMigrationShardingConfig( - "src/test/resources/bulk-migration-shards-secret.json"); - Shard shard1 = - new Shard( + "src/test/resources/bulk-migration-mySqlShards-secret.json"); + MySqlShard mySqlShard1 = + new MySqlShard( "", "1.1.1.1", "3306", @@ -201,10 +202,10 @@ public void readBulkMigrationShardFileWithSecrets() { null, "projects/123/secrets/secretA/versions/latest", ""); - shard1.getDbNameToLogicalShardIdMap().put("person1", "1-1-1-1-person"); - shard1.getDbNameToLogicalShardIdMap().put("person2", "1-1-1-1-person2"); - Shard shard2 = - new Shard( + mySqlShard1.getDbNameToLogicalShardIdMap().put("person1", "1-1-1-1-person"); + mySqlShard1.getDbNameToLogicalShardIdMap().put("person2", "1-1-1-1-person2"); + MySqlShard mySqlShard2 = + new MySqlShard( "", "1.1.1.2", "3306", @@ -214,10 +215,10 @@ public void readBulkMigrationShardFileWithSecrets() { null, "projects/123/secrets/secretB/versions/latest", ""); - shard2.getDbNameToLogicalShardIdMap().put("person1", "1-1-1-2-person"); - shard2.getDbNameToLogicalShardIdMap().put("person20", "1-1-1-2-person2"); - List expectedShards = new ArrayList<>(Arrays.asList(shard1, shard2)); + mySqlShard2.getDbNameToLogicalShardIdMap().put("person1", "1-1-1-2-person"); + mySqlShard2.getDbNameToLogicalShardIdMap().put("person20", "1-1-1-2-person2"); + List expectedMySqlShards = new ArrayList<>(Arrays.asList(mySqlShard1, mySqlShard2)); - assertEquals(shards, expectedShards); + assertEquals(mySqlShards, expectedMySqlShards); } } diff --git a/v2/spanner-common/terraform/samples/mysql-vpc-spanner-test-infra/README.md b/v2/spanner-common/terraform/samples/mysql-vpc-spanner-test-infra/README.md index 4ca84b11b5..e42c786ebd 100644 --- a/v2/spanner-common/terraform/samples/mysql-vpc-spanner-test-infra/README.md +++ b/v2/spanner-common/terraform/samples/mysql-vpc-spanner-test-infra/README.md @@ -87,7 +87,7 @@ configuration and creates the following resources - 3. **Firewall rules** - Rules to allow Dataflow VMs to communicate with each other and Datastream to connect to the MySQL instance via private connectivity. -4. **GCE VMs with MySQL shards** - Launches GCE VMs with MySQL setup on it inside +4. **GCE VMs with MySQL mySqlShards** - Launches GCE VMs with MySQL setup on it inside the specified VPC subnet. 5. **Spanner instance** - A spanner instance with the specified configuration. 6. **Spanner database** - A spanner database inside the instance created. @@ -154,8 +154,8 @@ resource_ids = { } resource_urls = { "mysql_instances" = { - "mysql-shard1" = "https://console.cloud.google.com/compute/instancesDetail/zones/us-central1-a/instances/mysql-shard1?project=" - "mysql-shard2" = "https://console.cloud.google.com/compute/instancesDetail/zones/us-central1-a/instances/mysql-shard2?project=" + "mysql-mySqlShard1" = "https://console.cloud.google.com/compute/instancesDetail/zones/us-central1-a/instances/mysql-mySqlShard1?project=" + "mysql-mySqlShard2" = "https://console.cloud.google.com/compute/instancesDetail/zones/us-central1-a/instances/mysql-mySqlShard2?project=" "mysql-shard3" = "https://console.cloud.google.com/compute/instancesDetail/zones/us-central1-a/instances/mysql-shard3?project=" } "network" = "https://console.cloud.google.com/networking/networks/details/sample-vpc-network?project=" diff --git a/v2/spanner-common/terraform/samples/mysql-vpc-spanner-test-infra/variables.tf b/v2/spanner-common/terraform/samples/mysql-vpc-spanner-test-infra/variables.tf index faba775e55..0f2d67a7ac 100644 --- a/v2/spanner-common/terraform/samples/mysql-vpc-spanner-test-infra/variables.tf +++ b/v2/spanner-common/terraform/samples/mysql-vpc-spanner-test-infra/variables.tf @@ -16,7 +16,7 @@ variable "vpc_params" { } variable "mysql_params" { - description = "Parameters for MySQL shards source configuration" + description = "Parameters for MySQL mySqlShards source configuration" type = list(object({ vm_name = string machine_type = optional(string, "n2-standard-2") diff --git a/v2/spanner-custom-shard/pom.xml b/v2/spanner-custom-shard/pom.xml index ecf3d0fd84..c2dd865c95 100644 --- a/v2/spanner-custom-shard/pom.xml +++ b/v2/spanner-custom-shard/pom.xml @@ -25,7 +25,7 @@ 1.0-SNAPSHOT - spanner-custom-shard + spanner-custom-mySqlShard com.google.cloud.teleport.v2 diff --git a/v2/spanner-to-sourcedb/README.md b/v2/spanner-to-sourcedb/README.md index abf8a4889a..c87b351479 100644 --- a/v2/spanner-to-sourcedb/README.md +++ b/v2/spanner-to-sourcedb/README.md @@ -34,7 +34,7 @@ These steps are achieved by the Spanner to SourceDb dataflow template. #### Consistency guarantees -The pipeline guarantees consistency at a primary key level. The pipeline creates shadow tables in Cloud Spanner, which contain the Spanner commit timestamp of the latest record that was successfully written on the shard for that table. The writes are guaranteed to be consistent up-to this commit timestamp for the given primary key. +The pipeline guarantees consistency at a primary key level. The pipeline creates shadow tables in Cloud Spanner, which contain the Spanner commit timestamp of the latest record that was successfully written on the mySqlShard for that table. The writes are guaranteed to be consistent up-to this commit timestamp for the given primary key. However, there is no order maintained across tables or even across various primary keys in the same table. This helps achieve faster replication. @@ -67,9 +67,9 @@ A few prerequisites must be considered before starting with reverse replication. 1. Ensure network connectivity between the source database and your GCP project, where your Dataflow jobs will run. - Allowlist Dataflow worker IPs on the MySQL instance so that they can access the MySQL IPs. - - Check that the MySQL credentials are correctly specified in the [source shards file](#sample-source-shards-file). + - Check that the MySQL credentials are correctly specified in the [source mySqlShards file](#sample-source-mySqlShards-file). - Check that the MySQL server is up. - - The MySQL user configured in the [source shards file](#sample-source-shards-file) should have [INSERT](https://dev.mysql.com/doc/refman/8.0/en/privileges-provided.html#priv_insert), [UPDATE](https://dev.mysql.com/doc/refman/8.0/en/privileges-provided.html#priv_update) and [DELETE](https://dev.mysql.com/doc/refman/8.0/en/privileges-provided.html#priv_delete) privileges on the database. + - The MySQL user configured in the [source mySqlShards file](#sample-source-mySqlShards-file) should have [INSERT](https://dev.mysql.com/doc/refman/8.0/en/privileges-provided.html#priv_insert), [UPDATE](https://dev.mysql.com/doc/refman/8.0/en/privileges-provided.html#priv_update) and [DELETE](https://dev.mysql.com/doc/refman/8.0/en/privileges-provided.html#priv_delete) privileges on the database. 2. Ensure that Dataflow permissions are present.[Basic permissions](https://cloud.google.com/dataflow/docs/guides/templates/using-flex-templates#before_you_begin:~:text=Grant%20roles%20to%20your%20Compute%20Engine%20default%20service%20account.%20Run%20the%20following%20command%20once%20for%20each%20of%20the%20following%20IAM%20roles%3A%20roles/dataflow.admin%2C%20roles/dataflow.worker%2C%20roles/bigquery.dataEditor%2C%20roles/pubsub.editor%2C%20roles/storage.objectAdmin%2C%20and%20roles/artifactregistry.reader) and [Flex template permissions](https://cloud.google.com/dataflow/docs/guides/templates/configuring-flex-templates#permissions). 3. Ensure that the port 12345 is open for communication among the Dataflow worker VMs.Please refer the Dataflow firewall [documentation](https://cloud.google.com/dataflow/docs/guides/routes-firewall#firewall_rules) for more. 4. Ensure the compute engine service account has the following permission: @@ -82,9 +82,9 @@ A few prerequisites must be considered before starting with reverse replication. 6. Ensure that gcloud authentication is done,refer [here](https://cloud.google.com/spanner/docs/getting-started/set-up#set_up_authentication_and_authorization). 7. Ensure that the target Spanner instance is ready. 8. Ensure that that [session file](https://googlecloudplatform.github.io/spanner-migration-tool/reports.html#session-file-ending-in-sessionjson) is uploaded to GCS (this requires a schema conversion to be done). -9. [Source shards file](./RunnigReverseReplication.md#sample-sourceshards-file) already uploaded to GCS. +9. [Source mySqlShards file](./RunnigReverseReplication.md#sample-sourceshards-file) already uploaded to GCS. 10. Resources needed for reverse replication incur cost. Make sure to read [cost](#cost). -11. Reverse replication uses shard identifier column per table to route the Spanner records to a given source shard.The column identified as the sharding column needs to be selected via Spanner Migration Tool when performing migration.The value of this column should be the logicalShardId value specified in the [source shard file](#sample-source-shards-file).In the event that the shard identifier column is not an existing column,the application code needs to be changed to populate this shard identifier column when writing to Spanner. Or use a custom shard identifier plugin to supply the shard identifier. +11. Reverse replication uses mySqlShard identifier column per table to route the Spanner records to a given source mySqlShard.The column identified as the sharding column needs to be selected via Spanner Migration Tool when performing migration.The value of this column should be the logicalShardId value specified in the [source mySqlShard file](#sample-source-mySqlShards-file).In the event that the mySqlShard identifier column is not an existing column,the application code needs to be changed to populate this mySqlShard identifier column when writing to Spanner. Or use a custom mySqlShard identifier plugin to supply the mySqlShard identifier. 12. The reverse replication pipeline uses GCS for dead letter queue handling. Ensure that the DLQ directory exists in GCS. 13. Create PubSub notification on the 'retry' folder of the DLQ directory. For this, create a [PubSub topic](https://cloud.google.com/pubsub/docs/create-topic), create a [PubSub subscription](https://cloud.google.com/pubsub/docs/create-subscription) for that topic. Configure [GCS notification](https://cloud.google.com/storage/docs/reporting-changes#command-line). The resulting subscription should be supplied as the dlqGcsPubSubSubscription Dataflow input parameter. @@ -126,27 +126,27 @@ A few prerequisites must be considered before starting with reverse replication. --subnetwork=https://www.googleapis.com/compute/v1/projects//regions//subnetworks/ ``` -### Sample source shards File +### Sample source mySqlShards File -This file contains meta data regarding the source MYSQL shards, which is used to connect to them. This should be present even if there is a single source database shard. +This file contains meta data regarding the source MYSQL mySqlShards, which is used to connect to them. This should be present even if there is a single source database mySqlShard. The database user password should be kept in [Secret Manager](#https://cloud.google.com/security/products/secret-manager) and it's URI needs to be specified in the file. The file should be a list of JSONs as: ```json [ { - "logicalShardId": "shard1", + "logicalShardId": "mySqlShard1", "host": "10.11.12.13", "user": "root", - "secretManagerUri":"projects/123/secrets/rev-cmek-cred-shard1/versions/latest", + "secretManagerUri":"projects/123/secrets/rev-cmek-cred-mySqlShard1/versions/latest", "port": "3306", "dbName": "db1" }, { - "logicalShardId": "shard2", + "logicalShardId": "mySqlShard2", "host": "10.11.12.14", "user": "root", - "secretManagerUri":"projects/123/secrets/rev-cmek-cred-shard2/versions/latest", + "secretManagerUri":"projects/123/secrets/rev-cmek-cred-mySqlShard2/versions/latest", "port": "3306", "dbName": "db2" } @@ -179,12 +179,12 @@ In addition, there are following application metrics exposed by the job: | Metric Name | Description | |---------------------------------------|----------------------------------------------------------------------------------------------------------------------------------| -| custom_shard_id_impl_latency_ms | Time taken for the execution of custom shard identifier logic. | +| custom_shard_id_impl_latency_ms | Time taken for the execution of custom mySqlShard identifier logic. | | data_record_count | The number of change stream records read. | | element_requeued_for_retry_count | Relevant for retryDLQ run mode, when the record gets enqueded back to severe folder for retry. | | elementsReconsumedFromDeadLetterQueue | The number of records read from the retry folder of DLQ directory. | -| records_written_to_source_\ | Number of records successfully written for the shard. | -| replication_lag_in_seconds_\| Replication lag min,max and count value for the shard.| +| records_written_to_source_\ | Number of records successfully written for the mySqlShard. | +| replication_lag_in_seconds_\| Replication lag min,max and count value for the mySqlShard.| | retryable_record_count | The number of records that are up for retry. | | severe_error_count | The number of permanent errors. | | skipped_record_count | The count of records that were skipped from reverse replication. | @@ -243,10 +243,10 @@ In this case, check if you observe the following: Records of below nature are dropped from reverse replication. Check the Dataflow logs to see if they are dropped: 1. Records which are forward migrated. -2. Shard Id based routing could not be performed since the shard id value could not be determined. +2. Shard Id based routing could not be performed since the mySqlShard id value could not be determined. 3. The record was deleted on Cloud Spanner and the deleted record was removed from Cloud Spanner due to lapse of retention period by the time the record was to be reverse replicated. 4. Check for issues in the dataflow job. This can include scaling issues, CPU utilization being more than 70% consistently. This can be checked via [CPU utilization](https://cloud.google.com/dataflow/docs/guides/using-monitoring-intf#cpu-use) section on the Dataflow job UI.Check for any errors in the jobor worker logs which could indicate restarts. Sometimes a worker might restart causing a delay in record processing. The CPU utilization would show multiple workers during the restart period. The number of workers could also be viewed via [here](https://cloud.google.com/dataflow/docs/guides/using-monitoring-intf#autoscaling). -5. When working with session file based shard identification logic, if the table of the change record does not exist in the session file, such records are written to skip directory and not reverse replicated. +5. When working with session file based mySqlShard identification logic, if the table of the change record does not exist in the session file, such records are written to skip directory and not reverse replicated. Check the Dataflow logs to see if records are being dropped. This can happen for records for which primary key cannot be determined on the source database. This can happen when: @@ -340,7 +340,7 @@ To check if there are worker restarts - in the Dataflow UI, navigate to the Job 1. Count the number of rows in Spanner and source databae - this may take long, but is it the only definitive way. If the reverse replication is still going on - the counts would not match due to replication lag - and an acceptable RPO should be considered to cut-back. -2. If the count of records is not possible - check the DataWatermark of the Write to SourceDb stage. This gives a rough estimate of the lag between Spanner and Source. Consider taking some buffer - say 5 minutes and add this to the lag. If this lag is acceptable - perform the cutback else wait for the pipeline to catchup. In addition to the DataWatermark, also check the DataFreshness and the mean replication lag for the shards, and once it is under acceptable RPO, cut-back can be done. Querying these metrics is listed in the [metrics](#metrics-for-dataflow-job) section. +2. If the count of records is not possible - check the DataWatermark of the Write to SourceDb stage. This gives a rough estimate of the lag between Spanner and Source. Consider taking some buffer - say 5 minutes and add this to the lag. If this lag is acceptable - perform the cutback else wait for the pipeline to catchup. In addition to the DataWatermark, also check the DataFreshness and the mean replication lag for the mySqlShards, and once it is under acceptable RPO, cut-back can be done. Querying these metrics is listed in the [metrics](#metrics-for-dataflow-job) section. Also, [alerts](https://cloud.google.com/monitoring/alerts) can be set up when the replciation lag, DataFreshness or DataWatermarks cross a threshold to take debugging actions. ### What to do in case of pause and resume scenarios @@ -383,7 +383,7 @@ The Dataflow jobs can be customized. Some use cases could be: 1. To customize the logic to filter records from reverse replication. 2. To handle some custom reverse transformation scenarios. -3. To customize shard level routing. +3. To customize mySqlShard level routing. To customize, checkout the open source template, add the custom logic, build and launch the open source template. @@ -392,13 +392,13 @@ Refer to [Spanner to SourceDb template](https://github.com/GoogleCloudPlatform/D ### Shard routing customization -In order to make it easier for users to customize the shard routing logic, the template accepts a GCS path that points to a custom jar and another input parameter that accepts the custom class name, which are used to invoke custom logic to perform shard identification. +In order to make it easier for users to customize the mySqlShard routing logic, the template accepts a GCS path that points to a custom jar and another input parameter that accepts the custom class name, which are used to invoke custom logic to perform mySqlShard identification. Steps to perfrom customization: -1. Write custom shard id fetcher logic [CustomShardIdFetcher.java](https://github.com/GoogleCloudPlatform/DataflowTemplates/blob/main/v2/spanner-custom-shard/src/main/java/com/custom/CustomShardIdFetcher.java). Details of the ShardIdRequest class can be found [here](https://github.com/GoogleCloudPlatform/DataflowTemplates/blob/main/v2/spanner-migrations-sdk/src/main/java/com/google/cloud/teleport/v2/spanner/utils/ShardIdRequest.java). -2. Build the [JAR](https://github.com/GoogleCloudPlatform/DataflowTemplates/tree/main/v2/spanner-custom-shard) and upload the jar to GCS +1. Write custom mySqlShard id fetcher logic [CustomShardIdFetcher.java](https://github.com/GoogleCloudPlatform/DataflowTemplates/blob/main/v2/spanner-custom-mySqlShard/src/main/java/com/custom/CustomShardIdFetcher.java). Details of the ShardIdRequest class can be found [here](https://github.com/GoogleCloudPlatform/DataflowTemplates/blob/main/v2/spanner-migrations-sdk/src/main/java/com/google/cloud/teleport/v2/spanner/utils/ShardIdRequest.java). +2. Build the [JAR](https://github.com/GoogleCloudPlatform/DataflowTemplates/tree/main/v2/spanner-custom-mySqlShard) and upload the jar to GCS 3. Invoke the reverse replication flow by passing the custom jar path and custom class path. -4. If any custom parameters are needed in the custom shard identification logic, they can be passed via the *shardingCustomParameters* input to the runner. These parameters will be passed to the *init* method of the custom class. The *init* method is invoked once per worker setup. +4. If any custom parameters are needed in the custom mySqlShard identification logic, they can be passed via the *shardingCustomParameters* input to the runner. These parameters will be passed to the *init* method of the custom class. The *init* method is invoked once per worker setup. ## Cost diff --git a/v2/spanner-to-sourcedb/README_Spanner_to_SourceDb.md b/v2/spanner-to-sourcedb/README_Spanner_to_SourceDb.md index f84df5c26c..1d7dd9a345 100644 --- a/v2/spanner-to-sourcedb/README_Spanner_to_SourceDb.md +++ b/v2/spanner-to-sourcedb/README_Spanner_to_SourceDb.md @@ -20,7 +20,7 @@ on [Metadata Annotations](https://github.com/GoogleCloudPlatform/DataflowTemplat * **spannerProjectId** : This is the name of the Cloud Spanner project. * **metadataInstance** : This is the instance to store the metadata used by the connector to control the consumption of the change stream API data. * **metadataDatabase** : This is the database to store the metadata used by the connector to control the consumption of the change stream API data. -* **sourceShardsFilePath** : Path to GCS file containing connection profile info for source shards. +* **sourceShardsFilePath** : Path to GCS file containing connection profile info for source mySqlShards. ### Optional parameters @@ -29,13 +29,13 @@ on [Metadata Annotations](https://github.com/GoogleCloudPlatform/DataflowTemplat * **shadowTablePrefix** : The prefix used to name shadow tables. Default: `shadow_`. * **sessionFilePath** : Session file path in Cloud Storage that contains mapping information from HarbourBridge. * **filtrationMode** : Mode of Filtration, decides how to drop certain records based on a criteria. Currently supported modes are: none (filter nothing), forward_migration (filter records written via the forward migration pipeline). Defaults to forward_migration. -* **shardingCustomJarPath** : Custom jar location in Cloud Storage that contains the customization logic for fetching shard id. Defaults to empty. -* **shardingCustomClassName** : Fully qualified class name having the custom shard id implementation. It is a mandatory field in case shardingCustomJarPath is specified. Defaults to empty. +* **shardingCustomJarPath** : Custom jar location in Cloud Storage that contains the customization logic for fetching mySqlShard id. Defaults to empty. +* **shardingCustomClassName** : Fully qualified class name having the custom mySqlShard id implementation. It is a mandatory field in case shardingCustomJarPath is specified. Defaults to empty. * **shardingCustomParameters** : String containing any custom parameters to be passed to the custom sharding class. Defaults to empty. * **sourceDbTimezoneOffset** : This is the timezone offset from UTC for the source database. Example value: +10:00. Defaults to: +00:00. * **dlqGcsPubSubSubscription** : The Pub/Sub subscription being used in a Cloud Storage notification policy for DLQ retry directory when running in regular mode. The name should be in the format of projects//subscriptions/. When set, the deadLetterQueueDirectory and dlqRetryMinutes are ignored. * **skipDirectoryName** : Records skipped from reverse replication are written to this directory. Default directory name is skip. -* **maxShardConnections** : This will come from shard file eventually. Defaults to: 10000. +* **maxShardConnections** : This will come from mySqlShard file eventually. Defaults to: 10000. * **deadLetterQueueDirectory** : The file path used when storing the error queue output. The default file path is a directory under the Dataflow job's temp location. * **dlqMaxRetryCount** : The max number of times temporary errors can be retried through DLQ. Defaults to 500. * **runMode** : This is the run mode type, whether regular or with retryDLQ.Default is regular. retryDLQ is used to retry the severe DLQ records only. diff --git a/v2/spanner-to-sourcedb/pom.xml b/v2/spanner-to-sourcedb/pom.xml index 841405fb9a..add3fa6277 100644 --- a/v2/spanner-to-sourcedb/pom.xml +++ b/v2/spanner-to-sourcedb/pom.xml @@ -82,6 +82,11 @@ beam-it-jdbc test + + com.datastax.oss + java-driver-core + 4.17.0 + com.google.cloud.teleport @@ -89,6 +94,18 @@ ${project.version} test + + com.datastax.oss + java-driver-core + 4.16.0 + compile + + + com.datastax.cassandra + cassandra-driver-core + 3.10.2 + compile + diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/SpannerToSourceDb.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/SpannerToSourceDb.java index e2cfa9a009..aeb7dc950d 100644 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/SpannerToSourceDb.java +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/SpannerToSourceDb.java @@ -27,27 +27,19 @@ import com.google.cloud.teleport.v2.common.UncaughtExceptionLogger; import com.google.cloud.teleport.v2.spanner.ddl.Ddl; import com.google.cloud.teleport.v2.spanner.migrations.schema.Schema; -import com.google.cloud.teleport.v2.spanner.migrations.shard.Shard; +import com.google.cloud.teleport.v2.spanner.migrations.shard.IShard; import com.google.cloud.teleport.v2.spanner.migrations.spanner.SpannerSchema; +import com.google.cloud.teleport.v2.spanner.migrations.utils.CassandraConfigFileReader; import com.google.cloud.teleport.v2.spanner.migrations.utils.SecretManagerAccessorImpl; import com.google.cloud.teleport.v2.spanner.migrations.utils.SessionFileReader; import com.google.cloud.teleport.v2.spanner.migrations.utils.ShardFileReader; import com.google.cloud.teleport.v2.templates.SpannerToSourceDb.Options; import com.google.cloud.teleport.v2.templates.changestream.TrimmedShardedDataChangeRecord; import com.google.cloud.teleport.v2.templates.constants.Constants; -import com.google.cloud.teleport.v2.templates.transforms.AssignShardIdFn; -import com.google.cloud.teleport.v2.templates.transforms.ConvertChangeStreamErrorRecordToFailsafeElementFn; -import com.google.cloud.teleport.v2.templates.transforms.ConvertDlqRecordToTrimmedShardedDataChangeRecordFn; -import com.google.cloud.teleport.v2.templates.transforms.FilterRecordsFn; -import com.google.cloud.teleport.v2.templates.transforms.PreprocessRecordsFn; -import com.google.cloud.teleport.v2.templates.transforms.SourceWriterTransform; -import com.google.cloud.teleport.v2.templates.transforms.UpdateDlqMetricsFn; +import com.google.cloud.teleport.v2.templates.transforms.*; import com.google.cloud.teleport.v2.templates.utils.ShadowTableCreator; import com.google.cloud.teleport.v2.transforms.DLQWriteTransform; import com.google.cloud.teleport.v2.values.FailsafeElement; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; import org.apache.beam.runners.dataflow.options.DataflowPipelineDebugOptions; import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions; import org.apache.beam.runners.dataflow.options.DataflowPipelineWorkerPoolOptions; @@ -63,11 +55,7 @@ import org.apache.beam.sdk.io.gcp.spanner.SpannerAccessor; import org.apache.beam.sdk.io.gcp.spanner.SpannerConfig; import org.apache.beam.sdk.io.gcp.spanner.SpannerIO; -import org.apache.beam.sdk.options.Default; -import org.apache.beam.sdk.options.PipelineOptions; -import org.apache.beam.sdk.options.PipelineOptionsFactory; -import org.apache.beam.sdk.options.StreamingOptions; -import org.apache.beam.sdk.options.ValueProvider; +import org.apache.beam.sdk.options.*; import org.apache.beam.sdk.transforms.Flatten; import org.apache.beam.sdk.transforms.MapElements; import org.apache.beam.sdk.transforms.ParDo; @@ -78,6 +66,10 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + /** This pipeline reads Spanner Change streams data and writes them to a source DB. */ @Template( name = "Spanner_to_SourceDb", @@ -367,6 +359,15 @@ public interface Options extends PipelineOptions, StreamingOptions { String getSourceType(); void setSourceType(String value); + + @TemplateParameter.GcsReadFile( + order = 10, + optional = false, + description = "Path to GCS file containing the the Cassandra Config details", + helpText = "Path to GCS file containing connection profile info for cassandra.") + String getCassandraConfigFilePath(); + + void setCassandraConfigFilePath(String value); } /** @@ -405,7 +406,13 @@ public static PipelineResult run(Options options) { pipeline.getOptions().as(DataflowPipelineWorkerPoolOptions.class).getMaxNumWorkers() > 0 ? pipeline.getOptions().as(DataflowPipelineWorkerPoolOptions.class).getMaxNumWorkers() : 1; - int connectionPoolSizePerWorker = (int) (options.getMaxShardConnections() / maxNumWorkers); + int connectionPoolSizePerWorker = 1; + if ("mysql".equals(options.getSourceType())) { + connectionPoolSizePerWorker = (int) (options.getMaxShardConnections() / maxNumWorkers); + } else { + connectionPoolSizePerWorker = (int) (options.getMaxShardConnections() / maxNumWorkers); + } + if (connectionPoolSizePerWorker < 1) { // This can happen when the number of workers is more than max. // This can cause overload on the source database. Error out and let the user know. @@ -463,17 +470,38 @@ public static PipelineResult run(Options options) { shadowTableCreator.createShadowTablesInSpanner(); Ddl ddl = SpannerSchema.getInformationSchemaAsDdl(spannerConfig); - ShardFileReader shardFileReader = new ShardFileReader(new SecretManagerAccessorImpl()); - List shards = shardFileReader.getOrderedShardDetails(options.getSourceShardsFilePath()); - String shardingMode = Constants.SHARDING_MODE_MULTI_SHARD; - if (shards.size() == 1) { - shardingMode = Constants.SHARDING_MODE_SINGLE_SHARD; - - Shard singleShard = shards.get(0); - if (singleShard.getLogicalShardId() == null) { - singleShard.setLogicalShardId(Constants.DEFAULT_SHARD_ID); - LOG.info( - "Logical shard id was not found, hence setting it to : " + Constants.DEFAULT_SHARD_ID); + + List iShards = new ArrayList<>(); + String shardingMode = Constants.SHARDING_MODE_SINGLE_SHARD; + + if ("mysql".equals(options.getSourceType())) { + ShardFileReader shardFileReader = new ShardFileReader(new SecretManagerAccessorImpl()); + iShards = shardFileReader.getOrderedShardDetails(options.getSourceShardsFilePath()); + shardingMode = Constants.SHARDING_MODE_MULTI_SHARD; + if (iShards.size() == 1) { + shardingMode = Constants.SHARDING_MODE_SINGLE_SHARD; + + IShard singleMySqlShard = iShards.get(0); + if (singleMySqlShard.getLogicalShardId() == null) { + singleMySqlShard.setLogicalShardId(Constants.DEFAULT_SHARD_ID); + LOG.info( + "Logical shard id was not found, hence setting it to : " + Constants.DEFAULT_SHARD_ID); + } + } + } else { + CassandraConfigFileReader cassandraConfigFileReader = new CassandraConfigFileReader(); + iShards = cassandraConfigFileReader.getCassandraShard(options.getCassandraConfigFilePath()); + LOG.info("Cassandra config is: {}", iShards.get(0)); + if (iShards.size() == 1) { + shardingMode = Constants.SHARDING_MODE_SINGLE_SHARD; + IShard singleCassandraShard = iShards.get(0); + if (singleCassandraShard.getLogicalShardId() == null) { + singleCassandraShard.setLogicalShardId(Constants.DEFAULT_SHARD_ID); + LOG.info( + "Logical shard id was not found, hence setting it to : " + Constants.DEFAULT_SHARD_ID); + } + }else{ + throw new IllegalArgumentException("Not Supporting more than one shard for cassandra"); } } boolean isRegularMode = "regular".equals(options.getRunMode()); @@ -555,13 +583,16 @@ public static PipelineResult run(Options options) { schema, ddl, shardingMode, - shards.get(0).getLogicalShardId(), + iShards.get(0).getLogicalShardId(), options.getSkipDirectoryName(), options.getShardingCustomJarPath(), options.getShardingCustomClassName(), options.getShardingCustomParameters(), options.getMaxShardConnections() - * shards.size()))) // currently assuming that all shards accept the same + * iShards.size(), + options.getSourceType(), + options.getMaxShardConnections() + ))) // currently assuming that all mySqlShards accept the same // number of max connections .setCoder( KvCoder.of( @@ -570,15 +601,17 @@ public static PipelineResult run(Options options) { .apply( "Write to source", new SourceWriterTransform( - shards, - schema, - spannerMetadataConfig, - options.getSourceDbTimezoneOffset(), - ddl, - options.getShadowTablePrefix(), - options.getSkipDirectoryName(), - connectionPoolSizePerWorker, - options.getSourceType())); + iShards, + schema, + spannerMetadataConfig, + options.getSourceDbTimezoneOffset(), + ddl, + options.getShadowTablePrefix(), + options.getSkipDirectoryName(), + connectionPoolSizePerWorker, + options.getSourceType(), + options.getMaxShardConnections() + )); PCollection> dlqPermErrorRecords = reconsumedElements diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/constants/Constants.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/constants/Constants.java index 31ed2b8428..7f7e1a9333 100644 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/constants/Constants.java +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/constants/Constants.java @@ -72,4 +72,5 @@ public class Constants { public static final String DEFAULT_SHARD_ID = "single_shard"; public static final String SOURCE_MYSQL = "mysql"; + public static final String SOURCE_CASSANDRA = "cassandra"; } diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/constants/package-info.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/constants/package-info.java index 048d1a51b6..1f0ad4724e 100644 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/constants/package-info.java +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/constants/package-info.java @@ -1,5 +1,5 @@ /* - * Copyright (C) 2023 Google LLC + * Copyright (C) 2024 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dbutils/connection/CassandraConnectionHelper.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dbutils/connection/CassandraConnectionHelper.java new file mode 100644 index 0000000000..32b41e12b4 --- /dev/null +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dbutils/connection/CassandraConnectionHelper.java @@ -0,0 +1,83 @@ +package com.google.cloud.teleport.v2.templates.dbutils.connection; + +import com.datastax.oss.driver.api.core.CqlSession; +import com.datastax.oss.driver.api.core.CqlSessionBuilder; +import com.datastax.oss.driver.api.core.config.DriverConfigLoader; +import com.datastax.oss.driver.api.core.config.DriverOption; +import com.datastax.oss.driver.api.core.config.ProgrammaticDriverConfigLoaderBuilder; +import com.datastax.oss.driver.api.core.config.TypedDriverOption; +import com.google.cloud.teleport.v2.spanner.migrations.shard.CassandraShard; +import com.google.cloud.teleport.v2.spanner.migrations.shard.IShard; +import com.google.cloud.teleport.v2.templates.exceptions.ConnectionException; +import com.google.cloud.teleport.v2.templates.models.ConnectionHelperRequest; +import java.net.InetSocketAddress; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class CassandraConnectionHelper implements IConnectionHelper { + + private static final Logger LOG = LoggerFactory.getLogger(CassandraConnectionHelper.class); + private static Map connectionPoolMap = null; + + @Override + public synchronized void init(ConnectionHelperRequest connectionHelperRequest) { + if (connectionPoolMap != null) { + return; + } + LOG.info("Initializing Cassandra connection pool with size: {}", connectionHelperRequest.getMaxConnections()); + connectionPoolMap = new HashMap<>(); + List iShards= connectionHelperRequest.getShards(); + + for(IShard ishard: iShards) { + CassandraShard cassandraShard = (CassandraShard) ishard; + cassandraShard.validate(); + + CqlSessionBuilder builder = CqlSession.builder() + .addContactPoint(new InetSocketAddress(cassandraShard.getHost(), Integer.parseInt(cassandraShard.getPort()))) + .withAuthCredentials(cassandraShard.getUser(), cassandraShard.getPassword()) + .withKeyspace(cassandraShard.getKeySpaceName()); + + ProgrammaticDriverConfigLoaderBuilder configLoaderBuilder = DriverConfigLoader.programmaticBuilder(); + configLoaderBuilder.withInt((DriverOption) TypedDriverOption.CONNECTION_POOL_LOCAL_SIZE, cassandraShard.getLocalPoolSize()); + configLoaderBuilder.withInt((DriverOption) TypedDriverOption.CONNECTION_POOL_REMOTE_SIZE, cassandraShard.getRemotePoolSize()); + builder.withConfigLoader(configLoaderBuilder.build()); + + CqlSession session = builder.build(); + String connectionKey = cassandraShard.getHost() + ":" + cassandraShard.getPort() + "/" + cassandraShard.getUser() + "/" + cassandraShard.getKeySpaceName(); + connectionPoolMap.put(connectionKey, session); + } + + + } + + @Override + public CqlSession getConnection(String connectionRequestKey) throws ConnectionException { + try { + if (connectionPoolMap == null) { + LOG.warn("Connection pool not initialized"); + return null; + } + CqlSession session = connectionPoolMap.get(connectionRequestKey); + if (session == null) { + LOG.warn("Connection pool not found for source connection: {}", connectionRequestKey); + return null; + } + return session; + } catch (Exception e) { + throw new ConnectionException(e); + } + } + + @Override + public boolean isConnectionPoolInitialized() { + return false; + } + + // for unit testing + public void setConnectionPoolMap(Map inputMap) { + connectionPoolMap = inputMap; + } +} \ No newline at end of file diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dbutils/connection/JdbcConnectionHelper.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dbutils/connection/JdbcConnectionHelper.java index 5dfc014aba..893e559efc 100644 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dbutils/connection/JdbcConnectionHelper.java +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dbutils/connection/JdbcConnectionHelper.java @@ -15,7 +15,8 @@ */ package com.google.cloud.teleport.v2.templates.dbutils.connection; -import com.google.cloud.teleport.v2.spanner.migrations.shard.Shard; +import com.google.cloud.teleport.v2.spanner.migrations.shard.IShard; +import com.google.cloud.teleport.v2.spanner.migrations.shard.MySqlShard; import com.google.cloud.teleport.v2.templates.exceptions.ConnectionException; import com.google.cloud.teleport.v2.templates.models.ConnectionHelperRequest; import com.zaxxer.hikari.HikariConfig; @@ -51,13 +52,13 @@ public synchronized void init(ConnectionHelperRequest connectionHelperRequest) { LOG.info( "Initializing connection pool with size: ", connectionHelperRequest.getMaxConnections()); connectionPoolMap = new HashMap<>(); - for (Shard shard : connectionHelperRequest.getShards()) { + for (IShard mySqlShard : connectionHelperRequest.getShards()) { String sourceConnectionUrl = - "jdbc:mysql://" + shard.getHost() + ":" + shard.getPort() + "/" + shard.getDbName(); + "jdbc:mysql://" + mySqlShard.getHost() + ":" + mySqlShard.getPort() + "/" + mySqlShard.getDbName(); HikariConfig config = new HikariConfig(); config.setJdbcUrl(sourceConnectionUrl); - config.setUsername(shard.getUserName()); - config.setPassword(shard.getPassword()); + config.setUsername(mySqlShard.getUser()); + config.setPassword(mySqlShard.getPassword()); config.setDriverClassName(connectionHelperRequest.getDriver()); config.setMaximumPoolSize(connectionHelperRequest.getMaxConnections()); config.setConnectionInitSql(connectionHelperRequest.getConnectionInitQuery()); @@ -77,7 +78,7 @@ public synchronized void init(ConnectionHelperRequest connectionHelperRequest) { } HikariDataSource ds = new HikariDataSource(config); - connectionPoolMap.put(sourceConnectionUrl + "/" + shard.getUserName(), ds); + connectionPoolMap.put(sourceConnectionUrl + "/" + mySqlShard.getUser(), ds); } } diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dbutils/dao/source/CassandraDao.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dbutils/dao/source/CassandraDao.java new file mode 100644 index 0000000000..463b14a0b1 --- /dev/null +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dbutils/dao/source/CassandraDao.java @@ -0,0 +1,54 @@ +package com.google.cloud.teleport.v2.templates.dbutils.dao.source; + +import com.datastax.oss.driver.api.core.CqlSession; +import com.datastax.oss.driver.api.core.cql.PreparedStatement; +import com.datastax.oss.driver.api.core.cql.BoundStatement; +import com.google.cloud.teleport.v2.templates.dbutils.connection.IConnectionHelper; +import com.google.cloud.teleport.v2.templates.exceptions.ConnectionException; +import com.google.cloud.teleport.v2.templates.models.DMLGeneratorResponse; +import com.google.cloud.teleport.v2.templates.models.PreparedStatementGeneratedResponse; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import java.util.List; + +public class CassandraDao implements IDao { + private static final Logger LOG = LoggerFactory.getLogger(CassandraDao.class); + private final String cassandraUrl; + private final String cassandraUser; + private final IConnectionHelper connectionHelper; + + public CassandraDao(String cassandraUrl, String cassandraUser, IConnectionHelper connectionHelper) { + this.cassandraUrl = cassandraUrl; + this.cassandraUser = cassandraUser; + this.connectionHelper = connectionHelper; + } + + @Override + public void write(DMLGeneratorResponse dmlGeneratorResponse) throws Exception { + try (CqlSession session = (CqlSession) connectionHelper.getConnection(this.cassandraUrl)) { // Ensure connection is obtained + if (session == null) { + throw new ConnectionException("Connection is null"); + } + if (dmlGeneratorResponse instanceof PreparedStatementGeneratedResponse) { + PreparedStatementGeneratedResponse preparedStatementGeneratedResponse = (PreparedStatementGeneratedResponse) dmlGeneratorResponse; + try { + String dmlStatement = preparedStatementGeneratedResponse.getDmlStatement(); + PreparedStatement preparedStatement = session.prepare(dmlStatement); + + List values = preparedStatementGeneratedResponse.getValues(); + if (dmlStatement.chars().filter(ch -> ch == '?').count() != values.size()) { + throw new IllegalArgumentException("Mismatch between placeholders and parameter count."); + } + BoundStatement boundStatement = preparedStatement.bind(values.toArray()); + session.execute(boundStatement); + }catch (Exception e){ + LOG.error(e.getMessage()); + } + + } else { + String simpleStatement = dmlGeneratorResponse.getDmlStatement(); + session.execute(simpleStatement); + } + } + } +} \ No newline at end of file diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dbutils/dao/source/IDao.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dbutils/dao/source/IDao.java index 93fe08f2c5..9942f04097 100644 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dbutils/dao/source/IDao.java +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dbutils/dao/source/IDao.java @@ -19,8 +19,8 @@ public interface IDao { /** * Executes a given write statement against the data source. * - * @param statement Query statement. + * @param dmlResponseGenerator Query statement. * @throws Exception If there is an error executing the statement. */ - void write(T statement) throws Exception; + void write(T dmlResponseGenerator) throws Exception; } diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dbutils/dao/source/JdbcDao.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dbutils/dao/source/JdbcDao.java index 82ace6e411..3bf6715bf3 100644 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dbutils/dao/source/JdbcDao.java +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dbutils/dao/source/JdbcDao.java @@ -17,11 +17,18 @@ import com.google.cloud.teleport.v2.templates.dbutils.connection.IConnectionHelper; import com.google.cloud.teleport.v2.templates.exceptions.ConnectionException; +import com.google.cloud.teleport.v2.templates.models.DMLGeneratorResponse; +import com.google.cloud.teleport.v2.templates.models.PreparedStatementGeneratedResponse; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import java.sql.Connection; +import java.sql.PreparedStatement; import java.sql.SQLException; import java.sql.Statement; -public class JdbcDao implements IDao { +public class JdbcDao implements IDao { + private static final Logger LOG = LoggerFactory.getLogger(JdbcDao.class); private String sqlUrl; private String sqlUser; @@ -34,26 +41,36 @@ public JdbcDao(String sqlUrl, String sqlUser, IConnectionHelper connectionHelper } @Override - public void write(String sqlStatement) throws SQLException, ConnectionException { + public void write(DMLGeneratorResponse dmlGeneratorResponse) throws SQLException, ConnectionException { Connection connObj = null; - Statement statement = null; - try { - connObj = (Connection) connectionHelper.getConnection(this.sqlUrl + "/" + this.sqlUser); if (connObj == null) { throw new ConnectionException("Connection is null"); } - statement = connObj.createStatement(); - statement.executeUpdate(sqlStatement); - - } finally { - - if (statement != null) { - statement.close(); + if (dmlGeneratorResponse instanceof PreparedStatementGeneratedResponse) { + PreparedStatementGeneratedResponse preparedStatementGeneratedResponse = (PreparedStatementGeneratedResponse) dmlGeneratorResponse; + try (PreparedStatement statement = connObj.prepareStatement(preparedStatementGeneratedResponse.getDmlStatement())) { + int index = 1; + for (Object value : preparedStatementGeneratedResponse.getValues()) { + statement.setObject(index++, value); // Bind values to placeholders + } + statement.executeUpdate(); + } + } else { + try (Statement statement = connObj.createStatement()) { + statement.executeUpdate(dmlGeneratorResponse.getDmlStatement()); + } } + } catch (SQLException e) { + LOG.error(e.getMessage()); + } finally { if (connObj != null) { - connObj.close(); + try { + connObj.close(); + } catch (SQLException e) { + LOG.error(e.getMessage()); + } } } } diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dbutils/dml/CassandraDMLGenerator.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dbutils/dml/CassandraDMLGenerator.java new file mode 100644 index 0000000000..040c2cf424 --- /dev/null +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dbutils/dml/CassandraDMLGenerator.java @@ -0,0 +1,354 @@ +/* + * Copyright (C) 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.templates.dbutils.dml; + +import com.google.cloud.teleport.v2.spanner.migrations.schema.*; +import com.google.cloud.teleport.v2.templates.models.DMLGeneratorRequest; +import com.google.cloud.teleport.v2.templates.models.DMLGeneratorResponse; +import com.google.cloud.teleport.v2.templates.models.PreparedStatementGeneratedResponse; +import com.google.cloud.teleport.v2.templates.models.RawStatementGeneratedResponse; +import org.json.JSONObject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.time.Instant; +import java.util.*; + +/** + * Creates DML statements For Cassandra + */ +public class CassandraDMLGenerator implements IDMLGenerator { + private static final Logger LOG = LoggerFactory.getLogger(CassandraDMLGenerator.class); + + /** + * @param dmlGeneratorRequest the request containing necessary information to construct the DML + * statement, including modification type, table schema, new values, and key values. + * @return + */ + @Override + public DMLGeneratorResponse getDMLStatement(DMLGeneratorRequest dmlGeneratorRequest) { + if (dmlGeneratorRequest + .getSchema() + .getSpannerToID() + .get(dmlGeneratorRequest.getSpannerTableName()) + == null) { + LOG.warn( + "The spanner table {} was not found in session file, dropping the record", + dmlGeneratorRequest.getSpannerTableName()); + return new RawStatementGeneratedResponse(""); + } + + String spannerTableId = + dmlGeneratorRequest + .getSchema() + .getSpannerToID() + .get(dmlGeneratorRequest.getSpannerTableName()) + .getName(); + SpannerTable spannerTable = dmlGeneratorRequest.getSchema().getSpSchema().get(spannerTableId); + + if (spannerTable == null) { + LOG.warn( + "The spanner table {} was not found in session file, dropping the record", + dmlGeneratorRequest.getSpannerTableName()); + return new RawStatementGeneratedResponse(""); + } + + SourceTable sourceTable = dmlGeneratorRequest.getSchema().getSrcSchema().get(spannerTableId); + if (sourceTable == null) { + LOG.warn("The table {} was not found in source", dmlGeneratorRequest.getSpannerTableName()); + return new RawStatementGeneratedResponse(""); + } + + if (sourceTable.getPrimaryKeys() == null || sourceTable.getPrimaryKeys().length == 0) { + LOG.warn( + "Cannot reverse replicate for table {} without primary key, skipping the record", + sourceTable.getName()); + return new RawStatementGeneratedResponse(""); + } + + Map pkColumnNameValues = + getPkColumnValues( + spannerTable, + sourceTable, + dmlGeneratorRequest.getNewValuesJson(), + dmlGeneratorRequest.getKeyValuesJson(), + dmlGeneratorRequest.getSourceDbTimezoneOffset()); + if (pkColumnNameValues == null) { + LOG.warn( + "Cannot reverse replicate for table {} without primary key, skipping the record", + sourceTable.getName()); + return new RawStatementGeneratedResponse(""); + } + + if ("INSERT".equals(dmlGeneratorRequest.getModType()) + || "UPDATE".equals(dmlGeneratorRequest.getModType())) { + return generateUpsertStatement( + spannerTable, + sourceTable, + dmlGeneratorRequest, + pkColumnNameValues + ); + + } else if ("DELETE".equals(dmlGeneratorRequest.getModType())) { + return getDeleteStatementCQL(sourceTable.getName(), pkColumnNameValues, Instant.now().toEpochMilli() * 1000); + } else { + LOG.warn("Unsupported modType: " + dmlGeneratorRequest.getModType()); + return new RawStatementGeneratedResponse(""); + } + } + + private static DMLGeneratorResponse generateUpsertStatement( + SpannerTable spannerTable, + SourceTable sourceTable, + DMLGeneratorRequest dmlGeneratorRequest, + Map pkColumnNameValues) { + Map columnNameValues = + getColumnValues( + spannerTable, + sourceTable, + dmlGeneratorRequest.getNewValuesJson(), + dmlGeneratorRequest.getKeyValuesJson(), + dmlGeneratorRequest.getSourceDbTimezoneOffset()); + return getUpsertStatementCQL( + sourceTable.getName(), + Instant.now().toEpochMilli() * 1000, + columnNameValues, + pkColumnNameValues + ); + } + + private static DMLGeneratorResponse getUpsertStatementCQL( + String tableName, + long timestamp, + Map columnNameValues, + Map pkColumnNameValues + ) { + + StringBuilder allColumns = new StringBuilder(); + StringBuilder placeholders = new StringBuilder(); + List values = new ArrayList<>(); + + for (Map.Entry entry : pkColumnNameValues.entrySet()) { + String colName = entry.getKey(); + Object colValue = entry.getValue(); + + allColumns.append(colName).append(", "); + placeholders.append("?, "); + values.add(colValue); + } + + for (Map.Entry entry : columnNameValues.entrySet()) { + String colName = entry.getKey(); + Object colValue = entry.getValue(); + + allColumns.append(colName).append(", "); + placeholders.append("?, "); + values.add(colValue); + } + + if (allColumns.length() > 0) { + allColumns.setLength(allColumns.length() - 2); + } + if (placeholders.length() > 0) { + placeholders.setLength(placeholders.length() - 2); + } + + String preparedStatement = "INSERT INTO " + tableName + " (" + allColumns + ") VALUES (" + placeholders + ") USING TIMESTAMP ?;"; + values.add(timestamp); + return new PreparedStatementGeneratedResponse(preparedStatement, values); + } + + + private static DMLGeneratorResponse getDeleteStatementCQL( + String tableName, + Map pkColumnNameValues, + long timestamp + ) { + + StringBuilder deleteConditions = new StringBuilder(); + List values = new ArrayList<>(); + + for (Map.Entry entry : pkColumnNameValues.entrySet()) { + String colName = entry.getKey(); + deleteConditions.append(colName).append(" = ? AND "); + values.add(entry.getValue()); + } + + if (deleteConditions.length() > 0) { + deleteConditions.setLength(deleteConditions.length() - 5); + } + + String preparedStatement = "DELETE FROM " + tableName + " WHERE " + deleteConditions + " USING TIMESTAMP ?;"; + + values.add(timestamp); + + return new PreparedStatementGeneratedResponse(preparedStatement, values); + } + + + private static Map getColumnValues( + SpannerTable spannerTable, + SourceTable sourceTable, + JSONObject newValuesJson, + JSONObject keyValuesJson, + String sourceDbTimezoneOffset + ) { + Map response = new HashMap<>(); + + /* + Get all non-primary key col ids from source table + For each - get the corresponding column name from spanner Schema + if the column cannot be found in spanner schema - continue to next, + as the column will be stored with default/null values + check if the column name found in Spanner schema exists in keyJson - + if so, get the string value + else + check if the column name found in Spanner schema exists in valuesJson - + if so, get the string value + if the column does not exist in any of the JSON - continue to next, + as the column will be stored with default/null values + */ + Set sourcePKs = sourceTable.getPrimaryKeySet(); + for (Map.Entry entry : sourceTable.getColDefs().entrySet()) { + SourceColumnDefinition sourceColDef = entry.getValue(); + + String colName = sourceColDef.getName(); + if (sourcePKs.contains(colName)) { + continue; // we only need non-primary keys + } + + String colId = entry.getKey(); + SpannerColumnDefinition spannerColDef = spannerTable.getColDefs().get(colId); + if (spannerColDef == null) { + continue; + } + String spannerColumnName = spannerColDef.getName(); + Object columnValue; + if (keyValuesJson.has(spannerColumnName)) { + // get the value based on Spanner and Source type + if (keyValuesJson.isNull(spannerColumnName)) { + response.put(sourceColDef.getName(), "NULL"); + continue; + } + columnValue = + getMappedColumnValue( + spannerColDef, sourceColDef, keyValuesJson, sourceDbTimezoneOffset); + } else if (newValuesJson.has(spannerColumnName)) { + // get the value based on Spanner and Source type + if (newValuesJson.isNull(spannerColumnName)) { + response.put(sourceColDef.getName(), "NULL"); + continue; + } + columnValue = + getMappedColumnValue( + spannerColDef, sourceColDef, newValuesJson, sourceDbTimezoneOffset); + } else { + continue; + } + + response.put(sourceColDef.getName(), columnValue); + } + + return response; + } + + private static Map getPkColumnValues( + SpannerTable spannerTable, + SourceTable sourceTable, + JSONObject newValuesJson, + JSONObject keyValuesJson, + String sourceDbTimezoneOffset + ) { + Map response = new HashMap<>(); + /* + Get all primary key col ids from source table + For each - get the corresponding column name from spanner Schema + if the column cannot be found in spanner schema - return null + check if the column name found in Spanner schema exists in keyJson - + if so, get the string value + else + check if the column name found in Spanner schema exists in valuesJson - + if so, get the string value + if the column does not exist in any of the JSON - return null + */ + ColumnPK[] sourcePKs = sourceTable.getPrimaryKeys(); + + for (ColumnPK currentSourcePK : sourcePKs) { + String colId = currentSourcePK.getColId(); + SourceColumnDefinition sourceColDef = sourceTable.getColDefs().get(colId); + SpannerColumnDefinition spannerColDef = spannerTable.getColDefs().get(colId); + if (spannerColDef == null) { + LOG.warn( + "The corresponding primary key column {} was not found in Spanner", + sourceColDef.getName()); + return null; + } + String spannerColumnName = spannerColDef.getName(); + Object columnValue; + if (keyValuesJson.has(spannerColumnName)) { + // get the value based on Spanner and Source type + if (keyValuesJson.isNull(spannerColumnName)) { + response.put(sourceColDef.getName(), "NULL"); + continue; + } + columnValue = + getMappedColumnValue( + spannerColDef, + sourceColDef, + keyValuesJson, + sourceDbTimezoneOffset + ); + } else if (newValuesJson.has(spannerColumnName)) { + // get the value based on Spanner and Source type + if (newValuesJson.isNull(spannerColumnName)) { + response.put(sourceColDef.getName(), "NULL"); + continue; + } + columnValue = + getMappedColumnValue( + spannerColDef, + sourceColDef, + newValuesJson, + sourceDbTimezoneOffset + ); + } else { + LOG.warn("The column {} was not found in input record", spannerColumnName); + return null; + } + + response.put(sourceColDef.getName(), columnValue); + } + + return response; + } + + private static Object getMappedColumnValue( + SpannerColumnDefinition spannerColDef, + SourceColumnDefinition sourceColDef, + JSONObject valuesJson, + String sourceDbTimezoneOffset) { + return TypeHandler.getColumnValueByType( + spannerColDef, + sourceColDef, + valuesJson, + sourceDbTimezoneOffset + ); + } + +} + + + diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dbutils/dml/MySQLDMLGenerator.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dbutils/dml/MySQLDMLGenerator.java index 8774fedfa9..b9f2719a9a 100644 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dbutils/dml/MySQLDMLGenerator.java +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dbutils/dml/MySQLDMLGenerator.java @@ -26,6 +26,8 @@ import java.util.Map; import java.util.Set; import java.util.stream.Collectors; + +import com.google.cloud.teleport.v2.templates.models.RawStatementGeneratedResponse; import org.apache.commons.lang3.StringUtils; import org.json.JSONObject; import org.slf4j.Logger; @@ -45,7 +47,7 @@ public DMLGeneratorResponse getDMLStatement(DMLGeneratorRequest dmlGeneratorRequ LOG.warn( "The spanner table {} was not found in session file, dropping the record", dmlGeneratorRequest.getSpannerTableName()); - return new DMLGeneratorResponse(""); + return new RawStatementGeneratedResponse(""); } String spannerTableId = @@ -60,20 +62,20 @@ public DMLGeneratorResponse getDMLStatement(DMLGeneratorRequest dmlGeneratorRequ LOG.warn( "The spanner table {} was not found in session file, dropping the record", dmlGeneratorRequest.getSpannerTableName()); - return new DMLGeneratorResponse(""); + return new RawStatementGeneratedResponse(""); } SourceTable sourceTable = dmlGeneratorRequest.getSchema().getSrcSchema().get(spannerTableId); if (sourceTable == null) { LOG.warn("The table {} was not found in source", dmlGeneratorRequest.getSpannerTableName()); - return new DMLGeneratorResponse(""); + return new RawStatementGeneratedResponse(""); } if (sourceTable.getPrimaryKeys() == null || sourceTable.getPrimaryKeys().length == 0) { LOG.warn( "Cannot reverse replicate for table {} without primary key, skipping the record", sourceTable.getName()); - return new DMLGeneratorResponse(""); + return new RawStatementGeneratedResponse(""); } Map pkcolumnNameValues = @@ -87,7 +89,7 @@ public DMLGeneratorResponse getDMLStatement(DMLGeneratorRequest dmlGeneratorRequ LOG.warn( "Cannot reverse replicate for table {} without primary key, skipping the record", sourceTable.getName()); - return new DMLGeneratorResponse(""); + return new RawStatementGeneratedResponse(""); } if ("INSERT".equals(dmlGeneratorRequest.getModType()) @@ -99,7 +101,7 @@ public DMLGeneratorResponse getDMLStatement(DMLGeneratorRequest dmlGeneratorRequ return getDeleteStatement(sourceTable.getName(), pkcolumnNameValues); } else { LOG.warn("Unsupported modType: " + dmlGeneratorRequest.getModType()); - return new DMLGeneratorResponse(""); + return new RawStatementGeneratedResponse(""); } } @@ -128,7 +130,7 @@ private static DMLGeneratorResponse getUpsertStatement( String returnVal = "INSERT INTO `" + tableName + "`(" + allColumns + ")" + " VALUES (" + allValues + ") "; - return new DMLGeneratorResponse(returnVal); + return new RawStatementGeneratedResponse(returnVal); } int index = 0; @@ -160,7 +162,7 @@ private static DMLGeneratorResponse getUpsertStatement( + "ON DUPLICATE KEY UPDATE " + updateValues; - return new DMLGeneratorResponse(returnVal); + return new RawStatementGeneratedResponse(returnVal); } private static DMLGeneratorResponse getDeleteStatement( @@ -180,7 +182,7 @@ private static DMLGeneratorResponse getDeleteStatement( } String returnVal = "DELETE FROM `" + tableName + "` WHERE " + deleteValues; - return new DMLGeneratorResponse(returnVal); + return new RawStatementGeneratedResponse(returnVal); } private static DMLGeneratorResponse generateUpsertStatement( diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dbutils/dml/TypeHandler.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dbutils/dml/TypeHandler.java new file mode 100644 index 0000000000..76ef72e583 --- /dev/null +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dbutils/dml/TypeHandler.java @@ -0,0 +1,376 @@ +/* + * Copyright (C) 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.templates.dbutils.dml; +import com.google.cloud.teleport.v2.spanner.migrations.schema.SourceColumnDefinition; +import com.google.cloud.teleport.v2.spanner.migrations.schema.SpannerColumnDefinition; +import org.json.JSONArray; +import org.json.JSONObject; + +import java.nio.ByteBuffer; +import java.sql.Timestamp; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.time.LocalDate; +import java.time.ZoneId; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; +import java.util.*; +import java.util.stream.Collectors; + +class TypeHandler { + private static Boolean handleCassandraBoolType(String colName, JSONObject valuesJson) { + return valuesJson.getBoolean(colName); + } + + private static Float handleCassandraFloatType(String colName, JSONObject valuesJson) { + return valuesJson.getBigDecimal(colName).floatValue(); + } + + private static Double handleCassandraDoubleType(String colName, JSONObject valuesJson) { + return valuesJson.getBigDecimal(colName).doubleValue(); + } + + private static ByteBuffer handleCassandraBlobType(String colName, JSONObject valuesJson) { + Object colValue = valuesJson.opt(colName); + if (colValue == null) { + return null; + } + return parseBlobType(colName, colValue); + + } + + private static ByteBuffer parseBlobType(String colName, Object colValue) { + byte[] byteArray; + + if (colValue instanceof byte[]) { + byteArray = (byte[]) colValue; + } else if (colValue instanceof String) { + byteArray = java.util.Base64.getDecoder().decode((String) colValue); + } else { + throw new IllegalArgumentException("Unsupported type for column " + colName); + } + + return ByteBuffer.wrap(byteArray); + } + + private static Date handleCassandraDateType(String colName, JSONObject valuesJson) { + return handleCassandraGenericDateType(colName, valuesJson, "yyyy-MM-dd"); + } + + private static Date handleCassandraTimestampType(String colName, JSONObject valuesJson) { + return handleCassandraGenericDateType(colName, valuesJson, "yyyy-MM-dd'T'HH:mm:ss.SSSZ"); + } + + private static Date handleCassandraGenericDateType(String colName, JSONObject valuesJson, String formatter) { + Object colValue = valuesJson.opt(colName); + if (colValue == null) { + return null; + } + + if (formatter == null) { + formatter = "yyyy-MM-dd'T'HH:mm:ss.SSSZ"; + } + + return parseDate(colName, colValue, formatter); + } + + private static Date parseDate(String colName, Object colValue, String formatter) { + Date date; + + if (colValue instanceof String) { + try { + date = new SimpleDateFormat(formatter).parse((String) colValue); + } catch (ParseException e) { + throw new IllegalArgumentException("Invalid timestamp format for column " + colName, e); + } + } else if (colValue instanceof java.util.Date) { + date = (java.util.Date) colValue; + } else if (colValue instanceof Long) { + date = new Date((Long) colValue); + } else { + throw new IllegalArgumentException("Unsupported type for column " + colName); + } + + return date; + } + + private static String handleCassandraTextType(String colName, JSONObject valuesJson) { + return valuesJson.optString(colName, null); // Get the value or null if the key is not found or the value is null + } + + private static UUID handleCassandraUuidType(String colName, JSONObject valuesJson) { + String uuidString = valuesJson.optString(colName, null); // Get the value or null if the key is not found or the value is null + + if (uuidString == null) { + return null; + } + + return UUID.fromString(uuidString); + + } + + private static Long handleCassandraBigintType(String colName, JSONObject valuesJson) { + return valuesJson.getBigInteger(colName).longValue(); + } + + private static Integer handleCassandraIntType(String colName, JSONObject valuesJson) { + return valuesJson.getBigInteger(colName).intValue(); + } + + private static List handleInt64ArrayType(String colName, JSONObject valuesJson) { + JSONArray jsonArray = valuesJson.getJSONArray(colName); + List colValueList = new ArrayList<>(); + + // Convert each element to Long and add it to the list + for (int i = 0; i < jsonArray.length(); i++) { + colValueList.add(jsonArray.getLong(i)); + } + + return colValueList; + } + + private static Set handleInt64SetType(String colName, JSONObject valuesJson) { + return new HashSet<>(handleInt64ArrayType(colName, valuesJson)); + } + + private static List handleInt64ArrayAsInt32Array(String colName, JSONObject valuesJson) { + return handleInt64ArrayType(colName, valuesJson).stream().map(Long::intValue).collect(Collectors.toList()); + } + + private static Set handleInt64ArrayAsInt32Set(String colName, JSONObject valuesJson) { + return handleInt64ArrayType(colName, valuesJson).stream().map(Long::intValue).collect(Collectors.toSet()); + } + + private static Set handleStringArrayType(String colName, JSONObject valuesJson) { + return valuesJson.getJSONArray(colName).toList().stream() + .map(String::valueOf) + .collect(Collectors.toSet()); + } + + private static List handleStringSetType(String colName, JSONObject valuesJson) { + return valuesJson.getJSONArray(colName).toList().stream() + .map(String::valueOf) + .collect(Collectors.toList()); + } + + // Handler for ARRAY (also serves as Set) + private static List handleBoolArrayType(String colName, JSONObject valuesJson) { + return valuesJson.getJSONArray(colName).toList().stream() + .map(obj -> obj instanceof String && Boolean.parseBoolean((String) obj)) + .collect(Collectors.toList()); + } + + private static Set handleBoolSetTypeString(String colName, JSONObject valuesJson) { + return valuesJson.getJSONArray(colName).toList().stream() + .map(obj -> obj instanceof String && Boolean.parseBoolean((String) obj)) + .collect(Collectors.toSet()); + } + + private static List handleFloat64ArrayType(String colName, JSONObject valuesJson) { + return valuesJson.getJSONArray(colName).toList().stream() + .map(obj -> { + if (obj instanceof Number) { + return ((Number) obj).doubleValue(); + } else if (obj instanceof String) { + try { + return Double.valueOf((String) obj); + } catch (NumberFormatException e) { + throw new IllegalArgumentException("Invalid number format for column " + colName, e); + } + } else { + throw new IllegalArgumentException("Unsupported type for column " + colName); + } + }) + .collect(Collectors.toList()); + } + + private static Set handleFloat64SetType(String colName, JSONObject valuesJson) { + return valuesJson.getJSONArray(colName).toList().stream() + .map(obj -> { + if (obj instanceof Number) { + return ((Number) obj).doubleValue(); + } else if (obj instanceof String) { + try { + return Double.valueOf((String) obj); + } catch (NumberFormatException e) { + throw new IllegalArgumentException("Invalid number format for column " + colName, e); + } + } else { + throw new IllegalArgumentException("Unsupported type for column " + colName); + } + }) + .collect(Collectors.toSet()); + } + + private static List handleFloatArrayType(String colName, JSONObject valuesJson) { + return handleFloat64ArrayType(colName, valuesJson).stream().map(Double::floatValue).collect(Collectors.toList()); + } + + private static Set handleFloatSetType(String colName, JSONObject valuesJson) { + return handleFloat64SetType(colName, valuesJson).stream().map(Double::floatValue).collect(Collectors.toSet()); + } + + private static List handleDateArrayType(String colName, JSONObject valuesJson) { + return valuesJson.getJSONArray(colName).toList().stream() + .map(obj -> parseDate(colName, obj, "yyyy-MM-dd")) + .collect(Collectors.toList()); + } + + private static Set handleDateSetType(String colName, JSONObject valuesJson) { + return new HashSet<>(handleDateArrayType(colName, valuesJson)); + } + + private static List handleTimestampArrayType(String colName, JSONObject valuesJson) { + return valuesJson.getJSONArray(colName).toList().stream() + .map(value -> { + return Timestamp.valueOf(parseDate(colName, value, "yyyy-MM-dd'T'HH:mm:ss.SSSZ").toString()); + }) + .collect(Collectors.toList()); + } + + private static Set handleTimestampSetType(String colName, JSONObject valuesJson) { + return new HashSet<>(handleTimestampArrayType(colName, valuesJson)); + } + + private static List handleByteArrayType(String colName, JSONObject valuesJson) { + return valuesJson.getJSONArray(colName).toList().stream() + .map(value -> { + return parseBlobType(colName, value); + }) + .collect(Collectors.toList()); + } + + private static Set handleByteSetType(String colName, JSONObject valuesJson) { + return new HashSet<>(handleByteArrayType(colName, valuesJson)); + } + + public static Object getColumnValueByType( + SpannerColumnDefinition spannerColDef, + SourceColumnDefinition sourceColDef, + JSONObject valuesJson, + String sourceDbTimezoneOffset + ) { + + String columnType = sourceColDef.getType().getName(); + Object colValue = null; + String colType = spannerColDef.getType().getName(); + String colName = spannerColDef.getName(); + + if ("FLOAT64".equals(colType)) { + colValue = TypeHandler.handleCassandraFloatType(colName, valuesJson); + } else if ("BOOL".equals(colType)) { + colValue = TypeHandler.handleCassandraBoolType(colName, valuesJson); + } else if ("STRING".equals(colType) && spannerColDef.getType().getIsArray()) { + colValue = TypeHandler.handleStringArrayType(colName, valuesJson); + } else if ("BYTES".equals(colType)) { + colValue = TypeHandler.handleCassandraBlobType(colName, valuesJson); + } else { + colValue = TypeHandler.handleCassandraTextType(colName, valuesJson); + } + + Object response = null; + // Ensure colValue is not null to avoid NullPointerExceptions + if (colValue == null) { + return colValue; + } + + switch (columnType.toLowerCase()) { + case "text": + case "varchar": + case "ascii": + // Handle text-like types + response = "'" + escapeCassandraString(String.valueOf(colValue)) + "'"; + break; + + case "timestamp": + case "datetime": + // Handle timestamp, adjust to UTC if necessary + response = convertToCassandraTimestamp(String.valueOf(colValue), sourceDbTimezoneOffset); + break; + + case "uuid": + case "timeuuid": + // UUIDs should be properly formatted for Cassandra + response = isValidUUID(String.valueOf(colValue)) ? colValue : "null"; + break; + + case "int": + case "bigint": + case "smallint": + case "tinyint": + case "varint": + case "counter": + case "double": + case "float": + case "decimal": + case "blob": + case "boolean": + response = colValue; + break; + case "date": + response = convertToCassandraDate((Date) colValue); + break; + + default: + throw new IllegalArgumentException("Invalid column " + colName + " do not have mapping created for " + columnType); + } + + return response; + } + + private static Integer convertToCQLDate(com.google.cloud.Date spannerDate) { + // Convert Google Cloud Date to an integer that represents the number of days since the epoch + java.sql.Date sqlDate = java.sql.Date.valueOf(spannerDate.toString()); + long millis = sqlDate.getTime(); + return (int) (millis / (1000 * 60 * 60 * 24)); + } + + private static String escapeCassandraString(String value) { + return value.replace("'", "''"); + } + + private static String convertToCassandraTimestamp(String value, String timezoneOffset) { + // Parse the timestamp and adjust to UTC if needed + ZonedDateTime dateTime = ZonedDateTime.parse(value); + return "'" + dateTime.withZoneSameInstant(ZoneOffset.UTC).toString() + "'"; + } + + private static String convertToCassandraDate(Date date) { + LocalDate localDate = date.toInstant().atZone(ZoneId.systemDefault()).toLocalDate(); + return localDate.toString(); // Returns the date in the format "yyyy-MM-dd" + } + + private static boolean isValidUUID(String value) { + try { + UUID.fromString(value); + return true; + } catch (IllegalArgumentException e) { + return false; + } + } + + private static String bytesToHex(byte[] bytes) { + StringBuilder hexString = new StringBuilder(); + for (byte b : bytes) { + String hex = Integer.toHexString(0xFF & b); + if (hex.length() == 1) { + hexString.append('0'); + } + hexString.append(hex); + } + return hexString.toString(); + } +} diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dbutils/processor/InputRecordProcessor.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dbutils/processor/InputRecordProcessor.java index 6fba8c3fe2..27e61cf2b3 100644 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dbutils/processor/InputRecordProcessor.java +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dbutils/processor/InputRecordProcessor.java @@ -65,7 +65,7 @@ public static void processRecord( LOG.warn("DML statement is empty for table: " + tableName); return; } - dao.write(dmlGeneratorResponse.getDmlStatement()); + dao.write(dmlGeneratorResponse); Counter numRecProcessedMetric = Metrics.counter(shardId, "records_written_to_source_" + shardId); diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dbutils/processor/SourceProcessorFactory.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dbutils/processor/SourceProcessorFactory.java index bde4b10fe7..b2c074174d 100644 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dbutils/processor/SourceProcessorFactory.java +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dbutils/processor/SourceProcessorFactory.java @@ -15,12 +15,15 @@ */ package com.google.cloud.teleport.v2.templates.dbutils.processor; -import com.google.cloud.teleport.v2.spanner.migrations.shard.Shard; +import com.google.cloud.teleport.v2.spanner.migrations.shard.IShard; import com.google.cloud.teleport.v2.templates.constants.Constants; +import com.google.cloud.teleport.v2.templates.dbutils.connection.CassandraConnectionHelper; import com.google.cloud.teleport.v2.templates.dbutils.connection.IConnectionHelper; import com.google.cloud.teleport.v2.templates.dbutils.connection.JdbcConnectionHelper; +import com.google.cloud.teleport.v2.templates.dbutils.dao.source.CassandraDao; import com.google.cloud.teleport.v2.templates.dbutils.dao.source.IDao; import com.google.cloud.teleport.v2.templates.dbutils.dao.source.JdbcDao; +import com.google.cloud.teleport.v2.templates.dbutils.dml.CassandraDMLGenerator; import com.google.cloud.teleport.v2.templates.dbutils.dml.IDMLGenerator; import com.google.cloud.teleport.v2.templates.dbutils.dml.MySQLDMLGenerator; import com.google.cloud.teleport.v2.templates.exceptions.UnsupportedSourceException; @@ -33,22 +36,36 @@ import java.util.function.Function; public class SourceProcessorFactory { - private static Map dmlGeneratorMap = - Map.of(Constants.SOURCE_MYSQL, new MySQLDMLGenerator()); + private static Map dmlGeneratorMap = new HashMap<>(); - private static Map connectionHelperMap = - Map.of(Constants.SOURCE_MYSQL, new JdbcConnectionHelper()); + private static Map connectionHelperMap = new HashMap<>(); private static Map driverMap = Map.of(Constants.SOURCE_MYSQL, "com.mysql.cj.jdbc.Driver"); - private static Map> connectionUrl = - Map.of( - Constants.SOURCE_MYSQL, - shard -> - "jdbc:mysql://" + shard.getHost() + ":" + shard.getPort() + "/" + shard.getDbName()); + private static Map> connectionUrl = new HashMap<>(); - private static Map, Integer, ConnectionHelperRequest>> + static { + + dmlGeneratorMap.put(Constants.SOURCE_MYSQL, new MySQLDMLGenerator()); + dmlGeneratorMap.put(Constants.SOURCE_CASSANDRA, new CassandraDMLGenerator()); + + connectionHelperMap.put(Constants.SOURCE_MYSQL, new JdbcConnectionHelper()); + connectionHelperMap.put(Constants.SOURCE_CASSANDRA, new CassandraConnectionHelper()); + + connectionUrl.put( + Constants.SOURCE_MYSQL, + shard -> + "jdbc:mysql://" + shard.getHost() + ":" + shard.getPort() + "/" + shard.getDbName() + ); + connectionUrl.put( + Constants.SOURCE_CASSANDRA, + shard -> + shard.getHost() + ":" + shard.getPort() + "/" + shard.getUser() + "/" + shard.getKeySpaceName() + ); + } + + private static Map, Integer, ConnectionHelperRequest>> connectionHelperRequestFactory = Map.of( Constants.SOURCE_MYSQL, @@ -70,16 +87,16 @@ public static void setConnectionHelperMap(Map connect * Creates a SourceProcessor instance for the specified source type. * * @param source the type of the source database - * @param shards the list of shards for the source + * @param iShards the list of mySqlShards for the source * @param maxConnections the maximum number of connections * @return a configured SourceProcessor instance * @throws Exception if the source type is invalid */ public static SourceProcessor createSourceProcessor( - String source, List shards, int maxConnections) throws UnsupportedSourceException { + String source, List iShards, int maxConnections) throws UnsupportedSourceException { IDMLGenerator dmlGenerator = getDMLGenerator(source); - initializeConnectionHelper(source, shards, maxConnections); - Map sourceDaoMap = createSourceDaoMap(source, shards); + initializeConnectionHelper(source, iShards, maxConnections); + Map sourceDaoMap = createSourceDaoMap(source, iShards); return SourceProcessor.builder().dmlGenerator(dmlGenerator).sourceDaoMap(sourceDaoMap).build(); } @@ -101,28 +118,28 @@ private static IConnectionHelper getConnectionHelper(String source) } private static void initializeConnectionHelper( - String source, List shards, int maxConnections) throws UnsupportedSourceException { + String source, List iShards, int maxConnections) throws UnsupportedSourceException { IConnectionHelper connectionHelper = getConnectionHelper(source); if (!connectionHelper.isConnectionPoolInitialized()) { ConnectionHelperRequest request = - createConnectionHelperRequest(source, shards, maxConnections); + createConnectionHelperRequest(source, iShards, maxConnections); connectionHelper.init(request); } } private static ConnectionHelperRequest createConnectionHelperRequest( - String source, List shards, int maxConnections) throws UnsupportedSourceException { + String source, List iShards, int maxConnections) throws UnsupportedSourceException { return Optional.ofNullable(connectionHelperRequestFactory.get(source)) - .map(factory -> factory.apply(shards, maxConnections)) + .map(factory -> factory.apply(iShards, maxConnections)) .orElseThrow( () -> new UnsupportedSourceException( "Invalid source type for ConnectionHelperRequest: " + source)); } - private static Map createSourceDaoMap(String source, List shards) + private static Map createSourceDaoMap(String source, List iShards) throws UnsupportedSourceException { - Function urlGenerator = + Function urlGenerator = Optional.ofNullable(connectionUrl.get(source)) .orElseThrow( () -> @@ -130,9 +147,10 @@ private static Map createSourceDaoMap(String source, List s "Invalid source type for URL generation: " + source)); Map sourceDaoMap = new HashMap<>(); - for (Shard shard : shards) { + for (IShard shard : iShards) { String connectionUrl = urlGenerator.apply(shard); - IDao sqlDao = new JdbcDao(connectionUrl, shard.getUserName(), getConnectionHelper(source)); + IDao sqlDao = source.equals(Constants.SOURCE_MYSQL) ? new JdbcDao(connectionUrl, shard.getUser(), getConnectionHelper(source)) + : new CassandraDao(connectionUrl, shard.getUser(), getConnectionHelper(source)); sourceDaoMap.put(shard.getLogicalShardId(), sqlDao); } return sourceDaoMap; diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/models/ConnectionHelperRequest.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/models/ConnectionHelperRequest.java index dc8c4aeaf7..6abfb92a86 100644 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/models/ConnectionHelperRequest.java +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/models/ConnectionHelperRequest.java @@ -15,7 +15,9 @@ */ package com.google.cloud.teleport.v2.templates.models; -import com.google.cloud.teleport.v2.spanner.migrations.shard.Shard; +import com.google.cloud.teleport.v2.spanner.migrations.cassandra.CassandraConfig; +import com.google.cloud.teleport.v2.spanner.migrations.shard.IShard; + import java.util.List; /** @@ -25,7 +27,7 @@ * database or a data source. It includes: * *
    - *
  • A list of {@link Shard} objects representing the database shards. + *
  • A list of {@link IShard} objects representing the database mySqlShards. *
  • Optional connection properties as a {@link String}. *
  • The maximum number of connections allowed. *
  • The name of the driver to connect to source. @@ -33,14 +35,14 @@ *
*/ public class ConnectionHelperRequest { - private List shards; + private List iShards; private String properties; private int maxConnections; private String driver; private String connectionInitQuery; - public List getShards() { - return shards; + public List getShards() { + return iShards; } public String getProperties() { @@ -60,15 +62,20 @@ public String getConnectionInitQuery() { } public ConnectionHelperRequest( - List shards, + List iShards, String properties, int maxConnections, String driver, String connectionInitQuery) { - this.shards = shards; + this.iShards = iShards; this.properties = properties; this.maxConnections = maxConnections; this.driver = driver; this.connectionInitQuery = connectionInitQuery; } + + public ConnectionHelperRequest(CassandraConfig cassandraConfig, int maxConnections) { + this.cassandraConfig = cassandraConfig; + this.maxConnections = maxConnections; + } } diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/models/DMLGeneratorResponse.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/models/DMLGeneratorResponse.java index 42479a362d..f1165ce779 100644 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/models/DMLGeneratorResponse.java +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/models/DMLGeneratorResponse.java @@ -15,18 +15,10 @@ */ package com.google.cloud.teleport.v2.templates.models; -public class DMLGeneratorResponse { - private String dmlStatement; +import java.util.List; - public String getDmlStatement() { - return dmlStatement; - } - - public void setDmlStatement(String dmlStatement) { - this.dmlStatement = dmlStatement; - } - - public DMLGeneratorResponse(String dmlStatement) { - this.dmlStatement = dmlStatement; - } +public interface DMLGeneratorResponse { + String getDmlStatement(); + Boolean isPreparedStatement(); + List getValues(); } diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/models/PreparedStatementGeneratedResponse.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/models/PreparedStatementGeneratedResponse.java new file mode 100644 index 0000000000..80d0b22438 --- /dev/null +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/models/PreparedStatementGeneratedResponse.java @@ -0,0 +1,28 @@ +package com.google.cloud.teleport.v2.templates.models; + +import java.util.List; + +public class PreparedStatementGeneratedResponse implements DMLGeneratorResponse { + private String dmlStatement; + private List values; + + public PreparedStatementGeneratedResponse(String dmlStatement, List values) { + this.dmlStatement = dmlStatement; + this.values = values; + } + + @Override + public String getDmlStatement() { + return dmlStatement; + } + + @Override + public Boolean isPreparedStatement() { + return true; + } + + @Override + public List getValues() { + return values; + } +} diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/models/RawStatementGeneratedResponse.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/models/RawStatementGeneratedResponse.java new file mode 100644 index 0000000000..9220ec8193 --- /dev/null +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/models/RawStatementGeneratedResponse.java @@ -0,0 +1,25 @@ +package com.google.cloud.teleport.v2.templates.models; +import java.util.List; + +public class RawStatementGeneratedResponse implements DMLGeneratorResponse { + private String dmlStatement; + + public RawStatementGeneratedResponse(String dmlStatement) { + this.dmlStatement = dmlStatement; + } + + @Override + public String getDmlStatement() { + return dmlStatement; + } + + @Override + public Boolean isPreparedStatement() { + return false; + } + + @Override + public List getValues() { + return List.of(); + } +} \ No newline at end of file diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/transforms/AssignShardIdFn.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/transforms/AssignShardIdFn.java index 76b78d4a4e..b8e551d346 100644 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/transforms/AssignShardIdFn.java +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/transforms/AssignShardIdFn.java @@ -37,13 +37,6 @@ import com.google.cloud.teleport.v2.templates.constants.Constants; import com.google.cloud.teleport.v2.templates.utils.ShardingLogicImplFetcher; import com.google.common.collect.ImmutableList; -import java.io.PrintWriter; -import java.io.StringWriter; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; import org.apache.beam.sdk.io.gcp.spanner.SpannerAccessor; import org.apache.beam.sdk.io.gcp.spanner.SpannerConfig; import org.apache.beam.sdk.io.gcp.spanner.changestreams.model.ModType; @@ -56,13 +49,24 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -/** This DoFn assigns the shardId as key to the record. */ +import java.io.PrintWriter; +import java.io.StringWriter; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * This DoFn assigns the shardId as key to the record. + */ public class AssignShardIdFn extends DoFn> { private static final Logger LOG = LoggerFactory.getLogger(AssignShardIdFn.class); private final SpannerConfig spannerConfig; + /* SpannerAccessor must be transient so that its value is not serialized at runtime. */ private transient SpannerAccessor spannerAccessor; @@ -90,6 +94,11 @@ public class AssignShardIdFn private final Long maxConnectionsAcrossAllShards; + private final Long maxConnections; + + private String sourceType; + + public AssignShardIdFn( SpannerConfig spannerConfig, Schema schema, @@ -100,7 +109,9 @@ public AssignShardIdFn( String customJarPath, String shardingCustomClassName, String shardingCustomParameters, - Long maxConnectionsAcrossAllShards) { + Long maxConnectionsAcrossAllShards, + String sourceType, + Long maxConnections) { this.spannerConfig = spannerConfig; this.schema = schema; this.ddl = ddl; @@ -111,6 +122,8 @@ public AssignShardIdFn( this.shardingCustomClassName = shardingCustomClassName; this.shardingCustomParameters = shardingCustomParameters; this.maxConnectionsAcrossAllShards = maxConnectionsAcrossAllShards; + this.sourceType = sourceType; + this.maxConnections = maxConnections; } // setSpannerAccessor is added to be used by unit tests @@ -128,7 +141,9 @@ public void setShardIdFetcher(IShardIdFetcher shardIdFetcher) { this.shardIdFetcher = shardIdFetcher; } - /** Setup function connects to Cloud Spanner. */ + /** + * Setup function connects to Cloud Spanner. + */ @Setup public void setup() { boolean retry = true; @@ -162,7 +177,9 @@ public void setup() { } } - /** Teardown function disconnects from the Cloud Spanner. */ + /** + * Teardown function disconnects from the Cloud Spanner. + */ @Teardown public void teardown() { if (spannerConfig != null) { @@ -181,59 +198,65 @@ public void processElement(ProcessContext c) throws Exception { String qualifiedShard = ""; String tableName = record.getTableName(); String keysJsonStr = record.getMod().getKeysJson(); + Long finalKey; try { - if (shardingMode.equals(Constants.SHARDING_MODE_SINGLE_SHARD)) { - record.setShard(this.shardName); - qualifiedShard = this.shardName; - } else { - // Skip from processing if table not in session File - String shardIdColumn = getShardIdColumnForTableName(tableName); - if (shardIdColumn.isEmpty()) { - LOG.warn( - "Writing record for table {} to skipped directory name {} since table not present in" - + " the session file.", - tableName, - skipDirName); - record.setShard(skipDirName); - qualifiedShard = skipDirName; + if ("mysql".equals(this.sourceType)) { + if (shardingMode.equals(Constants.SHARDING_MODE_SINGLE_SHARD)) { + record.setShard(this.shardName); + qualifiedShard = this.shardName; } else { - - JsonNode keysJson = mapper.readTree(keysJsonStr); - String newValueJsonStr = record.getMod().getNewValuesJson(); - JsonNode newValueJson = mapper.readTree(newValueJsonStr); - Map spannerRecord = new HashMap<>(); - // Query the spanner database in case of a DELETE event - if (record.getModType() == ModType.DELETE) { - spannerRecord = - fetchSpannerRecord( - tableName, - record.getCommitTimestamp(), - record.getServerTransactionId(), - keysJson); + // Skip from processing if table not in session File + String shardIdColumn = getShardIdColumnForTableName(tableName); + if (shardIdColumn.isEmpty()) { + LOG.warn( + "Writing record for table {} to skipped directory name {} since table not present in" + + " the session file.", + tableName, + skipDirName); + record.setShard(skipDirName); + qualifiedShard = skipDirName; } else { - spannerRecord = getSpannerRecordFromChangeStreamData(tableName, keysJson, newValueJson); - } - ShardIdRequest shardIdRequest = new ShardIdRequest(tableName, spannerRecord); - - ShardIdResponse shardIdResponse = getShardIdResponse(shardIdRequest); - qualifiedShard = shardIdResponse.getLogicalShardId(); - if (qualifiedShard == null || qualifiedShard.isEmpty() || qualifiedShard.contains("/")) { - throw new IllegalArgumentException( - "Invalid logical shard id value: " - + qualifiedShard - + " for spanner table: " - + record.getTableName()); + JsonNode keysJson = mapper.readTree(keysJsonStr); + String newValueJsonStr = record.getMod().getNewValuesJson(); + JsonNode newValueJson = mapper.readTree(newValueJsonStr); + Map spannerRecord = new HashMap<>(); + // Query the spanner database in case of a DELETE event + if (record.getModType() == ModType.DELETE) { + spannerRecord = + fetchSpannerRecord( + tableName, + record.getCommitTimestamp(), + record.getServerTransactionId(), + keysJson); + } else { + spannerRecord = getSpannerRecordFromChangeStreamData(tableName, keysJson, newValueJson); + } + ShardIdRequest shardIdRequest = new ShardIdRequest(tableName, spannerRecord); + + ShardIdResponse shardIdResponse = getShardIdResponse(shardIdRequest); + + qualifiedShard = shardIdResponse.getLogicalShardId(); + if (qualifiedShard == null || qualifiedShard.isEmpty() || qualifiedShard.contains("/")) { + throw new IllegalArgumentException( + "Invalid logical shard id value: " + + qualifiedShard + + " for spanner table: " + + record.getTableName()); + } } } - } - record.setShard(qualifiedShard); - String finalKeyString = tableName + "_" + keysJsonStr + "_" + qualifiedShard; - Long finalKey = - finalKeyString.hashCode() % maxConnectionsAcrossAllShards; // The total parallelism is - // maxConnectionsAcrossAllShards + record.setShard(qualifiedShard); + String finalKeyString = tableName + "_" + keysJsonStr + "_" + qualifiedShard; + finalKey = finalKeyString.hashCode() % maxConnectionsAcrossAllShards; + } else { + record.setShard(this.shardName); + qualifiedShard = this.shardName; + String finalKeyString = tableName + "_" + keysJsonStr + "_" + qualifiedShard; + finalKey = finalKeyString.hashCode() % maxConnections; + } c.output(KV.of(finalKey, record)); } catch (Exception e) { @@ -242,7 +265,7 @@ public void processElement(ProcessContext c) throws Exception { LOG.error("Error fetching shard Id column: " + e.getMessage() + ": " + errors.toString()); // The record has no shard hence will be sent to DLQ in subsequent steps String finalKeyString = record.getTableName() + "_" + keysJsonStr + "_" + skipDirName; - Long finalKey = finalKeyString.hashCode() % maxConnectionsAcrossAllShards; + finalKey = finalKeyString.hashCode() % maxConnectionsAcrossAllShards; c.output(KV.of(finalKey, record)); } } diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/transforms/SourceWriterFn.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/transforms/SourceWriterFn.java index f4af6d9780..56edf72bc2 100644 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/transforms/SourceWriterFn.java +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/transforms/SourceWriterFn.java @@ -15,6 +15,8 @@ */ package com.google.cloud.teleport.v2.templates.transforms; +import com.datastax.driver.core.Cluster; +import com.datastax.driver.core.Session; import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; @@ -26,7 +28,7 @@ import com.google.cloud.teleport.v2.spanner.migrations.convertors.ChangeEventSpannerConvertor; import com.google.cloud.teleport.v2.spanner.migrations.exceptions.ChangeEventConvertorException; import com.google.cloud.teleport.v2.spanner.migrations.schema.Schema; -import com.google.cloud.teleport.v2.spanner.migrations.shard.Shard; +import com.google.cloud.teleport.v2.spanner.migrations.shard.IShard; import com.google.cloud.teleport.v2.templates.changestream.ChangeStreamErrorRecord; import com.google.cloud.teleport.v2.templates.changestream.TrimmedShardedDataChangeRecord; import com.google.cloud.teleport.v2.templates.constants.Constants; @@ -40,11 +42,6 @@ import com.google.cloud.teleport.v2.templates.utils.ShadowTableRecord; import com.google.common.collect.ImmutableList; import com.google.gson.Gson; -import java.io.Serializable; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import java.util.stream.Collectors; import org.apache.beam.sdk.io.gcp.spanner.SpannerConfig; import org.apache.beam.sdk.metrics.Counter; import org.apache.beam.sdk.metrics.Distribution; @@ -55,6 +52,12 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.Serializable; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + /** This class writes to source based on commit timestamp captured in shadow table. */ public class SourceWriterFn extends DoFn, String> implements Serializable { @@ -77,7 +80,7 @@ public class SourceWriterFn extends DoFn shards; + private final List iShards; private final SpannerConfig spannerConfig; private transient SpannerDao spannerDao; private final Ddl ddl; @@ -86,9 +89,12 @@ public class SourceWriterFn extends DoFn shards, + List iShards, Schema schema, SpannerConfig spannerConfig, String sourceDbTimezoneOffset, @@ -96,11 +102,12 @@ public SourceWriterFn( String shadowTablePrefix, String skipDirName, int maxThreadPerDataflowWorker, - String source) { + String source + ) { this.schema = schema; this.sourceDbTimezoneOffset = sourceDbTimezoneOffset; - this.shards = shards; + this.iShards = iShards; this.spannerConfig = spannerConfig; this.ddl = ddl; this.shadowTablePrefix = shadowTablePrefix; @@ -130,7 +137,7 @@ public void setup() throws UnsupportedSourceException { mapper = new ObjectMapper(); mapper.enable(DeserializationFeature.USE_BIG_DECIMAL_FOR_FLOATS); sourceProcessor = - SourceProcessorFactory.createSourceProcessor(source, shards, maxThreadPerDataflowWorker); + SourceProcessorFactory.createSourceProcessor(source, iShards, maxThreadPerDataflowWorker); spannerDao = new SpannerDao(spannerConfig); } diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/transforms/SourceWriterTransform.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/transforms/SourceWriterTransform.java index e80f617eeb..4609877b9d 100644 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/transforms/SourceWriterTransform.java +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/transforms/SourceWriterTransform.java @@ -18,43 +18,40 @@ import com.google.auto.value.AutoValue; import com.google.cloud.teleport.v2.spanner.ddl.Ddl; import com.google.cloud.teleport.v2.spanner.migrations.schema.Schema; -import com.google.cloud.teleport.v2.spanner.migrations.shard.Shard; +import com.google.cloud.teleport.v2.spanner.migrations.shard.IShard; import com.google.cloud.teleport.v2.templates.changestream.TrimmedShardedDataChangeRecord; import com.google.cloud.teleport.v2.templates.constants.Constants; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; -import java.util.List; -import java.util.Map; import org.apache.beam.sdk.Pipeline; import org.apache.beam.sdk.io.gcp.spanner.SpannerConfig; import org.apache.beam.sdk.transforms.PTransform; import org.apache.beam.sdk.transforms.ParDo; -import org.apache.beam.sdk.values.KV; -import org.apache.beam.sdk.values.PCollection; -import org.apache.beam.sdk.values.PCollectionTuple; -import org.apache.beam.sdk.values.PInput; -import org.apache.beam.sdk.values.POutput; -import org.apache.beam.sdk.values.PValue; -import org.apache.beam.sdk.values.TupleTag; -import org.apache.beam.sdk.values.TupleTagList; - -/** Takes an input of change stream events and writes them to the source database. */ +import org.apache.beam.sdk.values.*; + +import java.util.List; +import java.util.Map; + +/** + * Takes an input of change stream events and writes them to the source database. + */ public class SourceWriterTransform extends PTransform< - PCollection>, SourceWriterTransform.Result> { + PCollection>, SourceWriterTransform.Result> { private final Schema schema; private final String sourceDbTimezoneOffset; - private final List shards; + private final List iShards; private final SpannerConfig spannerConfig; private final Ddl ddl; private final String shadowTablePrefix; private final String skipDirName; private final int maxThreadPerDataflowWorker; private final String source; + private final Long maxConnections; public SourceWriterTransform( - List shards, + List iShards, Schema schema, SpannerConfig spannerConfig, String sourceDbTimezoneOffset, @@ -62,41 +59,44 @@ public SourceWriterTransform( String shadowTablePrefix, String skipDirName, int maxThreadPerDataflowWorker, - String source) { + String source, + Long maxConnections) { this.schema = schema; this.sourceDbTimezoneOffset = sourceDbTimezoneOffset; - this.shards = shards; + this.iShards = iShards; this.spannerConfig = spannerConfig; this.ddl = ddl; this.shadowTablePrefix = shadowTablePrefix; this.skipDirName = skipDirName; this.maxThreadPerDataflowWorker = maxThreadPerDataflowWorker; this.source = source; + this.maxConnections = maxConnections; } @Override public SourceWriterTransform.Result expand( PCollection> input) { - PCollectionTuple sourceWriteResults = - input.apply( + PCollectionTuple sourceWriteResults; + sourceWriteResults = input.apply( "Write to sourcedb", ParDo.of( - new SourceWriterFn( - this.shards, - this.schema, - this.spannerConfig, - this.sourceDbTimezoneOffset, - this.ddl, - this.shadowTablePrefix, - this.skipDirName, - this.maxThreadPerDataflowWorker, - this.source)) - .withOutputTags( - Constants.SUCCESS_TAG, - TupleTagList.of(Constants.PERMANENT_ERROR_TAG) - .and(Constants.RETRYABLE_ERROR_TAG) - .and(Constants.SKIPPED_TAG))); + new SourceWriterFn( + this.iShards, + this.schema, + this.spannerConfig, + this.sourceDbTimezoneOffset, + this.ddl, + this.shadowTablePrefix, + this.skipDirName, + this.maxThreadPerDataflowWorker, + this.source + )) + .withOutputTags( + Constants.SUCCESS_TAG, + TupleTagList.of(Constants.PERMANENT_ERROR_TAG) + .and(Constants.RETRYABLE_ERROR_TAG) + .and(Constants.SKIPPED_TAG))); return Result.create( sourceWriteResults.get(Constants.SUCCESS_TAG), @@ -105,7 +105,9 @@ public SourceWriterTransform.Result expand( sourceWriteResults.get(Constants.SKIPPED_TAG)); } - /** Container class for the results of this transform. */ + /** + * Container class for the results of this transform. + */ @AutoValue public abstract static class Result implements POutput { diff --git a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/SpannerToSourceDbCustomShardIT.java b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/SpannerToSourceDbCustomMySqlShardIT.java similarity index 79% rename from v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/SpannerToSourceDbCustomShardIT.java rename to v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/SpannerToSourceDbCustomMySqlShardIT.java index 6f56432066..baad0f4512 100644 --- a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/SpannerToSourceDbCustomShardIT.java +++ b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/SpannerToSourceDbCustomMySqlShardIT.java @@ -22,7 +22,7 @@ import com.google.cloud.spanner.Mutation; import com.google.cloud.teleport.metadata.SkipDirectRunnerTest; import com.google.cloud.teleport.metadata.TemplateIntegrationTest; -import com.google.cloud.teleport.v2.spanner.migrations.shard.Shard; +import com.google.cloud.teleport.v2.spanner.migrations.shard.MySqlShard; import com.google.common.io.Resources; import com.google.gson.Gson; import com.google.gson.JsonArray; @@ -53,9 +53,9 @@ @Category({TemplateIntegrationTest.class, SkipDirectRunnerTest.class}) @TemplateIntegrationTest(SpannerToSourceDb.class) @RunWith(JUnit4.class) -public class SpannerToSourceDbCustomShardIT extends SpannerToSourceDbITBase { +public class SpannerToSourceDbCustomMySqlShardIT extends SpannerToSourceDbITBase { - private static final Logger LOG = LoggerFactory.getLogger(SpannerToSourceDbCustomShardIT.class); + private static final Logger LOG = LoggerFactory.getLogger(SpannerToSourceDbCustomMySqlShardIT.class); private static final String SPANNER_DDL_RESOURCE = "SpannerToSourceDbCustomShardIT/spanner-schema.sql"; @@ -64,7 +64,7 @@ public class SpannerToSourceDbCustomShardIT extends SpannerToSourceDbITBase { "SpannerToSourceDbCustomShardIT/mysql-schema.sql"; private static final String TABLE = "Users"; - private static final HashSet testInstances = new HashSet<>(); + private static final HashSet testInstances = new HashSet<>(); private static PipelineLauncher.LaunchInfo jobInfo; public static SpannerResourceManager spannerResourceManager; private static SpannerResourceManager spannerMetadataResourceManager; @@ -82,21 +82,21 @@ public class SpannerToSourceDbCustomShardIT extends SpannerToSourceDbITBase { @Before public void setUp() throws IOException, InterruptedException { skipBaseCleanup = true; - synchronized (SpannerToSourceDbCustomShardIT.class) { + synchronized (SpannerToSourceDbCustomMySqlShardIT.class) { testInstances.add(this); if (jobInfo == null) { spannerResourceManager = - createSpannerDatabase(SpannerToSourceDbCustomShardIT.SPANNER_DDL_RESOURCE); + createSpannerDatabase(SpannerToSourceDbCustomMySqlShardIT.SPANNER_DDL_RESOURCE); spannerMetadataResourceManager = createSpannerMetadataDatabase(); jdbcResourceManagerShardA = MySQLResourceManager.builder(testName + "shardA").build(); createMySQLSchema( - jdbcResourceManagerShardA, SpannerToSourceDbCustomShardIT.MYSQL_SCHEMA_FILE_RESOURCE); + jdbcResourceManagerShardA, SpannerToSourceDbCustomMySqlShardIT.MYSQL_SCHEMA_FILE_RESOURCE); jdbcResourceManagerShardB = MySQLResourceManager.builder(testName + "shardB").build(); createMySQLSchema( - jdbcResourceManagerShardB, SpannerToSourceDbCustomShardIT.MYSQL_SCHEMA_FILE_RESOURCE); + jdbcResourceManagerShardB, SpannerToSourceDbCustomMySqlShardIT.MYSQL_SCHEMA_FILE_RESOURCE); gcsResourceManager = GcsResourceManager.builder(artifactBucketName, getClass().getSimpleName(), credentials) @@ -106,7 +106,7 @@ public void setUp() throws IOException, InterruptedException { createAndUploadShardConfigToGcs(); gcsResourceManager.uploadArtifact( "input/session.json", - Resources.getResource(SpannerToSourceDbCustomShardIT.SESSION_FILE_RESOURCE).getPath()); + Resources.getResource(SpannerToSourceDbCustomMySqlShardIT.SESSION_FILE_RESOURCE).getPath()); pubsubResourceManager = setUpPubSubResourceManager(); subscriptionName = createPubsubResources( @@ -134,7 +134,7 @@ public void setUp() throws IOException, InterruptedException { */ @AfterClass public static void cleanUp() throws IOException { - for (SpannerToSourceDbCustomShardIT instance : testInstances) { + for (SpannerToSourceDbCustomMySqlShardIT instance : testInstances) { instance.tearDownBase(); } ResourceManagerUtils.cleanResources( @@ -204,30 +204,30 @@ private void writeSpannerDataForSingers(int singerId, String firstName, String s } private void createAndUploadShardConfigToGcs() throws IOException { - Shard shard = new Shard(); - shard.setLogicalShardId("testShardA"); - shard.setUser(jdbcResourceManagerShardA.getUsername()); - shard.setHost(jdbcResourceManagerShardA.getHost()); - shard.setPassword(jdbcResourceManagerShardA.getPassword()); - shard.setPort(String.valueOf(jdbcResourceManagerShardA.getPort())); - shard.setDbName(jdbcResourceManagerShardA.getDatabaseName()); - JsonObject jsObj = (JsonObject) new Gson().toJsonTree(shard).getAsJsonObject(); + MySqlShard mySqlShard = new MySqlShard(); + mySqlShard.setLogicalShardId("testShardA"); + mySqlShard.setUser(jdbcResourceManagerShardA.getUsername()); + mySqlShard.setHost(jdbcResourceManagerShardA.getHost()); + mySqlShard.setPassword(jdbcResourceManagerShardA.getPassword()); + mySqlShard.setPort(String.valueOf(jdbcResourceManagerShardA.getPort())); + mySqlShard.setDbName(jdbcResourceManagerShardA.getDatabaseName()); + JsonObject jsObj = (JsonObject) new Gson().toJsonTree(mySqlShard).getAsJsonObject(); jsObj.remove("secretManagerUri"); // remove field secretManagerUri - Shard shardB = new Shard(); - shardB.setLogicalShardId("testShardB"); - shardB.setUser(jdbcResourceManagerShardB.getUsername()); - shardB.setHost(jdbcResourceManagerShardB.getHost()); - shardB.setPassword(jdbcResourceManagerShardB.getPassword()); - shardB.setPort(String.valueOf(jdbcResourceManagerShardB.getPort())); - shardB.setDbName(jdbcResourceManagerShardB.getDatabaseName()); - JsonObject jsObjB = (JsonObject) new Gson().toJsonTree(shardB).getAsJsonObject(); + MySqlShard mySqlShardB = new MySqlShard(); + mySqlShardB.setLogicalShardId("testShardB"); + mySqlShardB.setUser(jdbcResourceManagerShardB.getUsername()); + mySqlShardB.setHost(jdbcResourceManagerShardB.getHost()); + mySqlShardB.setPassword(jdbcResourceManagerShardB.getPassword()); + mySqlShardB.setPort(String.valueOf(jdbcResourceManagerShardB.getPort())); + mySqlShardB.setDbName(jdbcResourceManagerShardB.getDatabaseName()); + JsonObject jsObjB = (JsonObject) new Gson().toJsonTree(mySqlShardB).getAsJsonObject(); jsObjB.remove("secretManagerUri"); // remove field secretManagerUri JsonArray ja = new JsonArray(); ja.add(jsObj); ja.add(jsObjB); String shardFileContents = ja.toString(); - LOG.info("Shard file contents: {}", shardFileContents); - gcsResourceManager.createArtifact("input/shard.json", shardFileContents); + LOG.info("MySqlShard file contents: {}", shardFileContents); + gcsResourceManager.createArtifact("input/mySqlShard.json", shardFileContents); } } diff --git a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/SpannerToSourceDbITBase.java b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/SpannerToSourceDbITBase.java index 781b4c4a2e..1518a1d11a 100644 --- a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/SpannerToSourceDbITBase.java +++ b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/SpannerToSourceDbITBase.java @@ -17,7 +17,7 @@ import static org.apache.beam.it.truthmatchers.PipelineAsserts.assertThatPipeline; -import com.google.cloud.teleport.v2.spanner.migrations.shard.Shard; +import com.google.cloud.teleport.v2.spanner.migrations.shard.MySqlShard; import com.google.common.io.Resources; import com.google.gson.Gson; import com.google.gson.JsonArray; @@ -99,20 +99,20 @@ public SubscriptionName createPubsubResources( protected void createAndUploadShardConfigToGcs( GcsResourceManager gcsResourceManager, MySQLResourceManager jdbcResourceManager) throws IOException { - Shard shard = new Shard(); - shard.setLogicalShardId("Shard1"); - shard.setUser(jdbcResourceManager.getUsername()); - shard.setHost(jdbcResourceManager.getHost()); - shard.setPassword(jdbcResourceManager.getPassword()); - shard.setPort(String.valueOf(jdbcResourceManager.getPort())); - shard.setDbName(jdbcResourceManager.getDatabaseName()); - JsonObject jsObj = new Gson().toJsonTree(shard).getAsJsonObject(); + MySqlShard mySqlShard = new MySqlShard(); + mySqlShard.setLogicalShardId("Shard1"); + mySqlShard.setUser(jdbcResourceManager.getUsername()); + mySqlShard.setHost(jdbcResourceManager.getHost()); + mySqlShard.setPassword(jdbcResourceManager.getPassword()); + mySqlShard.setPort(String.valueOf(jdbcResourceManager.getPort())); + mySqlShard.setDbName(jdbcResourceManager.getDatabaseName()); + JsonObject jsObj = new Gson().toJsonTree(mySqlShard).getAsJsonObject(); jsObj.remove("secretManagerUri"); // remove field secretManagerUri JsonArray ja = new JsonArray(); ja.add(jsObj); String shardFileContents = ja.toString(); - LOG.info("Shard file contents: {}", shardFileContents); - gcsResourceManager.createArtifact("input/shard.json", shardFileContents); + LOG.info("MySqlShard file contents: {}", shardFileContents); + gcsResourceManager.createArtifact("input/mySqlShard.json", shardFileContents); } public PipelineLauncher.LaunchInfo launchDataflowJob( diff --git a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/SpannerToSourceDbInterleaveMultiShardIT.java b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/SpannerToSourceDbInterleaveMultiMySqlShardIT.java similarity index 87% rename from v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/SpannerToSourceDbInterleaveMultiShardIT.java rename to v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/SpannerToSourceDbInterleaveMultiMySqlShardIT.java index 25a1b1b991..bcc7587b50 100644 --- a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/SpannerToSourceDbInterleaveMultiShardIT.java +++ b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/SpannerToSourceDbInterleaveMultiMySqlShardIT.java @@ -24,7 +24,7 @@ import com.google.cloud.spanner.Mutation; import com.google.cloud.teleport.metadata.SkipDirectRunnerTest; import com.google.cloud.teleport.metadata.TemplateIntegrationTest; -import com.google.cloud.teleport.v2.spanner.migrations.shard.Shard; +import com.google.cloud.teleport.v2.spanner.migrations.shard.MySqlShard; import com.google.common.io.Resources; import com.google.gson.Gson; import com.google.gson.JsonArray; @@ -56,10 +56,10 @@ @Category({TemplateIntegrationTest.class, SkipDirectRunnerTest.class}) @TemplateIntegrationTest(SpannerToSourceDb.class) @RunWith(JUnit4.class) -public class SpannerToSourceDbInterleaveMultiShardIT extends SpannerToSourceDbITBase { +public class SpannerToSourceDbInterleaveMultiMySqlShardIT extends SpannerToSourceDbITBase { private static final Logger LOG = - LoggerFactory.getLogger(SpannerToSourceDbInterleaveMultiShardIT.class); + LoggerFactory.getLogger(SpannerToSourceDbInterleaveMultiMySqlShardIT.class); private static final String SPANNER_DDL_RESOURCE = "SpannerToSourceDbInterleaveMultiShardIT/spanner-schema.sql"; @@ -68,7 +68,7 @@ public class SpannerToSourceDbInterleaveMultiShardIT extends SpannerToSourceDbIT private static final String MYSQL_DDL_RESOURCE = "SpannerToSourceDbInterleaveMultiShardIT/mysql-schema.sql"; - private static HashSet testInstances = new HashSet<>(); + private static HashSet testInstances = new HashSet<>(); private static PipelineLauncher.LaunchInfo jobInfo; public static SpannerResourceManager spannerResourceManager; private static SpannerResourceManager spannerMetadataResourceManager; @@ -86,20 +86,20 @@ public class SpannerToSourceDbInterleaveMultiShardIT extends SpannerToSourceDbIT @Before public void setUp() throws IOException { skipBaseCleanup = true; - synchronized (SpannerToSourceDbInterleaveMultiShardIT.class) { + synchronized (SpannerToSourceDbInterleaveMultiMySqlShardIT.class) { testInstances.add(this); if (jobInfo == null) { spannerResourceManager = - createSpannerDatabase(SpannerToSourceDbInterleaveMultiShardIT.SPANNER_DDL_RESOURCE); + createSpannerDatabase(SpannerToSourceDbInterleaveMultiMySqlShardIT.SPANNER_DDL_RESOURCE); spannerMetadataResourceManager = createSpannerMetadataDatabase(); jdbcResourceManagerShardA = MySQLResourceManager.builder(testName + "shardA").build(); createMySQLSchema( - jdbcResourceManagerShardA, SpannerToSourceDbInterleaveMultiShardIT.MYSQL_DDL_RESOURCE); + jdbcResourceManagerShardA, SpannerToSourceDbInterleaveMultiMySqlShardIT.MYSQL_DDL_RESOURCE); jdbcResourceManagerShardB = MySQLResourceManager.builder(testName + "shardB").build(); createMySQLSchema( - jdbcResourceManagerShardB, SpannerToSourceDbInterleaveMultiShardIT.MYSQL_DDL_RESOURCE); + jdbcResourceManagerShardB, SpannerToSourceDbInterleaveMultiMySqlShardIT.MYSQL_DDL_RESOURCE); gcsResourceManager = GcsResourceManager.builder(artifactBucketName, getClass().getSimpleName(), credentials) @@ -135,7 +135,7 @@ public void setUp() throws IOException { */ @AfterClass public static void cleanUp() throws IOException { - for (SpannerToSourceDbInterleaveMultiShardIT instance : testInstances) { + for (SpannerToSourceDbInterleaveMultiMySqlShardIT instance : testInstances) { instance.tearDownBase(); } ResourceManagerUtils.cleanResources( @@ -355,30 +355,30 @@ private void assertDeletedRowsInMySQL() throws InterruptedException { } private void createAndUploadShardConfigToGcs() throws IOException { - Shard shard = new Shard(); - shard.setLogicalShardId("shardA"); - shard.setUser(jdbcResourceManagerShardA.getUsername()); - shard.setHost(jdbcResourceManagerShardA.getHost()); - shard.setPassword(jdbcResourceManagerShardA.getPassword()); - shard.setPort(String.valueOf(jdbcResourceManagerShardA.getPort())); - shard.setDbName(jdbcResourceManagerShardA.getDatabaseName()); - JsonObject jsObj = (JsonObject) new Gson().toJsonTree(shard).getAsJsonObject(); + MySqlShard mySqlShard = new MySqlShard(); + mySqlShard.setLogicalShardId("shardA"); + mySqlShard.setUser(jdbcResourceManagerShardA.getUsername()); + mySqlShard.setHost(jdbcResourceManagerShardA.getHost()); + mySqlShard.setPassword(jdbcResourceManagerShardA.getPassword()); + mySqlShard.setPort(String.valueOf(jdbcResourceManagerShardA.getPort())); + mySqlShard.setDbName(jdbcResourceManagerShardA.getDatabaseName()); + JsonObject jsObj = (JsonObject) new Gson().toJsonTree(mySqlShard).getAsJsonObject(); jsObj.remove("secretManagerUri"); // remove field secretManagerUri - Shard shardB = new Shard(); - shardB.setLogicalShardId("shardB"); - shardB.setUser(jdbcResourceManagerShardB.getUsername()); - shardB.setHost(jdbcResourceManagerShardB.getHost()); - shardB.setPassword(jdbcResourceManagerShardB.getPassword()); - shardB.setPort(String.valueOf(jdbcResourceManagerShardB.getPort())); - shardB.setDbName(jdbcResourceManagerShardB.getDatabaseName()); - JsonObject jsObjB = (JsonObject) new Gson().toJsonTree(shardB).getAsJsonObject(); + MySqlShard mySqlShardB = new MySqlShard(); + mySqlShardB.setLogicalShardId("mySqlShardB"); + mySqlShardB.setUser(jdbcResourceManagerShardB.getUsername()); + mySqlShardB.setHost(jdbcResourceManagerShardB.getHost()); + mySqlShardB.setPassword(jdbcResourceManagerShardB.getPassword()); + mySqlShardB.setPort(String.valueOf(jdbcResourceManagerShardB.getPort())); + mySqlShardB.setDbName(jdbcResourceManagerShardB.getDatabaseName()); + JsonObject jsObjB = (JsonObject) new Gson().toJsonTree(mySqlShardB).getAsJsonObject(); jsObjB.remove("secretManagerUri"); // remove field secretManagerUri JsonArray ja = new JsonArray(); ja.add(jsObj); ja.add(jsObjB); String shardFileContents = ja.toString(); - LOG.info("Shard file contents: {}", shardFileContents); - gcsResourceManager.createArtifact("input/shard.json", shardFileContents); + LOG.info("MySqlShard file contents: {}", shardFileContents); + gcsResourceManager.createArtifact("input/mySqlShard.json", shardFileContents); } } diff --git a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/SpannerToSourceDbLTBase.java b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/SpannerToSourceDbLTBase.java index 43c0453adf..443083ee64 100644 --- a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/SpannerToSourceDbLTBase.java +++ b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/SpannerToSourceDbLTBase.java @@ -15,7 +15,7 @@ */ package com.google.cloud.teleport.v2.templates; -import com.google.cloud.teleport.v2.spanner.migrations.shard.Shard; +import com.google.cloud.teleport.v2.spanner.migrations.shard.MySqlShard; import com.google.common.base.MoreObjects; import com.google.common.io.Resources; import com.google.gson.Gson; @@ -165,14 +165,14 @@ public void createAndUploadShardConfigToGcs( for (int i = 0; i < 1; ++i) { if (jdbcResourceManagers.get(i) instanceof MySQLResourceManager) { MySQLResourceManager resourceManager = (MySQLResourceManager) jdbcResourceManagers.get(i); - Shard shard = new Shard(); - shard.setLogicalShardId("Shard" + (i + 1)); - shard.setUser(jdbcResourceManagers.get(i).getUsername()); - shard.setHost(resourceManager.getHost()); - shard.setPassword(jdbcResourceManagers.get(i).getPassword()); - shard.setPort(String.valueOf(resourceManager.getPort())); - shard.setDbName(jdbcResourceManagers.get(i).getDatabaseName()); - JsonObject jsObj = (JsonObject) new Gson().toJsonTree(shard).getAsJsonObject(); + MySqlShard mySqlShard = new MySqlShard(); + mySqlShard.setLogicalShardId("MySqlShard" + (i + 1)); + mySqlShard.setUser(jdbcResourceManagers.get(i).getUsername()); + mySqlShard.setHost(resourceManager.getHost()); + mySqlShard.setPassword(jdbcResourceManagers.get(i).getPassword()); + mySqlShard.setPort(String.valueOf(resourceManager.getPort())); + mySqlShard.setDbName(jdbcResourceManagers.get(i).getDatabaseName()); + JsonObject jsObj = (JsonObject) new Gson().toJsonTree(mySqlShard).getAsJsonObject(); jsObj.remove("secretManagerUri"); // remove field secretManagerUri ja.add(jsObj); } else { @@ -181,7 +181,7 @@ public void createAndUploadShardConfigToGcs( } } String shardFileContents = ja.toString(); - LOG.info("Shard file contents: {}", shardFileContents); + LOG.info("MySqlShard file contents: {}", shardFileContents); gcsResourceManager.createArtifact("input/shard.json", shardFileContents); } diff --git a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/dbutils/dao/JdbcDaoTest.java b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/dbutils/dao/JdbcDaoTest.java index dad21878c6..445a5f910f 100644 --- a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/dbutils/dao/JdbcDaoTest.java +++ b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/dbutils/dao/JdbcDaoTest.java @@ -24,6 +24,11 @@ import com.google.cloud.teleport.v2.templates.dbutils.connection.JdbcConnectionHelper; import com.google.cloud.teleport.v2.templates.dbutils.dao.source.JdbcDao; import com.google.cloud.teleport.v2.templates.exceptions.ConnectionException; + +import com.google.cloud.teleport.v2.templates.dbutils.connection.JdbcConnectionHelper; +import com.google.cloud.teleport.v2.templates.dbutils.dao.source.JdbcDao; +import com.google.cloud.teleport.v2.templates.exceptions.ConnectionException; + import com.zaxxer.hikari.HikariDataSource; import java.sql.Connection; import java.sql.Statement; diff --git a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/dbutils/dao/SpannerDaoTest.java b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/dbutils/dao/SpannerDaoTest.java index 882c379587..7a1781e0fa 100644 --- a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/dbutils/dao/SpannerDaoTest.java +++ b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/dbutils/dao/SpannerDaoTest.java @@ -13,6 +13,7 @@ * License for the specific language governing permissions and limitations under * the License. */ + package com.google.cloud.teleport.v2.templates.dbutils.dao; import static com.google.common.truth.Truth.assertThat; diff --git a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/dbutils/dml/MySQLDMLGeneratorTest.java b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/dbutils/dml/MySQLDMLGeneratorTest.java index b01d139a6d..5eb8465ece 100644 --- a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/dbutils/dml/MySQLDMLGeneratorTest.java +++ b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/dbutils/dml/MySQLDMLGeneratorTest.java @@ -13,6 +13,7 @@ * License for the specific language governing permissions and limitations under * the License. */ + package com.google.cloud.teleport.v2.templates.dbutils.dml; import static org.junit.Assert.assertTrue; @@ -29,9 +30,11 @@ import com.google.cloud.teleport.v2.spanner.migrations.schema.SyntheticPKey; import com.google.cloud.teleport.v2.spanner.migrations.utils.SessionFileReader; import com.google.cloud.teleport.v2.templates.changestream.TrimmedShardedDataChangeRecord; + import com.google.cloud.teleport.v2.templates.dbutils.processor.InputRecordProcessor; import com.google.cloud.teleport.v2.templates.models.DMLGeneratorRequest; import com.google.cloud.teleport.v2.templates.models.DMLGeneratorResponse; + import com.google.gson.FieldNamingPolicy; import com.google.gson.GsonBuilder; import java.io.InputStream; diff --git a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/dbutils/processor/SourceProcessorFactoryTest.java b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/dbutils/processor/SourceProcessorFactoryTest.java index 3b9c0e64bf..f079dcdfcf 100644 --- a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/dbutils/processor/SourceProcessorFactoryTest.java +++ b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/dbutils/processor/SourceProcessorFactoryTest.java @@ -18,7 +18,7 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doNothing; -import com.google.cloud.teleport.v2.spanner.migrations.shard.Shard; +import com.google.cloud.teleport.v2.spanner.migrations.shard.MySqlShard; import com.google.cloud.teleport.v2.templates.constants.Constants; import com.google.cloud.teleport.v2.templates.dbutils.connection.JdbcConnectionHelper; import com.google.cloud.teleport.v2.templates.dbutils.dao.source.JdbcDao; @@ -37,9 +37,9 @@ public class SourceProcessorFactoryTest { @Test public void testCreateSourceProcessor_validSource() throws Exception { - List shards = + List mySqlShards = Arrays.asList( - new Shard( + new MySqlShard( "shard1", "localhost", "3306", @@ -56,7 +56,7 @@ public void testCreateSourceProcessor_validSource() throws Exception { Map.of(Constants.SOURCE_MYSQL, mockConnectionHelper)); SourceProcessor processor = SourceProcessorFactory.createSourceProcessor( - Constants.SOURCE_MYSQL, shards, maxConnections); + Constants.SOURCE_MYSQL, mySqlShards, maxConnections); Assert.assertNotNull(processor); Assert.assertTrue(processor.getDmlGenerator() instanceof MySQLDMLGenerator); @@ -66,9 +66,9 @@ public void testCreateSourceProcessor_validSource() throws Exception { @Test(expected = UnsupportedSourceException.class) public void testCreateSourceProcessor_invalidSource() throws Exception { - List shards = + List mySqlShards = Arrays.asList( - new Shard( + new MySqlShard( "shard1", "localhost", "3306", @@ -80,6 +80,6 @@ public void testCreateSourceProcessor_invalidSource() throws Exception { "")); int maxConnections = 10; - SourceProcessorFactory.createSourceProcessor("invalid_source", shards, maxConnections); + SourceProcessorFactory.createSourceProcessor("invalid_source", mySqlShards, maxConnections); } } diff --git a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/processor/SourceProcessorFactoryTest.java b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/processor/SourceProcessorFactoryTest.java new file mode 100644 index 0000000000..f80a52974d --- /dev/null +++ b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/processor/SourceProcessorFactoryTest.java @@ -0,0 +1,87 @@ +/* + * Copyright (C) 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package com.google.cloud.teleport.v2.templates.processor; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doNothing; + +import com.google.cloud.teleport.v2.spanner.migrations.shard.MySqlShard; +import com.google.cloud.teleport.v2.templates.constants.Constants; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +import com.google.cloud.teleport.v2.templates.dbutils.connection.JdbcConnectionHelper; +import com.google.cloud.teleport.v2.templates.dbutils.dao.source.JdbcDao; +import com.google.cloud.teleport.v2.templates.dbutils.dml.MySQLDMLGenerator; +import com.google.cloud.teleport.v2.templates.dbutils.processor.SourceProcessor; +import com.google.cloud.teleport.v2.templates.dbutils.processor.SourceProcessorFactory; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.mockito.Mockito; + +@RunWith(JUnit4.class) +public class SourceProcessorFactoryTest { + @Test + public void testCreateSourceProcessor_validSource() throws Exception { + List mySqlShards = + Arrays.asList( + new MySqlShard( + "shard1", + "localhost", + "3306", + "myuser", + "mypassword", + "mydatabase", + "mynamespace", + "projects/myproject/secrets/mysecret/versions/latest", + "")); + int maxConnections = 10; + JdbcConnectionHelper mockConnectionHelper = Mockito.mock(JdbcConnectionHelper.class); + doNothing().when(mockConnectionHelper).init(any()); + SourceProcessorFactory.setConnectionHelperMap( + Map.of(Constants.SOURCE_MYSQL, mockConnectionHelper)); + SourceProcessor processor = + SourceProcessorFactory.createSourceProcessor( + Constants.SOURCE_MYSQL, mySqlShards, maxConnections); + + Assert.assertNotNull(processor); + Assert.assertTrue(processor.getDmlGenerator() instanceof MySQLDMLGenerator); + Assert.assertEquals(1, processor.getSourceDaoMap().size()); + Assert.assertTrue(processor.getSourceDaoMap().get("shard1") instanceof JdbcDao); + } + + @Test(expected = InvalidSourceException.class) + public void testCreateSourceProcessor_invalidSource() throws Exception { + List mySqlShards = + Arrays.asList( + new MySqlShard( + "shard1", + "localhost", + "3306", + "myuser", + "mypassword", + "mydatabase", + "mynamespace", + "projects/myproject/secrets/mysecret/versions/latest", + "")); + int maxConnections = 10; + + SourceProcessorFactory.createSourceProcessor("invalid_source", mySqlShards, maxConnections); + } +} diff --git a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/transforms/AssignShardIdFnTest.java b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/transforms/AssignMySqlShardIdFnTest.java similarity index 99% rename from v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/transforms/AssignShardIdFnTest.java rename to v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/transforms/AssignMySqlShardIdFnTest.java index 1413be3d2e..1dc8329268 100644 --- a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/transforms/AssignShardIdFnTest.java +++ b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/transforms/AssignMySqlShardIdFnTest.java @@ -68,9 +68,9 @@ import org.mockito.junit.MockitoJUnit; import org.mockito.junit.MockitoRule; -/** Tests for AssignShardIdFnTest class. */ +/** Tests for AssignMySqlShardIdFnTest class. */ @FixMethodOrder(MethodSorters.NAME_ASCENDING) -public class AssignShardIdFnTest { +public class AssignMySqlShardIdFnTest { @Rule public final transient TestPipeline pipeline = TestPipeline.create(); @Rule public final MockitoRule mocktio = MockitoJUnit.rule(); diff --git a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/transforms/SourceWriterFnTest.java b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/transforms/SourceWriterFnTest.java index f2bac7247d..8ff4fa9a25 100644 --- a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/transforms/SourceWriterFnTest.java +++ b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/transforms/SourceWriterFnTest.java @@ -37,7 +37,7 @@ import com.google.cloud.teleport.v2.spanner.migrations.schema.SpannerColumnType; import com.google.cloud.teleport.v2.spanner.migrations.schema.SpannerTable; import com.google.cloud.teleport.v2.spanner.migrations.schema.SyntheticPKey; -import com.google.cloud.teleport.v2.spanner.migrations.shard.Shard; +import com.google.cloud.teleport.v2.spanner.migrations.shard.MySqlShard; import com.google.cloud.teleport.v2.spanner.migrations.utils.SessionFileReader; import com.google.cloud.teleport.v2.templates.changestream.ChangeStreamErrorRecord; import com.google.cloud.teleport.v2.templates.changestream.TrimmedShardedDataChangeRecord; @@ -78,7 +78,7 @@ public class SourceWriterFnTest { @Mock private DoFn.ProcessContext processContext; private static Gson gson = new Gson(); - private Shard testShard; + private MySqlShard testMySqlShard; private Schema testSchema; private Ddl testDdl; private String testSourceDbTimezoneOffset; @@ -115,13 +115,13 @@ public void doBeforeEachTest() throws Exception { .when(mockSqlDao) .write(contains("12345")); // to test code path of generic exception doNothing().when(mockSqlDao).write(contains("parent1")); - testShard = new Shard(); - testShard.setLogicalShardId("shardA"); - testShard.setUser("test"); - testShard.setHost("test"); - testShard.setPassword("test"); - testShard.setPort("1234"); - testShard.setDbName("test"); + testMySqlShard = new MySqlShard(); + testMySqlShard.setLogicalShardId("shardA"); + testMySqlShard.setUser("test"); + testMySqlShard.setHost("test"); + testMySqlShard.setPassword("test"); + testMySqlShard.setPort("1234"); + testMySqlShard.setDbName("test"); testSchema = SessionFileReader.read("src/test/resources/sourceWriterUTSession.json"); testSourceDbTimezoneOffset = "+00:00"; @@ -140,7 +140,7 @@ public void testSourceIsAhead() throws Exception { when(processContext.element()).thenReturn(KV.of(1L, record)); SourceWriterFn sourceWriterFn = new SourceWriterFn( - ImmutableList.of(testShard), + ImmutableList.of(testMySqlShard), testSchema, mockSpannerConfig, testSourceDbTimezoneOffset, @@ -167,7 +167,7 @@ public void testSourceIsAheadWithSameCommitTimestamp() throws Exception { when(processContext.element()).thenReturn(KV.of(1L, record)); SourceWriterFn sourceWriterFn = new SourceWriterFn( - ImmutableList.of(testShard), + ImmutableList.of(testMySqlShard), testSchema, mockSpannerConfig, testSourceDbTimezoneOffset, @@ -193,7 +193,7 @@ public void testSourceIsBehind() throws Exception { when(processContext.element()).thenReturn(KV.of(1L, record)); SourceWriterFn sourceWriterFn = new SourceWriterFn( - ImmutableList.of(testShard), + ImmutableList.of(testMySqlShard), testSchema, mockSpannerConfig, testSourceDbTimezoneOffset, @@ -219,7 +219,7 @@ public void testNoShard() throws Exception { when(processContext.element()).thenReturn(KV.of(1L, record)); SourceWriterFn sourceWriterFn = new SourceWriterFn( - ImmutableList.of(testShard), + ImmutableList.of(testMySqlShard), testSchema, mockSpannerConfig, testSourceDbTimezoneOffset, @@ -249,7 +249,7 @@ public void testSkipShard() throws Exception { when(processContext.element()).thenReturn(KV.of(1L, record)); SourceWriterFn sourceWriterFn = new SourceWriterFn( - ImmutableList.of(testShard), + ImmutableList.of(testMySqlShard), testSchema, mockSpannerConfig, testSourceDbTimezoneOffset, @@ -277,7 +277,7 @@ public void testPermanentError() throws Exception { when(processContext.element()).thenReturn(KV.of(1L, record)); SourceWriterFn sourceWriterFn = new SourceWriterFn( - ImmutableList.of(testShard), + ImmutableList.of(testMySqlShard), testSchema, mockSpannerConfig, testSourceDbTimezoneOffset, @@ -309,7 +309,7 @@ public void testRetryableError() throws Exception { when(processContext.element()).thenReturn(KV.of(1L, record)); SourceWriterFn sourceWriterFn = new SourceWriterFn( - ImmutableList.of(testShard), + ImmutableList.of(testMySqlShard), testSchema, mockSpannerConfig, testSourceDbTimezoneOffset, @@ -337,7 +337,7 @@ public void testRetryableErrorForForeignKey() throws Exception { when(processContext.element()).thenReturn(KV.of(1L, record)); SourceWriterFn sourceWriterFn = new SourceWriterFn( - ImmutableList.of(testShard), + ImmutableList.of(testMySqlShard), testSchema, mockSpannerConfig, testSourceDbTimezoneOffset, @@ -367,7 +367,7 @@ public void testRetryableErrorConnectionFailure() throws Exception { when(processContext.element()).thenReturn(KV.of(1L, record)); SourceWriterFn sourceWriterFn = new SourceWriterFn( - ImmutableList.of(testShard), + ImmutableList.of(testMySqlShard), testSchema, mockSpannerConfig, testSourceDbTimezoneOffset, @@ -397,7 +397,7 @@ public void testPermanentConnectionFailure() throws Exception { when(processContext.element()).thenReturn(KV.of(1L, record)); SourceWriterFn sourceWriterFn = new SourceWriterFn( - ImmutableList.of(testShard), + ImmutableList.of(testMySqlShard), testSchema, mockSpannerConfig, testSourceDbTimezoneOffset, @@ -427,7 +427,7 @@ public void testPermanentGenericException() throws Exception { when(processContext.element()).thenReturn(KV.of(1L, record)); SourceWriterFn sourceWriterFn = new SourceWriterFn( - ImmutableList.of(testShard), + ImmutableList.of(testMySqlShard), testSchema, mockSpannerConfig, testSourceDbTimezoneOffset, @@ -456,7 +456,7 @@ public void testDMLEmpty() throws Exception { when(processContext.element()).thenReturn(KV.of(1L, record)); SourceWriterFn sourceWriterFn = new SourceWriterFn( - ImmutableList.of(testShard), + ImmutableList.of(testMySqlShard), getSchemaObject(), mockSpannerConfig, testSourceDbTimezoneOffset, diff --git a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/utils/ShardIdFetcherImplTest.java b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/utils/MySqlShardIdFetcherImplTest.java similarity index 99% rename from v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/utils/ShardIdFetcherImplTest.java rename to v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/utils/MySqlShardIdFetcherImplTest.java index bc54bf38ee..d06b18450d 100644 --- a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/utils/ShardIdFetcherImplTest.java +++ b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/utils/MySqlShardIdFetcherImplTest.java @@ -35,7 +35,7 @@ import org.junit.runners.JUnit4; @RunWith(JUnit4.class) -public final class ShardIdFetcherImplTest { +public final class MySqlShardIdFetcherImplTest { @Test public void testSucessShardIdentification() { diff --git a/v2/spanner-to-sourcedb/terraform/Spanner_Change_Streams_to_Sharded_File_Sink/dataflow_job.tf b/v2/spanner-to-sourcedb/terraform/Spanner_Change_Streams_to_Sharded_File_Sink/dataflow_job.tf index e63faf45ec..4c23262ed8 100644 --- a/v2/spanner-to-sourcedb/terraform/Spanner_Change_Streams_to_Sharded_File_Sink/dataflow_job.tf +++ b/v2/spanner-to-sourcedb/terraform/Spanner_Change_Streams_to_Sharded_File_Sink/dataflow_job.tf @@ -107,7 +107,7 @@ variable "filtrationMode" { variable "sourceShardsFilePath" { type = string - description = "Source shard details file path in Cloud Storage that contains connection profile of source shards. Atleast one shard information is expected." + description = "Source mySqlShard details file path in Cloud Storage that contains connection profile of source mySqlShards. Atleast one mySqlShard information is expected." } @@ -137,13 +137,13 @@ variable "runMode" { variable "shardingCustomJarPath" { type = string - description = "Custom jar location in Cloud Storage that contains the customization logic for fetching shard id. Defaults to empty." + description = "Custom jar location in Cloud Storage that contains the customization logic for fetching mySqlShard id. Defaults to empty." default = null } variable "shardingCustomClassName" { type = string - description = "Fully qualified class name having the custom shard id implementation. It is a mandatory field in case shardingCustomJarPath is specified. Defaults to empty." + description = "Fully qualified class name having the custom mySqlShard id implementation. It is a mandatory field in case shardingCustomJarPath is specified. Defaults to empty." default = null } diff --git a/v2/spanner-to-sourcedb/terraform/samples/spanner-to-sharded-mysql/README.md b/v2/spanner-to-sourcedb/terraform/samples/spanner-to-sharded-mysql/README.md index d7a20ba662..dea3afc184 100644 --- a/v2/spanner-to-sourcedb/terraform/samples/spanner-to-sharded-mysql/README.md +++ b/v2/spanner-to-sourcedb/terraform/samples/spanner-to-sharded-mysql/README.md @@ -2,7 +2,7 @@ > **_SCENARIO:_** This Terraform example illustrates launching a reverse replication > jobs to replicate spanner writes for a sharded MySQL source, setting up all the required cloud infrastructure. -> **Details of MySQL shards are needed as input.** +> **Details of MySQL mySqlShards are needed as input.** ## Terraform permissions @@ -143,10 +143,10 @@ Given these assumptions, it uses a supplied source database connection configuration and creates the following resources - 1. **Firewall rules** - These rules allow Dataflow VMs to connect to each other - and allow Dataflow VMs to connect to the source MySQL shards. + and allow Dataflow VMs to connect to the source MySQL mySqlShards. 2. **GCS buckets** - A GCS bucket to hold reverse replication metadata, such as - session and source shards files. -3. **GCS objects** - Generates source shards configuration file to upload to + session and source mySqlShards files. +3. **GCS objects** - Generates source mySqlShards configuration file to upload to GCS. 4. **Pubsub topic and subscription** - This contains GCS object notifications as files are written to GCS for DLQ retrials. @@ -298,9 +298,9 @@ If you want Terraform to only create the GCS bucket but skip its deletion during `terraform destroy`, you will have to use the `terraform state rm` API to delete GCS resource from being traced by Terraform after the `apply` command. -### Source shard configuration file +### Source mySqlShard configuration file -Source shard configuration file that is supplied to the dataflow is automatically +Source mySqlShard configuration file that is supplied to the dataflow is automatically created by Terraform. A sample file that this uploaded to GCS looks like below - @@ -309,18 +309,18 @@ below - ```json [ { - "logicalShardId": "shard1", + "logicalShardId": "mySqlShard1", "host": "10.11.12.13", "user": "root", - "secretManagerUri":"projects/123/secrets/rev-cmek-cred-shard1/versions/latest", + "secretManagerUri":"projects/123/secrets/rev-cmek-cred-mySqlShard1/versions/latest", "port": "3306", "dbName": "db1" }, { - "logicalShardId": "shard2", + "logicalShardId": "mySqlShard2", "host": "10.11.12.14", "user": "root", - "secretManagerUri":"projects/123/secrets/rev-cmek-cred-shard2/versions/latest", + "secretManagerUri":"projects/123/secrets/rev-cmek-cred-mySqlShard2/versions/latest", "port": "3306", "dbName": "db2" } @@ -332,7 +332,7 @@ below - ```json [ { - "logicalShardId": "shard1", + "logicalShardId": "mySqlShard1", "host": "10.11.12.13", "user": "root", "password":"", @@ -340,7 +340,7 @@ below - "dbName": "db1" }, { - "logicalShardId": "shard2", + "logicalShardId": "mySqlShard2", "host": "10.11.12.14", "user": "root", "password":"", diff --git a/v2/spanner-to-sourcedb/terraform/samples/spanner-to-sharded-mysql/main.tf b/v2/spanner-to-sourcedb/terraform/samples/spanner-to-sharded-mysql/main.tf index 2fa85f99dd..7c906f36bd 100644 --- a/v2/spanner-to-sourcedb/terraform/samples/spanner-to-sharded-mysql/main.tf +++ b/v2/spanner-to-sourcedb/terraform/samples/spanner-to-sharded-mysql/main.tf @@ -61,7 +61,7 @@ resource "google_storage_bucket_object" "session_file_object" { bucket = google_storage_bucket.reverse_replication_bucket.id } -# Auto-generate the source shards file from the Terraform configuration and +# Auto-generate the source mySqlShards file from the Terraform configuration and # upload it to GCS. resource "google_storage_bucket_object" "source_shards_file_object" { depends_on = [google_project_service.enabled_apis] diff --git a/v2/spanner-to-sourcedb/terraform/samples/spanner-to-sharded-mysql/terraform.tfvars b/v2/spanner-to-sourcedb/terraform/samples/spanner-to-sharded-mysql/terraform.tfvars index fbcff479ac..69be7647d7 100644 --- a/v2/spanner-to-sourcedb/terraform/samples/spanner-to-sharded-mysql/terraform.tfvars +++ b/v2/spanner-to-sourcedb/terraform/samples/spanner-to-sharded-mysql/terraform.tfvars @@ -49,7 +49,7 @@ dataflow_params = { dlq_gcs_pub_sub_subscription = "projects//subscriptions/my-subscription" # Replace with your project ID and subscription name # Optional name of the directory to skip skip_directory_name = "skip-directory" - # Optional maximum number of shard connections + # Optional maximum number of mySqlShard connections max_shard_connections = "10" # Optional dead letter queue directory dead_letter_queue_directory = "gs://my-bucket/dlq" @@ -104,35 +104,35 @@ dataflow_params = { shard_list = [ { - # Logical ID of the shard - logicalShardId = "shard1" - # Hostname or IP address of the shard - host = "" # Replace with the shard's hostname or IP address - # Username for connecting to the shard + # Logical ID of the mySqlShard + logicalShardId = "mySqlShard1" + # Hostname or IP address of the mySqlShard + host = "" # Replace with the mySqlShard's hostname or IP address + # Username for connecting to the mySqlShard user = "root" # URI of the Secret Manager secret containing the password (optional) - secretManagerUri = "projects//secrets/shard1-password/versions/latest" # Replace with your project ID and secret name - # Password for connecting to the shard (optional, use either this or secretManagerUri) + secretManagerUri = "projects//secrets/mySqlShard1-password/versions/latest" # Replace with your project ID and secret name + # Password for connecting to the mySqlShard (optional, use either this or secretManagerUri) password = null - # Port number for connecting to the shard + # Port number for connecting to the mySqlShard port = "3306" - # Name of the database on the shard + # Name of the database on the mySqlShard dbName = "db1" }, { - # Logical ID of the shard - logicalShardId = "shard2" - # Hostname or IP address of the shard - host = "" # Replace with the shard's hostname or IP address - # Username for connecting to the shard + # Logical ID of the mySqlShard + logicalShardId = "mySqlShard2" + # Hostname or IP address of the mySqlShard + host = "" # Replace with the mySqlShard's hostname or IP address + # Username for connecting to the mySqlShard user = "root" # URI of the Secret Manager secret containing the password (optional) - secretManagerUri = "projects//secrets/shard2-password/versions/latest" # Replace with your project ID and secret name - # Password for connecting to the shard (optional, use either this or secretManagerUri) + secretManagerUri = "projects//secrets/mySqlShard2-password/versions/latest" # Replace with your project ID and secret name + # Password for connecting to the mySqlShard (optional, use either this or secretManagerUri) password = null - # Port number for connecting to the shard + # Port number for connecting to the mySqlShard port = "3306" - # Name of the database on the shard + # Name of the database on the mySqlShard dbName = "db2" } ] \ No newline at end of file diff --git a/v2/spanner-to-sourcedb/terraform/samples/spanner-to-sharded-mysql/terraform_simple.tfvars b/v2/spanner-to-sourcedb/terraform/samples/spanner-to-sharded-mysql/terraform_simple.tfvars index 1f39bf23ba..5655213da0 100644 --- a/v2/spanner-to-sourcedb/terraform/samples/spanner-to-sharded-mysql/terraform_simple.tfvars +++ b/v2/spanner-to-sourcedb/terraform/samples/spanner-to-sharded-mysql/terraform_simple.tfvars @@ -24,10 +24,10 @@ dataflow_params = { # Shards shard_list = [ { - logicalShardId = "" # Value of the shard populated in Spanner - host = "