Skip to content

Commit 8640368

Browse files
committed
Tighten constraints on TopicWiseConfigs
Currently if any config option starts with `topic.` prefix, the connector will attempt to match it against regular expression for TopicWiseConfigs. If it fails then whole connector fails to start. This is a problem when launching on Confluent Cloud which adds for example `topic.creation.default.max.message.bytes` config. It mistakenly gets through the first check because it starts with `topic.` and then trips up on regular expression. It seems there could be many more configs like this that pertain to Kafka topic but are not this connectors TopicWiseConfigs. Adding suffix check should be sufficient and it should not break existing configurations as long as those adhered to the existing documentation.
1 parent dc54792 commit 8640368

File tree

1 file changed

+15
-1
lines changed

1 file changed

+15
-1
lines changed

src/main/java/io/connect/scylladb/ScyllaDbSinkConnectorConfig.java

+15-1
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,8 @@
2525
import io.confluent.kafka.connect.utils.config.ValidEnum;
2626
import io.confluent.kafka.connect.utils.config.ValidPort;
2727
import io.connect.scylladb.topictotable.TopicConfigs;
28+
import org.slf4j.Logger;
29+
import org.slf4j.LoggerFactory;
2830

2931
/**
3032
* Configuration class for {@link ScyllaDbSinkConnector}.
@@ -62,6 +64,10 @@ public class ScyllaDbSinkConnectorConfig extends AbstractConfig {
6264
private static final Pattern TOPIC_KS_TABLE_SETTING_PATTERN =
6365
Pattern.compile("topic\\.([a-zA-Z0-9._-]+)\\.([^.]+|\"[\"]+\")\\.([^.]+|\"[\"]+\")\\.(mapping|consistencyLevel|ttlSeconds|deletesEnabled)$");
6466

67+
private static final String[] TOPIC_WISE_CONFIGS_VALID_SUFFIXES = {".mapping",".consistencyLevel",".ttlSeconds",".deletesEnabled"};
68+
69+
private static final Logger log = LoggerFactory.getLogger(ScyllaDbSinkConnectorConfig.class);
70+
6571
static final Set<String> CLIENT_COMPRESSION = ImmutableSet.of("none", "lz4", "snappy");
6672

6773
static final Set<String> TABLE_COMPRESSORS = ImmutableSet.of("SnappyCompressor", "LZ4Compressor", "DeflateCompressor", "none");
@@ -111,8 +117,9 @@ public ScyllaDbSinkConnectorConfig(Map<?, ?> originals) {
111117
Map<String, Map<String, String>> topicWiseConfigsMap = new HashMap<>();
112118
for (final Map.Entry<String, String> entry : ((Map<String, String>) originals).entrySet()) {
113119
final String name2 = entry.getKey();
114-
if (name2.startsWith("topic.")) {
120+
if (name2.startsWith("topic.") && hasTopicWiseConfigSuffix(name2)) {
115121
final String topicName = this.tryMatchTopicName(name2);
122+
log.debug("Interpreting " + name2 + " as custom TopicWiseConfig for topic " + topicName);
116123
final Map<String, String> topicMap = topicWiseConfigsMap.computeIfAbsent(topicName, t -> new HashMap());
117124
topicMap.put(name2.split("\\.")[name2.split("\\.").length - 1], entry.getValue());
118125
}
@@ -567,6 +574,13 @@ private String tryMatchTopicName(final String name) {
567574
throw new IllegalArgumentException("The setting: " + name + " does not match topic.keyspace.table nor topic.codec regular expression pattern");
568575
}
569576

577+
private boolean hasTopicWiseConfigSuffix(final String name) {
578+
for (String suffix : TOPIC_WISE_CONFIGS_VALID_SUFFIXES) {
579+
if (name.endsWith(suffix)) return true;
580+
}
581+
return false;
582+
}
583+
570584
private static String[] toStringArray(Object[] arr){
571585
return Arrays.stream(arr).map(Object::toString).toArray(String[]::new);
572586
}

0 commit comments

Comments
 (0)