Skip to content

Add the BT Connector for Flink 2.1 #91

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
techienidhipatel wants to merge 1 commit into
base: main
Choose a base branch
from
+162 −15
Open

Add the BT Connector for Flink 2.1 #91

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
153 changes: 153 additions & 0 deletions connectors/bigtable/flink-connector-gcp-bigtable/shade-pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,153 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>

<groupId>com.google.flink.connector.gcp</groupId>
<artifactId>flink-connector-gcp-bigtable-shaded</artifactId>
<version>2.0.0</version>
<packaging>jar</packaging>

<properties>
<maven.compiler.source>11</maven.compiler.source>
<maven.compiler.target>11</maven.compiler.target>
<flink.version>2.1.0</flink.version>
</properties>

<dependencyManagement>
<dependencies>
<dependency>
<groupId>com.google.cloud</groupId>
<artifactId>libraries-bom</artifactId>
<version>26.52.0</version>
<type>pom</type>
<scope>import</scope>
</dependency>
</dependencies>
</dependencyManagement>

<dependencies>
<dependency>
<groupId>com.google.flink.connector.gcp</groupId>
<artifactId>flink-connector-gcp-bigtable</artifactId>
<version>0.2.0</version>
</dependency>

<!-- Flink dependencies (provided - available at runtime) -->
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-core</artifactId>
<version>${flink.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-connector-base</artifactId>
<version>${flink.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-streaming-java</artifactId>
<version>${flink.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-table-common</artifactId>
<version>${flink.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-table-runtime</artifactId>
<version>${flink.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-metrics-dropwizard</artifactId>
<version>${flink.version}</version>
<scope>provided</scope>
</dependency>
</dependencies>

<build>
<finalName>flink-connector-gcp-bigtable-shaded-2.0.0</finalName>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>3.5.1</version>
<executions>
<execution>
<id>shade-flink</id>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
<configuration>
<shadedArtifactAttached>false</shadedArtifactAttached>
<createDependencyReducedPom>false</createDependencyReducedPom>
<transformers>
<transformer
implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
</transformers>
<artifactSet>
<excludes>
<!-- Exclude Flink dependencies (provided by runtime) -->
<exclude>org.apache.flink:*</exclude>
<!-- Exclude test and emulator dependencies -->
<!-- Exclude Avro to avoid classloader/version conflicts with Flink/Monorail -->
<exclude>org.apache.avro:avro</exclude>

<exclude>com.google.cloud:google-cloud-bigtable-emulator*</exclude>
<!-- Exclude jsr305 -->
<exclude>com.google.code.findbugs:jsr305</exclude>
</excludes>
</artifactSet>
<filters>
<filter>
<artifact>*:*</artifact>
<excludes>
<exclude>META-INF/*.DSA</exclude>
<exclude>META-INF/*.MF</exclude>
<exclude>META-INF/*.RSA</exclude>
<exclude>META-INF/*.SF</exclude>
</excludes>
</filter>
</filters>
<relocations>
<relocation>
<pattern>io.grpc</pattern>
<shadedPattern>com.google.flink.connector.gcp.bigtable.shaded.io.grpc</shadedPattern>
</relocation>
<relocation>
<pattern>com.google.protobuf</pattern>
<shadedPattern>com.google.flink.connector.gcp.bigtable.shaded.com.google.protobuf</shadedPattern>
</relocation>
<relocation>
<pattern>com.google.common</pattern>
<shadedPattern>com.google.flink.connector.gcp.bigtable.shaded.com.google.common</shadedPattern>
</relocation>
<relocation>
<pattern>io.opencensus</pattern>
<shadedPattern>com.google.flink.connector.gcp.bigtable.shaded.io.opencensus</shadedPattern>
</relocation>
<relocation>
<pattern>com.google.api</pattern>
<shadedPattern>com.google.flink.connector.gcp.bigtable.shaded.com.google.api</shadedPattern>
</relocation>
<relocation>
<pattern>com.google.cloud</pattern>
<shadedPattern>com.google.flink.connector.gcp.bigtable.shaded.com.google.cloud</shadedPattern>
</relocation>
</relocations>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>
6 changes: 0 additions & 6 deletions ...ctor-gcp-bigtable/src/main/java/com/google/flink/connector/gcp/bigtable/BigtableSink.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,12 +68,6 @@ public static <T> Builder<T> builder() {

private static final Logger logger = LoggerFactory.getLogger(BigtableSink.class);

@Override
public SinkWriter<T> createWriter(Sink.InitContext sinkInitContext) throws IOException {
// https://nightlies.apache.org/flink/flink-docs-release-1.19/api/java/org/apache/flink/api/connector/sink2/Sink.InitContext.html
// Discussion: https://lists.apache.org/thread/ydjypynwrh56s7x64lo5jt5wofl4jgk9
throw new UnsupportedOperationException("Not supported. InitContext is deprecated");
}

@Override
public SinkWriter<T> createWriter(WriterInitContext sinkInitContext) throws IOException {
Expand Down
2 changes: 1 addition & 1 deletion ...able/src/main/java/com/google/flink/connector/gcp/bigtable/examples/WriteExactlyOnce.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.api.common.typeinfo.Types;
import org.apache.flink.api.connector.source.util.ratelimit.RateLimiterStrategy;
import org.apache.flink.api.java.utils.ParameterTool;
import org.apache.flink.util.ParameterTool;
import org.apache.flink.connector.datagen.source.DataGeneratorSource;
import org.apache.flink.connector.datagen.source.GeneratorFunction;
import org.apache.flink.formats.avro.typeutils.GenericRecordAvroTypeInfo;
Expand Down
2 changes: 1 addition & 1 deletion ...le/src/main/java/com/google/flink/connector/gcp/bigtable/examples/WriteGenericRecord.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.api.common.typeinfo.Types;
import org.apache.flink.api.connector.source.util.ratelimit.RateLimiterStrategy;
import org.apache.flink.api.java.utils.ParameterTool;
import org.apache.flink.util.ParameterTool;
import org.apache.flink.connector.datagen.source.DataGeneratorSource;
import org.apache.flink.connector.datagen.source.GeneratorFunction;
import org.apache.flink.formats.avro.typeutils.GenericRecordAvroTypeInfo;
Expand Down
2 changes: 1 addition & 1 deletion .../main/java/com/google/flink/connector/gcp/bigtable/examples/WriteGenericRecordNested.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.api.common.typeinfo.Types;
import org.apache.flink.api.connector.source.util.ratelimit.RateLimiterStrategy;
import org.apache.flink.api.java.utils.ParameterTool;
import org.apache.flink.util.ParameterTool;
import org.apache.flink.connector.datagen.source.DataGeneratorSource;
import org.apache.flink.connector.datagen.source.GeneratorFunction;
import org.apache.flink.formats.avro.typeutils.GenericRecordAvroTypeInfo;
Expand Down
2 changes: 1 addition & 1 deletion ...bigtable/src/main/java/com/google/flink/connector/gcp/bigtable/examples/WriteRowData.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.api.common.typeinfo.Types;
import org.apache.flink.api.connector.source.util.ratelimit.RateLimiterStrategy;
import org.apache.flink.api.java.utils.ParameterTool;
import org.apache.flink.util.ParameterTool;
import org.apache.flink.connector.datagen.source.DataGeneratorSource;
import org.apache.flink.connector.datagen.source.GeneratorFunction;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
Expand Down
2 changes: 1 addition & 1 deletion ...le/src/main/java/com/google/flink/connector/gcp/bigtable/examples/WriteRowDataNested.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
import org.apache.flink.api.common.functions.MapFunction;
import org.apache.flink.api.common.typeinfo.Types;
import org.apache.flink.api.connector.source.util.ratelimit.RateLimiterStrategy;
import org.apache.flink.api.java.utils.ParameterTool;
import org.apache.flink.util.ParameterTool;
import org.apache.flink.connector.datagen.source.DataGeneratorSource;
import org.apache.flink.connector.datagen.source.GeneratorFunction;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
Expand Down
2 changes: 1 addition & 1 deletion ...igtable/src/main/java/com/google/flink/connector/gcp/bigtable/examples/WriteTableAPI.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@

package com.google.flink.connector.gcp.bigtable.examples;

import org.apache.flink.api.java.utils.ParameterTool;
import org.apache.flink.util.ParameterTool;
import org.apache.flink.connector.datagen.table.DataGenConnectorOptions;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.table.api.DataTypes;
Expand Down
2 changes: 1 addition & 1 deletion ...e/src/main/java/com/google/flink/connector/gcp/bigtable/examples/WriteTableAPINested.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@

package com.google.flink.connector.gcp.bigtable.examples;

import org.apache.flink.api.java.utils.ParameterTool;
import org.apache.flink.util.ParameterTool;
import org.apache.flink.connector.datagen.table.DataGenConnectorOptions;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.table.api.DataTypes;
Expand Down
2 changes: 1 addition & 1 deletion ...ble/src/main/java/com/google/flink/connector/gcp/bigtable/examples/WriteWithFunction.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
import org.apache.flink.api.common.eventtime.WatermarkStrategy;
import org.apache.flink.api.common.typeinfo.Types;
import org.apache.flink.api.connector.source.util.ratelimit.RateLimiterStrategy;
import org.apache.flink.api.java.utils.ParameterTool;
import org.apache.flink.util.ParameterTool;
import org.apache.flink.connector.datagen.source.DataGeneratorSource;
import org.apache.flink.connector.datagen.source.GeneratorFunction;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
Expand Down
2 changes: 1 addition & 1 deletion connectors/bigtable/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
</modules>

<properties>
<flink.version>1.20.0</flink.version>
<flink.version>2.1.0</flink.version>
<maven.compiler.source>11</maven.compiler.source>
<maven.compiler.target>11</maven.compiler.target>
<slf4j.version>1.7.36</slf4j.version>
Expand Down