diff --git a/infrastructure/helm-chart/charts/tools/charts/flink/BUILD b/infrastructure/helm-chart/charts/tools/charts/flink/BUILD new file mode 100644 index 000000000..5bb7f8f59 --- /dev/null +++ b/infrastructure/helm-chart/charts/tools/charts/flink/BUILD @@ -0,0 +1,32 @@ +load("@rules_pkg//:pkg.bzl", "pkg_tar") +load("@com_github_airyhq_bazel_tools//helm:helm.bzl", "helm_template_test") +load("//tools/build:helm.bzl", "helm_push_develop", "helm_push_release") + +filegroup( + name = "files", + srcs = glob( + ["**/*"], + exclude = ["BUILD"], + ), + visibility = ["//visibility:public"], +) + +pkg_tar( + name = "package", + srcs = [":files"], + extension = "tgz", + strip_prefix = "./", +) + +helm_template_test( + name = "template", + chart = ":package", +) + +helm_push_develop( + chart = ":package", +) + +helm_push_release( + chart = ":package", +) diff --git a/infrastructure/helm-chart/charts/tools/charts/flink/Chart.yaml b/infrastructure/helm-chart/charts/tools/charts/flink/Chart.yaml new file mode 100644 index 000000000..f6a32ed28 --- /dev/null +++ b/infrastructure/helm-chart/charts/tools/charts/flink/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v2 +appVersion: "1.0" +description: Flink +name: flink +version: 1.0 diff --git a/infrastructure/helm-chart/charts/tools/charts/flink/templates/configuration.yaml b/infrastructure/helm-chart/charts/tools/charts/flink/templates/configuration.yaml new file mode 100644 index 000000000..0e2edb551 --- /dev/null +++ b/infrastructure/helm-chart/charts/tools/charts/flink/templates/configuration.yaml @@ -0,0 +1,61 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: flink-config + labels: + app: flink +data: + flink-conf.yaml: |+ + jobmanager.rpc.address: flink-jobmanager + taskmanager.numberOfTaskSlots: 2 + blob.server.port: 6124 + jobmanager.rpc.port: 6123 + taskmanager.rpc.port: 6122 + jobmanager.memory.process.size: 1600m + taskmanager.memory.process.size: 1728m + parallelism.default: 2 + log4j-console.properties: |+ + # This affects logging for both user code and Flink + rootLogger.level = INFO + rootLogger.appenderRef.console.ref = ConsoleAppender + rootLogger.appenderRef.rolling.ref = RollingFileAppender + + # Uncomment this if you want to _only_ change Flink's logging + #logger.flink.name = org.apache.flink + #logger.flink.level = INFO + + # The following lines keep the log level of common libraries/connectors on + # log level INFO. The root logger does not override this. You have to manually + # change the log levels here. + logger.pekko.name = org.apache.pekko + logger.pekko.level = INFO + logger.kafka.name= org.apache.kafka + logger.kafka.level = INFO + logger.hadoop.name = org.apache.hadoop + logger.hadoop.level = INFO + logger.zookeeper.name = org.apache.zookeeper + logger.zookeeper.level = INFO + + # Log all infos to the console + appender.console.name = ConsoleAppender + appender.console.type = CONSOLE + appender.console.layout.type = PatternLayout + appender.console.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n + + # Log all infos in the given rolling file + appender.rolling.name = RollingFileAppender + appender.rolling.type = RollingFile + appender.rolling.append = false + appender.rolling.fileName = ${sys:log.file} + appender.rolling.filePattern = ${sys:log.file}.%i + appender.rolling.layout.type = PatternLayout + appender.rolling.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n + appender.rolling.policies.type = Policies + appender.rolling.policies.size.type = SizeBasedTriggeringPolicy + appender.rolling.policies.size.size=100MB + appender.rolling.strategy.type = DefaultRolloverStrategy + appender.rolling.strategy.max = 10 + + # Suppress the irrelevant (wrong) warnings from the Netty channel handler + logger.netty.name = org.jboss.netty.channel.DefaultChannelPipeline + logger.netty.level = OFF diff --git a/infrastructure/helm-chart/charts/tools/charts/flink/templates/jobmanager-service.yaml b/infrastructure/helm-chart/charts/tools/charts/flink/templates/jobmanager-service.yaml new file mode 100644 index 000000000..38a92b7df --- /dev/null +++ b/infrastructure/helm-chart/charts/tools/charts/flink/templates/jobmanager-service.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Service +metadata: + name: flink-jobmanager +spec: + type: ClusterIP + ports: + - name: rpc + port: 6123 + - name: blob-server + port: 6124 + - name: webui + port: 8081 + - name: restapi + port: 8083 + selector: + app: flink + component: jobmanager diff --git a/infrastructure/helm-chart/charts/tools/charts/flink/templates/jobmanager.yaml b/infrastructure/helm-chart/charts/tools/charts/flink/templates/jobmanager.yaml new file mode 100644 index 000000000..6a76b5f94 --- /dev/null +++ b/infrastructure/helm-chart/charts/tools/charts/flink/templates/jobmanager.yaml @@ -0,0 +1,46 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: flink-jobmanager +spec: + replicas: 1 + selector: + matchLabels: + app: flink + component: jobmanager + template: + metadata: + labels: + app: flink + component: jobmanager + spec: + containers: + - name: jobmanager + image: ghcr.io/airyhq/flink:release + args: ["jobmanager"] + ports: + - containerPort: 6123 + name: rpc + - containerPort: 6124 + name: blob-server + - containerPort: 8081 + name: webui + livenessProbe: + tcpSocket: + port: 6123 + initialDelaySeconds: 30 + periodSeconds: 60 + volumeMounts: + - name: flink-config-volume + mountPath: /opt/flink/conf + securityContext: + runAsUser: 9999 # refers to user _flink_ from official flink image, change if necessary + volumes: + - name: flink-config-volume + configMap: + name: flink-config + items: + - key: flink-conf.yaml + path: flink-conf.yaml + - key: log4j-console.properties + path: log4j-console.properties diff --git a/infrastructure/helm-chart/charts/tools/charts/flink/templates/taskmanager.yaml b/infrastructure/helm-chart/charts/tools/charts/flink/templates/taskmanager.yaml new file mode 100644 index 000000000..5f6d68e46 --- /dev/null +++ b/infrastructure/helm-chart/charts/tools/charts/flink/templates/taskmanager.yaml @@ -0,0 +1,42 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: flink-taskmanager +spec: + replicas: 2 + selector: + matchLabels: + app: flink + component: taskmanager + template: + metadata: + labels: + app: flink + component: taskmanager + spec: + containers: + - name: taskmanager + image: ghcr.io/airyhq/flink:release + args: ["taskmanager"] + ports: + - containerPort: 6122 + name: rpc + livenessProbe: + tcpSocket: + port: 6122 + initialDelaySeconds: 30 + periodSeconds: 60 + volumeMounts: + - name: flink-config-volume + mountPath: /opt/flink/conf/ + securityContext: + runAsUser: 9999 # refers to user _flink_ from official flink image, change if necessary + volumes: + - name: flink-config-volume + configMap: + name: flink-config + items: + - key: flink-conf.yaml + path: flink-conf.yaml + - key: log4j-console.properties + path: log4j-console.properties diff --git a/infrastructure/helm-chart/charts/tools/charts/flink/values.yaml b/infrastructure/helm-chart/charts/tools/charts/flink/values.yaml new file mode 100644 index 000000000..7843d48d6 --- /dev/null +++ b/infrastructure/helm-chart/charts/tools/charts/flink/values.yaml @@ -0,0 +1,3 @@ +name: flink +mandatory: false +enabled: true diff --git a/infrastructure/images/flink/Dockerfile b/infrastructure/images/flink/Dockerfile new file mode 100644 index 000000000..03b915a41 --- /dev/null +++ b/infrastructure/images/flink/Dockerfile @@ -0,0 +1,4 @@ +FROM flink:1.17.2-scala_2.12-java8 + +# Copy the Kafka connector +COPY files/flink-sql-connector-kafka-1.17.2.jar /opt/flink/lib/flink-sql-connector-kafka-1.17.2.jar diff --git a/infrastructure/images/flink/Makefile b/infrastructure/images/flink/Makefile new file mode 100644 index 000000000..ed9e561dc --- /dev/null +++ b/infrastructure/images/flink/Makefile @@ -0,0 +1,6 @@ +build: + docker build -t flink . + +release: build + docker tag flink ghcr.io/airyhq/flink:release + docker push ghcr.io/airyhq/flink:release diff --git a/infrastructure/images/flink/files/flink-sql-connector-kafka-1.17.2.jar b/infrastructure/images/flink/files/flink-sql-connector-kafka-1.17.2.jar new file mode 100644 index 000000000..013845f7d Binary files /dev/null and b/infrastructure/images/flink/files/flink-sql-connector-kafka-1.17.2.jar differ