diff --git a/README.md b/README.md index 45d0d65..90ac4fe 100644 --- a/README.md +++ b/README.md @@ -99,6 +99,12 @@ _Decodable provides a managed PyFlink service. Learn more [here](https://docs.de | [Decodable CI/CD](declarative-cicd) | An example of using Decodable with GitHub Actions| | [Decodable CLI Docker image](cli-docker) | An example Dockerfile for running the Decodable CLI under Docker.| +### Kubernetes + +| Example | Description | +|-------------------------------------------------------|-------------| +| [Fink on Kubernetes](flink-on-kubernetes) | An example of running Flink via the Flink Kubernetes Operator| + ## License This code base is available under the Apache License, version 2. diff --git a/flink-on-kubernetes/README.md b/flink-on-kubernetes/README.md new file mode 100644 index 0000000..ca2c212 --- /dev/null +++ b/flink-on-kubernetes/README.md @@ -0,0 +1,5 @@ +# Get Running With Flink on Kubernetes + +An extensive example showing how to run Flink on Kubernetes. + +Touching on a range of topics such as installation and set-up, creating container images for your own Flink jobs, fault tolerance and high availability, savepoint management, observability, and more. diff --git a/flink-on-kubernetes/flink/basic.yaml b/flink-on-kubernetes/flink/basic.yaml new file mode 100644 index 0000000..1c28e63 --- /dev/null +++ b/flink-on-kubernetes/flink/basic.yaml @@ -0,0 +1,45 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +apiVersion: flink.apache.org/v1beta1 +kind: FlinkDeployment +metadata: + name: basic-example +spec: + image: flink:1.20-java17 + flinkVersion: v1_20 + flinkConfiguration: + taskmanager.numberOfTaskSlots: "2" + serviceAccount: flink + jobManager: + resource: + memory: "2048m" + cpu: 1 + taskManager: + resource: + memory: "2048m" + cpu: 1 + job: + jarURI: local:///opt/flink/examples/streaming/StateMachineExample.jar + parallelism: 2 + upgradeMode: stateless + ingress: + template: "localhost/{{name}}(/|$)(.*)" + className: "nginx" + annotations: + nginx.ingress.kubernetes.io/rewrite-target: "/$2" \ No newline at end of file diff --git a/flink-on-kubernetes/flink/custom-job-ha.yaml b/flink-on-kubernetes/flink/custom-job-ha.yaml new file mode 100644 index 0000000..08a1055 --- /dev/null +++ b/flink-on-kubernetes/flink/custom-job-ha.yaml @@ -0,0 +1,70 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +apiVersion: flink.apache.org/v1beta1 +kind: FlinkDeployment +metadata: + name: custom-job-ha +spec: + image: decodable-examples/hello-world-job:1.0 + flinkVersion: v1_20 + flinkConfiguration: + taskmanager.numberOfTaskSlots: "2" + + s3.access.key: minio + s3.secret.key: minio123 + s3.endpoint: http://minio-service.default.svc.cluster.local:9000 + s3.path.style.access: "true" + s3.entropy.key: _entropy_ + s3.entropy.length: "4" + + execution.checkpointing.interval: "5000" + + state.backend: rocksdb + state.backend.incremental: "true" + state.checkpoints.dir: s3://flink-data/_entropy_/checkpoints + state.savepoints.dir: s3://flink-data/savepoints + + high-availability.type: kubernetes + high-availability.storageDir: s3://flink-data/ha + serviceAccount: flink + jobManager: + resource: + memory: "2048m" + cpu: 1 + taskManager: + resource: + memory: "2048m" + cpu: 1 + podTemplate: + spec: + containers: + - name: flink-main-container + env: + - name: ENABLE_BUILT_IN_PLUGINS + value: "flink-s3-fs-presto-1.20.0.jar" + job: + jarURI: local:///opt/flink-jobs/hello-world-job-1.0.jar + parallelism: 2 + upgradeMode: savepoint + state: running + ingress: + template: "localhost/{{name}}(/|$)(.*)" + className: "nginx" + annotations: + nginx.ingress.kubernetes.io/rewrite-target: "/$2" diff --git a/flink-on-kubernetes/flink/custom-job-s3.yaml b/flink-on-kubernetes/flink/custom-job-s3.yaml new file mode 100644 index 0000000..5621d8d --- /dev/null +++ b/flink-on-kubernetes/flink/custom-job-s3.yaml @@ -0,0 +1,70 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +apiVersion: flink.apache.org/v1beta1 +kind: FlinkDeployment +metadata: + name: custom-job-s3 +spec: + image: decodable-examples/hello-world-job:1.0 + flinkVersion: v1_20 + flinkConfiguration: + taskmanager.numberOfTaskSlots: "2" + + s3.access.key: minio + s3.secret.key: minio123 + s3.endpoint: http://minio-service.default.svc.cluster.local:9000 + s3.path.style.access: "true" + s3.entropy.key: _entropy_ + s3.entropy.length: "4" + + execution.checkpointing.interval: "5000" + + state.backend: rocksdb + state.backend.incremental: "true" + state.checkpoints.dir: s3://flink-data/_entropy_/checkpoints + state.savepoints.dir: s3://flink-data/savepoints + + high-availability.type: kubernetes + high-availability.storageDir: s3://flink-data/ha + serviceAccount: flink + jobManager: + resource: + memory: "2048m" + cpu: 2 + taskManager: + resource: + memory: "2048m" + cpu: 2 + podTemplate: + spec: + containers: + - name: flink-main-container + env: + - name: ENABLE_BUILT_IN_PLUGINS + value: "flink-s3-fs-presto-1.20.0.jar" + job: + jarURI: s3://job-files/hello-world-job-1.0.jar + parallelism: 2 + upgradeMode: savepoint + state: running + ingress: + template: "localhost/{{name}}(/|$)(.*)" + className: "nginx" + annotations: + nginx.ingress.kubernetes.io/rewrite-target: "/$2" \ No newline at end of file diff --git a/flink-on-kubernetes/flink/custom-job.yaml b/flink-on-kubernetes/flink/custom-job.yaml new file mode 100644 index 0000000..ad9a275 --- /dev/null +++ b/flink-on-kubernetes/flink/custom-job.yaml @@ -0,0 +1,45 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +apiVersion: flink.apache.org/v1beta1 +kind: FlinkDeployment +metadata: + name: custom-job +spec: + image: decodable-examples/hello-world-job:1.0 + flinkVersion: v1_20 + flinkConfiguration: + taskmanager.numberOfTaskSlots: "2" + serviceAccount: flink + jobManager: + resource: + memory: "2048m" + cpu: 1 + taskManager: + resource: + memory: "2048m" + cpu: 1 + job: + jarURI: local:///opt/flink-jobs/hello-world-job-1.0.jar + parallelism: 1 + upgradeMode: stateless + ingress: + template: "localhost/{{name}}(/|$)(.*)" + className: "nginx" + annotations: + nginx.ingress.kubernetes.io/rewrite-target: "/$2" \ No newline at end of file diff --git a/flink-on-kubernetes/flink/logging-job.yaml b/flink-on-kubernetes/flink/logging-job.yaml new file mode 100644 index 0000000..cf5c5ce --- /dev/null +++ b/flink-on-kubernetes/flink/logging-job.yaml @@ -0,0 +1,111 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +apiVersion: flink.apache.org/v1beta1 +kind: FlinkDeployment +metadata: + name: logging-job +spec: + image: decodable-examples/hello-world-job:1.0 + flinkVersion: v1_20 + flinkConfiguration: + taskmanager.numberOfTaskSlots: "2" + serviceAccount: flink + jobManager: + resource: + memory: "2048m" + cpu: 1 + taskManager: + resource: + memory: "2048m" + cpu: 1 + job: + jarURI: local:///opt/flink/examples/streaming/StateMachineExample.jar + parallelism: 2 + upgradeMode: stateless + logConfiguration: + log4j-console.properties: |+ + ################################################################################ + # Licensed to the Apache Software Foundation (ASF) under one + # or more contributor license agreements. See the NOTICE file + # distributed with this work for additional information + # regarding copyright ownership. The ASF licenses this file + # to you under the Apache License, Version 2.0 (the + # "License"); you may not use this file except in compliance + # with the License. You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + ################################################################################ + + # This affects logging for both user code and Flink + rootLogger.level = INFO + rootLogger.appenderRef.console.ref = ConsoleAppender + rootLogger.appenderRef.rolling.ref = RollingFileAppender + + # Uncomment this if you want to _only_ change Flink's logging + #logger.flink.name = org.apache.flink + #logger.flink.level = INFO + + # The following lines keep the log level of common libraries/connectors on + # log level INFO. The root logger does not override this. You have to manually + # change the log levels here. + logger.akka.name = akka + logger.akka.level = INFO + logger.kafka.name= org.apache.kafka + logger.kafka.level = INFO + logger.hadoop.name = org.apache.hadoop + logger.hadoop.level = INFO + logger.zookeeper.name = org.apache.zookeeper + logger.zookeeper.level = INFO + + # Log all infos to the console + appender.console.name = ConsoleAppender + appender.console.type = CONSOLE + appender.console.layout.type = JsonTemplateLayout + + # Log all infos in the given rolling file + appender.rolling.name = RollingFileAppender + appender.rolling.type = RollingFile + appender.rolling.append = false + appender.rolling.fileName = ${sys:log.file} + appender.rolling.filePattern = ${sys:log.file}.%i + appender.rolling.layout.type = PatternLayout + appender.rolling.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n + appender.rolling.policies.type = Policies + appender.rolling.policies.size.type = SizeBasedTriggeringPolicy + appender.rolling.policies.size.size=100MB + appender.rolling.strategy.type = DefaultRolloverStrategy + appender.rolling.strategy.max = 10 + + # Suppress the irrelevant (wrong) warnings from the Netty channel handler + logger.netty.name = org.apache.flink.shaded.akka.org.jboss.netty.channel.DefaultChannelPipeline + logger.netty.level = OFF + + # The monitor interval in seconds to enable log4j automatic reconfiguration + # monitorInterval = 30 + ingress: + template: "localhost/{{name}}(/|$)(.*)" + className: "nginx" + annotations: + nginx.ingress.kubernetes.io/rewrite-target: "/$2" diff --git a/flink-on-kubernetes/flink/savepoint.yaml b/flink-on-kubernetes/flink/savepoint.yaml new file mode 100644 index 0000000..21e464d --- /dev/null +++ b/flink-on-kubernetes/flink/savepoint.yaml @@ -0,0 +1,10 @@ +apiVersion: flink.apache.org/v1beta1 +kind: FlinkStateSnapshot +metadata: + name: example-savepoint +spec: + backoffLimit: 1 + jobReference: + kind: FlinkDeployment + name: custom-job-ha + savepoint: {} diff --git a/flink-on-kubernetes/heimdall/deploy-ingress-nginx.yaml b/flink-on-kubernetes/heimdall/deploy-ingress-nginx.yaml new file mode 100644 index 0000000..881b17a --- /dev/null +++ b/flink-on-kubernetes/heimdall/deploy-ingress-nginx.yaml @@ -0,0 +1,675 @@ +apiVersion: v1 +kind: Namespace +metadata: + labels: + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/name: ingress-nginx + name: ingress-nginx +--- +apiVersion: v1 +automountServiceAccountToken: true +kind: ServiceAccount +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/part-of: ingress-nginx + app.kubernetes.io/version: 1.12.0-beta.0 + name: ingress-nginx + namespace: ingress-nginx +--- +apiVersion: v1 +automountServiceAccountToken: true +kind: ServiceAccount +metadata: + labels: + app.kubernetes.io/component: admission-webhook + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/part-of: ingress-nginx + app.kubernetes.io/version: 1.12.0-beta.0 + name: ingress-nginx-admission + namespace: ingress-nginx +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/part-of: ingress-nginx + app.kubernetes.io/version: 1.12.0-beta.0 + name: ingress-nginx + namespace: ingress-nginx +rules: +- apiGroups: + - "" + resources: + - namespaces + verbs: + - get +- apiGroups: + - "" + resources: + - configmaps + - pods + - secrets + - endpoints + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - services + verbs: + - get + - list + - watch +- apiGroups: + - networking.k8s.io + resources: + - ingresses + verbs: + - get + - list + - watch +- apiGroups: + - networking.k8s.io + resources: + - ingresses/status + verbs: + - update +- apiGroups: + - networking.k8s.io + resources: + - ingressclasses + verbs: + - get + - list + - watch +- apiGroups: + - coordination.k8s.io + resourceNames: + - ingress-nginx-leader + resources: + - leases + verbs: + - get + - update +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - list + - watch + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + app.kubernetes.io/component: admission-webhook + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/part-of: ingress-nginx + app.kubernetes.io/version: 1.12.0-beta.0 + name: ingress-nginx-admission + namespace: ingress-nginx +rules: +- apiGroups: + - "" + resources: + - secrets + verbs: + - get + - create +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/part-of: ingress-nginx + app.kubernetes.io/version: 1.12.0-beta.0 + name: ingress-nginx +rules: +- apiGroups: + - "" + resources: + - configmaps + - endpoints + - nodes + - pods + - secrets + - namespaces + verbs: + - list + - watch +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - list + - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - services + verbs: + - get + - list + - watch +- apiGroups: + - networking.k8s.io + resources: + - ingresses + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch +- apiGroups: + - networking.k8s.io + resources: + - ingresses/status + verbs: + - update +- apiGroups: + - networking.k8s.io + resources: + - ingressclasses + verbs: + - get + - list + - watch +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - list + - watch + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: admission-webhook + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/part-of: ingress-nginx + app.kubernetes.io/version: 1.12.0-beta.0 + name: ingress-nginx-admission +rules: +- apiGroups: + - admissionregistration.k8s.io + resources: + - validatingwebhookconfigurations + verbs: + - get + - update +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/part-of: ingress-nginx + app.kubernetes.io/version: 1.12.0-beta.0 + name: ingress-nginx + namespace: ingress-nginx +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: ingress-nginx +subjects: +- kind: ServiceAccount + name: ingress-nginx + namespace: ingress-nginx +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + app.kubernetes.io/component: admission-webhook + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/part-of: ingress-nginx + app.kubernetes.io/version: 1.12.0-beta.0 + name: ingress-nginx-admission + namespace: ingress-nginx +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: ingress-nginx-admission +subjects: +- kind: ServiceAccount + name: ingress-nginx-admission + namespace: ingress-nginx +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/part-of: ingress-nginx + app.kubernetes.io/version: 1.12.0-beta.0 + name: ingress-nginx +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ingress-nginx +subjects: +- kind: ServiceAccount + name: ingress-nginx + namespace: ingress-nginx +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app.kubernetes.io/component: admission-webhook + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/part-of: ingress-nginx + app.kubernetes.io/version: 1.12.0-beta.0 + name: ingress-nginx-admission +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ingress-nginx-admission +subjects: +- kind: ServiceAccount + name: ingress-nginx-admission + namespace: ingress-nginx +--- +apiVersion: v1 +data: null +kind: ConfigMap +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/part-of: ingress-nginx + app.kubernetes.io/version: 1.12.0-beta.0 + name: ingress-nginx-controller + namespace: ingress-nginx +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/part-of: ingress-nginx + app.kubernetes.io/version: 1.12.0-beta.0 + name: ingress-nginx-controller + namespace: ingress-nginx +spec: + ipFamilies: + - IPv4 + ipFamilyPolicy: SingleStack + ports: + - appProtocol: http + name: http + port: 80 + protocol: TCP + targetPort: http + - appProtocol: https + name: https + port: 443 + protocol: TCP + targetPort: https + selector: + app.kubernetes.io/component: controller + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/name: ingress-nginx + type: LoadBalancer +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/part-of: ingress-nginx + app.kubernetes.io/version: 1.12.0-beta.0 + name: ingress-nginx-controller-admission + namespace: ingress-nginx +spec: + ports: + - appProtocol: https + name: https-webhook + port: 443 + targetPort: webhook + selector: + app.kubernetes.io/component: controller + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/name: ingress-nginx + type: ClusterIP +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/part-of: ingress-nginx + app.kubernetes.io/version: 1.12.0-beta.0 + name: ingress-nginx-controller + namespace: ingress-nginx +spec: + minReadySeconds: 0 + revisionHistoryLimit: 10 + selector: + matchLabels: + app.kubernetes.io/component: controller + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/name: ingress-nginx + strategy: + rollingUpdate: + maxUnavailable: 1 + type: RollingUpdate + template: + metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/part-of: ingress-nginx + app.kubernetes.io/version: 1.12.0-beta.0 + spec: + containers: + - args: + - /nginx-ingress-controller + - --election-id=ingress-nginx-leader + - --controller-class=k8s.io/ingress-nginx + - --ingress-class=nginx + - --configmap=$(POD_NAMESPACE)/ingress-nginx-controller + - --validating-webhook=:8443 + - --validating-webhook-certificate=/usr/local/certificates/cert + - --validating-webhook-key=/usr/local/certificates/key + - --watch-ingress-without-class=true + - --publish-status-address=localhost + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: LD_PRELOAD + value: /usr/local/lib/libmimalloc.so + image: registry.k8s.io/ingress-nginx/controller:v1.12.0-beta.0@sha256:9724476b928967173d501040631b23ba07f47073999e80e34b120e8db5f234d5 + imagePullPolicy: IfNotPresent + lifecycle: + preStop: + exec: + command: + - /wait-shutdown + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: 10254 + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + name: controller + ports: + - containerPort: 80 + hostPort: 80 + name: http + protocol: TCP + - containerPort: 443 + hostPort: 443 + name: https + protocol: TCP + - containerPort: 8443 + name: webhook + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /healthz + port: 10254 + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + resources: + requests: + cpu: 100m + memory: 90Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + add: + - NET_BIND_SERVICE + drop: + - ALL + readOnlyRootFilesystem: false + runAsGroup: 82 + runAsNonRoot: true + runAsUser: 101 + seccompProfile: + type: RuntimeDefault + volumeMounts: + - mountPath: /usr/local/certificates/ + name: webhook-cert + readOnly: true + dnsPolicy: ClusterFirst + nodeSelector: + kubernetes.io/os: linux + kubernetes.io/hostname: my-cluster-worker + serviceAccountName: ingress-nginx + terminationGracePeriodSeconds: 0 + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/master + operator: Equal + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + operator: Equal + volumes: + - name: webhook-cert + secret: + secretName: ingress-nginx-admission +--- +apiVersion: batch/v1 +kind: Job +metadata: + labels: + app.kubernetes.io/component: admission-webhook + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/part-of: ingress-nginx + app.kubernetes.io/version: 1.12.0-beta.0 + name: ingress-nginx-admission-create + namespace: ingress-nginx +spec: + template: + metadata: + labels: + app.kubernetes.io/component: admission-webhook + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/part-of: ingress-nginx + app.kubernetes.io/version: 1.12.0-beta.0 + name: ingress-nginx-admission-create + spec: + containers: + - args: + - create + - --host=ingress-nginx-controller-admission,ingress-nginx-controller-admission.$(POD_NAMESPACE).svc + - --namespace=$(POD_NAMESPACE) + - --secret-name=ingress-nginx-admission + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: registry.k8s.io/ingress-nginx/kube-webhook-certgen:v1.4.4@sha256:a9f03b34a3cbfbb26d103a14046ab2c5130a80c3d69d526ff8063d2b37b9fd3f + imagePullPolicy: IfNotPresent + name: create + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsGroup: 65532 + runAsNonRoot: true + runAsUser: 65532 + seccompProfile: + type: RuntimeDefault + nodeSelector: + kubernetes.io/os: linux + restartPolicy: OnFailure + serviceAccountName: ingress-nginx-admission +--- +apiVersion: batch/v1 +kind: Job +metadata: + labels: + app.kubernetes.io/component: admission-webhook + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/part-of: ingress-nginx + app.kubernetes.io/version: 1.12.0-beta.0 + name: ingress-nginx-admission-patch + namespace: ingress-nginx +spec: + template: + metadata: + labels: + app.kubernetes.io/component: admission-webhook + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/part-of: ingress-nginx + app.kubernetes.io/version: 1.12.0-beta.0 + name: ingress-nginx-admission-patch + spec: + containers: + - args: + - patch + - --webhook-name=ingress-nginx-admission + - --namespace=$(POD_NAMESPACE) + - --patch-mutating=false + - --secret-name=ingress-nginx-admission + - --patch-failure-policy=Fail + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: registry.k8s.io/ingress-nginx/kube-webhook-certgen:v1.4.4@sha256:a9f03b34a3cbfbb26d103a14046ab2c5130a80c3d69d526ff8063d2b37b9fd3f + imagePullPolicy: IfNotPresent + name: patch + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsGroup: 65532 + runAsNonRoot: true + runAsUser: 65532 + seccompProfile: + type: RuntimeDefault + nodeSelector: + kubernetes.io/os: linux + restartPolicy: OnFailure + serviceAccountName: ingress-nginx-admission +--- +apiVersion: networking.k8s.io/v1 +kind: IngressClass +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/part-of: ingress-nginx + app.kubernetes.io/version: 1.12.0-beta.0 + name: nginx +spec: + controller: k8s.io/ingress-nginx +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + labels: + app.kubernetes.io/component: admission-webhook + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/part-of: ingress-nginx + app.kubernetes.io/version: 1.12.0-beta.0 + name: ingress-nginx-admission +webhooks: +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: ingress-nginx-controller-admission + namespace: ingress-nginx + path: /networking/v1/ingresses + port: 443 + failurePolicy: Fail + matchPolicy: Equivalent + name: validate.nginx.ingress.kubernetes.io + rules: + - apiGroups: + - networking.k8s.io + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - ingresses + sideEffects: None diff --git a/flink-on-kubernetes/heimdall/heimdall.yaml b/flink-on-kubernetes/heimdall/heimdall.yaml new file mode 100644 index 0000000..6b2917c --- /dev/null +++ b/flink-on-kubernetes/heimdall/heimdall.yaml @@ -0,0 +1,75 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: heimdall-service-account +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: heimdall-role +rules: + - apiGroups: + - "" + - flink.apache.org + resources: + - flinkdeployments + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: heimdall-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: heimdall-role +subjects: + - kind: ServiceAccount + name: heimdall-service-account +--- +apiVersion: v1 +kind: Pod +metadata: + name: heimdall + labels: + app: heimdall +spec: + serviceAccountName: "heimdall-service-account" + containers: + - name: heimdall + image: ghcr.io/sap1ens/heimdall:0.7.0 + ports: + - containerPort: 8080 + env: + - name: HEIMDALL_ENDPOINT_PATH_PATTERNS_FLINK_UI + value: "http://localhost/$jobName/" +--- +kind: Service +apiVersion: v1 +metadata: + name: heimdall +spec: + selector: + app: heimdall + ports: + - port: 8080 +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: heimdall +spec: + ingressClassName: nginx + rules: + - host: localhost + http: + paths: + - path: /heimdall(/|$)(.*) + pathType: ImplementationSpecific + backend: + service: + name: heimdall-service + port: + number: 8080 +--- diff --git a/flink-on-kubernetes/hello-world-job/Dockerfile b/flink-on-kubernetes/hello-world-job/Dockerfile new file mode 100644 index 0000000..31a31a4 --- /dev/null +++ b/flink-on-kubernetes/hello-world-job/Dockerfile @@ -0,0 +1,16 @@ +FROM maven:3-eclipse-temurin-21 AS build + +COPY pom.xml . +RUN mvn dependency:go-offline + +COPY . . +RUN mvn clean verify -o + +FROM flink:1.20-java17 + +RUN cd /opt/flink/lib && \ + curl -sO https://repo1.maven.org/maven2/org/apache/logging/log4j/log4j-layout-template-json/2.17.1/log4j-layout-template-json-2.17.1.jar +# curl -sO https://repo1.maven.org/maven2/com/fasterxml/jackson/core/jackson-core/2.18.2/jackson-core-2.18.2.jar && \ +# curl -sO https://repo1.maven.org/maven2/com/fasterxml/jackson/core/jackson-annotations/2.18.2/jackson-annotations-2.18.2.jar && \ +# curl -sO https://repo1.maven.org/maven2/com/fasterxml/jackson/core/jackson-databind/2.18.2/jackson-databind-2.18.2.jar +COPY --from=build --chown=flink:flink target/hello-world-job-1.0.jar /opt/flink-jobs/hello-world-job-1.0.jar diff --git a/flink-on-kubernetes/hello-world-job/pom.xml b/flink-on-kubernetes/hello-world-job/pom.xml new file mode 100644 index 0000000..1c54a00 --- /dev/null +++ b/flink-on-kubernetes/hello-world-job/pom.xml @@ -0,0 +1,182 @@ + + + 4.0.0 + + co.decodable.demos + hello-world-job + 1.0 + jar + + Flink Quickstart Job + + + UTF-8 + 1.20.0 + 17 + 2.12 + ${target.java.version} + ${target.java.version} + 2.17.1 + + + + + apache.snapshots + Apache Development Snapshot Repository + https://repository.apache.org/content/repositories/snapshots/ + + false + + + true + + + + + + + + + org.apache.flink + flink-streaming-java + ${flink.version} + provided + + + org.apache.flink + flink-clients + ${flink.version} + provided + + + + + + + + + + org.apache.logging.log4j + log4j-slf4j-impl + ${log4j.version} + runtime + + + org.apache.logging.log4j + log4j-api + ${log4j.version} + runtime + + + org.apache.logging.log4j + log4j-core + ${log4j.version} + runtime + + + org.apache.logging.log4j + log4j-layout-template-json + ${log4j.version} + runtime + + + + + + + + + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.1 + + ${target.java.version} + ${target.java.version} + + + + + + + org.apache.maven.plugins + maven-shade-plugin + 3.1.1 + + + + package + + shade + + + false + + + org.apache.flink:flink-shaded-force-shading + com.google.code.findbugs:jsr305 + org.slf4j:* + org.apache.logging.log4j:* + + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + + co.decodable.demos.flinkhelloworld.DataStreamJob + + + + + + + + + + + + + + + diff --git a/flink-on-kubernetes/hello-world-job/src/main/java/co/decodable/demos/flinkhelloworld/DataStreamJob.java b/flink-on-kubernetes/hello-world-job/src/main/java/co/decodable/demos/flinkhelloworld/DataStreamJob.java new file mode 100644 index 0000000..c4a9c66 --- /dev/null +++ b/flink-on-kubernetes/hello-world-job/src/main/java/co/decodable/demos/flinkhelloworld/DataStreamJob.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package co.decodable.demos.flinkhelloworld; + +import org.apache.flink.api.common.eventtime.WatermarkStrategy; +import org.apache.flink.api.common.typeinfo.Types; +import org.apache.flink.api.connector.source.util.ratelimit.RateLimiterStrategy; +import org.apache.flink.connector.datagen.source.DataGeneratorSource; +import org.apache.flink.connector.datagen.source.GeneratorFunction; +import org.apache.flink.streaming.api.datastream.DataStreamSource; +import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; +import org.apache.flink.streaming.api.functions.sink.PrintSink; + +/** + * Skeleton for a Flink DataStream Job. + * + *

For a tutorial how to write a Flink application, check the + * tutorials and examples on the Flink Website. + * + *

To package your application into a JAR file for execution, run + * 'mvn clean package' on the command line. + * + *

If you change the name of the main class (with the public static void main(String[] args)) + * method, change the respective entry in the POM.xml file (simply search for 'mainClass'). + */ +public class DataStreamJob { + + public static void main(String[] args) throws Exception { + final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); +// env.setParallelism(2); +// env.enableCheckpointing(5000, CheckpointingMode.EXACTLY_ONCE); + + /* + // sets the checkpoint storage where checkpoint snapshots will be written + Configuration config = new Configuration(); + config.set(CheckpointingOptions.CHECKPOINT_STORAGE, "filesystem"); + config.set(CheckpointingOptions.CHECKPOINTS_DIRECTORY, "s3://flink-data/checkpoints"); + config.set(CheckpointingOptions.SAVEPOINT_DIRECTORY, "s3://flink-data/savepoints"); + config.setString("s3.access.key", "minio"); + config.setString("s3.secret.key", "minio123"); + config.setString("s3.endpoint", "http://localhost:9000"); + config.setString("s3.path.style.access", "true"); + + FileSystem.initialize(config, null); + + env.configure(config); +*/ + + GeneratorFunction generatorFunction = index -> "Number: " + index; + long numberOfRecords = 100_000; + + DataGeneratorSource source = new DataGeneratorSource<>( + generatorFunction, + numberOfRecords, + RateLimiterStrategy.perSecond(1), + Types.STRING); + + DataStreamSource stream = + env.fromSource(source, + WatermarkStrategy.noWatermarks(), + "Generator Source"); + + PrintSink sink = new PrintSink<>(true); + + stream.sinkTo(sink); + + env.execute("Flink Java API Skeleton"); + } +} diff --git a/flink-on-kubernetes/helm-values.yaml b/flink-on-kubernetes/helm-values.yaml new file mode 100644 index 0000000..31e53b4 --- /dev/null +++ b/flink-on-kubernetes/helm-values.yaml @@ -0,0 +1,9 @@ +defaultConfiguration: + create: true + append: true + flink-conf.yaml: |+ + kubernetes.operator.default-configuration.flink-version.v1_18.env.java.opts.all: --add-exports=java.base/sun.net.util=ALL-UNNAMED --add-exports=java.rmi/sun.rmi.registry=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED --add-exports=java.security.jgss/sun.security.krb5=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.text=ALL-UNNAMED --add-opens=java.base/java.time=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.locks=ALL-UNNAMED + kubernetes.operator.default-configuration.flink-version.v1_19.env.java.default-opts.all: --add-exports=java.base/sun.net.util=ALL-UNNAMED --add-exports=java.rmi/sun.rmi.registry=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED --add-exports=java.security.jgss/sun.security.krb5=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.text=ALL-UNNAMED --add-opens=java.base/java.time=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.locks=ALL-UNNAMED + kubernetes.operator.default-configuration.flink-version.v1_20.env.java.default-opts.all: --add-exports=java.base/sun.net.util=ALL-UNNAMED --add-exports=java.rmi/sun.rmi.registry=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED --add-exports=jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED --add-exports=java.security.jgss/sun.security.krb5=ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.text=ALL-UNNAMED --add-opens=java.base/java.time=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.locks=ALL-UNNAMED + + kubernetes.operator.jm-deployment.shutdown-ttl: 30000 diff --git a/flink-on-kubernetes/kind-cluster.yml b/flink-on-kubernetes/kind-cluster.yml new file mode 100644 index 0000000..0b189ff --- /dev/null +++ b/flink-on-kubernetes/kind-cluster.yml @@ -0,0 +1,15 @@ +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +nodes: + - role: control-plane + - role: worker + extraMounts: + - hostPath: /tmp/kind-worker + containerPath: /files + extraPortMappings: + - containerPort: 80 + hostPort: 80 + protocol: TCP + - containerPort: 443 + hostPort: 443 + protocol: TCP diff --git a/flink-on-kubernetes/logging/elastic-flow.yaml b/flink-on-kubernetes/logging/elastic-flow.yaml new file mode 100644 index 0000000..9d52575 --- /dev/null +++ b/flink-on-kubernetes/logging/elastic-flow.yaml @@ -0,0 +1,22 @@ +apiVersion: logging.banzaicloud.io/v1beta1 +kind: Flow +metadata: + name: es-flow +spec: + filters: + - tag_normaliser: {} + - parser: + remove_key_name_field: true + reserve_data: true + parse: + type: json + time_key: "@timestamp" + match: + - select: + labels: + component: jobmanager + - select: + labels: + component: taskmanager + globalOutputRefs: + - es-output diff --git a/flink-on-kubernetes/logging/elastic-output.yaml b/flink-on-kubernetes/logging/elastic-output.yaml new file mode 100644 index 0000000..ef697bd --- /dev/null +++ b/flink-on-kubernetes/logging/elastic-output.yaml @@ -0,0 +1,21 @@ +apiVersion: logging.banzaicloud.io/v1beta1 +kind: ClusterOutput +metadata: + name: es-output +spec: + elasticsearch: + host: quickstart-es-http.logging.svc.cluster.local + port: 9200 + scheme: https + ssl_verify: false + ssl_version: TLSv1_2 + user: elastic + password: + valueFrom: + secretKeyRef: + name: quickstart-es-elastic-user + key: elastic + buffer: + timekey: 1m + timekey_wait: 30s + timekey_use_utc: true diff --git a/flink-on-kubernetes/logging/elastic.yaml b/flink-on-kubernetes/logging/elastic.yaml new file mode 100644 index 0000000..2be749e --- /dev/null +++ b/flink-on-kubernetes/logging/elastic.yaml @@ -0,0 +1,11 @@ +apiVersion: elasticsearch.k8s.elastic.co/v1 +kind: Elasticsearch +metadata: + name: quickstart +spec: + version: 8.17.0 + nodeSets: + - name: default + count: 1 + config: + node.store.allow_mmap: false diff --git a/flink-on-kubernetes/logging/kibana.yaml b/flink-on-kubernetes/logging/kibana.yaml new file mode 100644 index 0000000..42a699c --- /dev/null +++ b/flink-on-kubernetes/logging/kibana.yaml @@ -0,0 +1,9 @@ +apiVersion: kibana.k8s.elastic.co/v1 +kind: Kibana +metadata: + name: quickstart +spec: + version: 8.17.0 + count: 1 + elasticsearchRef: + name: quickstart diff --git a/flink-on-kubernetes/logging/logging.yaml b/flink-on-kubernetes/logging/logging.yaml new file mode 100644 index 0000000..9b32801 --- /dev/null +++ b/flink-on-kubernetes/logging/logging.yaml @@ -0,0 +1,8 @@ +apiVersion: logging.banzaicloud.io/v1beta1 +kind: Logging +metadata: + name: default-logging-simple +spec: + fluentd: {} + fluentbit: {} + controlNamespace: logging diff --git a/flink-on-kubernetes/storage/minio-deployment.yaml b/flink-on-kubernetes/storage/minio-deployment.yaml new file mode 100644 index 0000000..89d8f33 --- /dev/null +++ b/flink-on-kubernetes/storage/minio-deployment.yaml @@ -0,0 +1,50 @@ +apiVersion: apps/v1 # for k8s versions before 1.9.0 use apps/v1beta2 and before 1.8.0 use extensions/v1beta1 +kind: Deployment +metadata: + # This name uniquely identifies the Deployment + name: minio-deployment +spec: + selector: + matchLabels: + app: minio + strategy: + type: Recreate + template: + metadata: + labels: + # Label is used as selector in the service. + app: minio + spec: + # Refer to the PVC created earlier + volumes: + - name: storage + persistentVolumeClaim: + # Name of the PVC created earlier + claimName: minio-pv-claim + containers: + - name: minio + # Pulls the default Minio image from Docker Hub + image: minio/minio:latest + args: + - server + - /storage + - --console-address + - :9001 + env: + # Minio access key and secret key + - name: MINIO_ROOT_USER + value: "minio" + - name: MINIO_ROOT_PASSWORD + value: "minio123" + # Disabling animation due to https://github.com/minio/console/issues/2539 + - name: MINIO_BROWSER_LOGIN_ANIMATION + value: "off" + ports: + - containerPort: 9000 + hostPort: 9000 + - containerPort: 9001 + hostPort: 9001 + # Mount the volume into the pod + volumeMounts: + - name: storage # must match the volume name, above + mountPath: "/storage" diff --git a/flink-on-kubernetes/storage/minio-pvc.yaml b/flink-on-kubernetes/storage/minio-pvc.yaml new file mode 100644 index 0000000..5afa596 --- /dev/null +++ b/flink-on-kubernetes/storage/minio-pvc.yaml @@ -0,0 +1,30 @@ +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: kind-pv +spec: + storageClassName: standard + accessModes: + - ReadWriteOnce + capacity: + storage: 8Gi + hostPath: + path: /files/ +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + # This name uniquely identifies the PVC. Will be used in deployment below. + name: minio-pv-claim + labels: + app: minio-storage-claim +spec: + # Read more about access modes here: http://kubernetes.io/docs/user-guide/persistent-volumes/#access-modes + accessModes: + - ReadWriteOnce + volumeName: kind-pv + resources: + # This is the request for storage. Should be available in the cluster. + requests: + storage: 8Gi diff --git a/flink-on-kubernetes/storage/minio-service.yaml b/flink-on-kubernetes/storage/minio-service.yaml new file mode 100644 index 0000000..19e7a83 --- /dev/null +++ b/flink-on-kubernetes/storage/minio-service.yaml @@ -0,0 +1,20 @@ +apiVersion: v1 +kind: Service +metadata: + name: minio-service + namespace: default + labels: + app: minio-service +spec: + type: ClusterIP + selector: + app: minio + ports: + - name: api + protocol: TCP + port: 9000 + targetPort: 9000 + - name: webui + protocol: TCP + port: 9001 + targetPort: 9001 \ No newline at end of file