Initial commit.

conf/conf.toml

@@ -0,0 +1,9 @@
+# Configuration constants
+
+[kafka]
+KAFKA_BOOTSTRAP_SERVERS = "localhost:9092"
+KAFKA_TOPIC = "my_topic"
+
+[flink]
+FLINK_JOB_MANAGER = "localhost:8081"
+FLINK_PARALLELISM = 4

conf/flink-config.yaml

@@ -0,0 +1,36 @@
+# Flink configuration file
+
+# JobManager configuration
+jobmanager.rpc.address: localhost
+jobmanager.rpc.port: 6123
+jobmanager.heap.size: 1024m
+jobmanager.rpc.numThreads: 8
+
+# TaskManager configuration
+taskmanager.memory.process.size: 1024m
+taskmanager.memory.managed.size: 768m
+taskmanager.numberOfTaskSlots: 2
+taskmanager.rpc.numThreads: 8
+
+# High Availability configuration
+high-availability: zookeeper
+high-availability.storageDir: hdfs://localhost:9000/flink/recovery
+high-availability.zookeeper.quorum: localhost:2181
+high-availability.zookeeper.path.root: /flink
+
+# Parallelism and resource allocation
+parallelism.default: 2
+parallelism.max: 8
+taskmanager.cpu.cores: 4
+
+# State backend configuration
+state.backend: rocksdb
+state.checkpoints.dir: hdfs://localhost:9000/flink/checkpoints
+state.backend.rocksdb.memory.managed: true
+state.backend.rocksdb.predefined-options: FLASH_SSD_OPTIMIZED
+
+# Logging configuration
+log4j.logger.org.apache.flink: INFO,console
+log4j.logger.org.apache.flink.client.cli: INFO,console
+log4j.logger.org.apache.flink.runtime.rest: INFO,console
+log4j.logger.org.apache.flink.runtime.rpc: INFO,console

requirements.txt

@@ -0,0 +1,7 @@
+asyncio
+aiohttp
+aioresponses
+apache-flink
+apache-kafka
+pandas
+pyflink

scripts/run.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+# Start Flink cluster
+flink/bin/start-cluster.sh
+
+# Submit PyFlink job
+flink/bin/flink run -py ./scripts/word_count.py
+
+# Stop Flink cluster
+flink/bin/stop-cluster.sh

setup/setup.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+conda create -n flink python=3.8
+conda activate flink
+
+python -m pip install apache-flink
+pip install pyflink
+
+export FLINK_CONDA_HOME=$(dirname $(dirname $CONDA_EXE))