docker-compose.yml

version: '3.7'

services:


  #-------------------------------#
  # zookeeper                     #
  #-------------------------------#

  zookeeper:
    image: confluentinc/cp-zookeeper:6.1.1
    container_name: zookeeper
    hostname: zookeeper
    ports:
        - "2181:2181"
    environment:
        ZOOKEEPER_CLIENT_PORT: 2181
        ZOOKEEPER_SERVER_ID: "1"
        ZOOKEEPER_HEAP_OPTS: -Xmx512m -Xmx512m
    # mem_limit: 1g
    networks: 
        - fmap-int


  #-------------------------------#
  # kafka server                  #
  #-------------------------------#

  kafkaserver:
    image: confluentinc/cp-kafka:6.1.1
    container_name: kafkaserver
    hostname: kafkaserver
    depends_on:
      - zookeeper
    ports:
      - "9092:9092"
      - "9101:9101"
    environment:
      KAFKA_BROKER_ID: 0
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafkaserver:9092
      KAFKA_DEFAULT_REPLICATION_FACTOR: 2
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 2
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 2
      KAFKA_HEAP_OPTS: -Xmx512m -Xmx512m
    # mem_limit: 1g
    networks: 
        - fmap-int
        - fmap-com 


  #-------------------------------#
  # kafka (create topics)         #
  #-------------------------------#

  kafka-create-topics:
    image: confluentinc/cp-kafka:6.1.1
    depends_on:
        - kafkaserver
    container_name: kafka-create-topics
    hostname: kafka-create-topics
    command: "bash -c 'echo Waiting for Kafka to be ready... && \
                        cub kafka-ready -b kafkaserver:9092 1 200 && \
                        kafka-topics --create --topic rds-signal --if-not-exists --zookeeper zookeeper:2181 --partitions 1 --replication-factor 1 && \
                        kafka-topics --create --topic rds-signal-output --if-not-exists --zookeeper zookeeper:2181 --partitions 1 --replication-factor 1 &&\
                        sleep infinity'"
    environment:
        KAFKA_BROKER_ID: ignored
        KAFKA_ZOOKEEPER_CONNECT: ignored
        KAFKA_HEAP_OPTS: -Xmx512m -Xmx512m
    # mem_limit: 1g
    networks:
        - fmap-int


  #-------------------------------#
  # kafka stream (🤮)             #
  #-------------------------------#

  kafkastream:
    build: 
        context: kafkastream
    image: fmap:stream
    container_name: kafkastream
    depends_on: 
        - kafkaserver
        - kafka-create-topics
    networks:
        - fmap-int


  #-------------------------------#
  # kafka (worker)                #
  #-------------------------------#

  kafkaworker1:
    image: confluentinc/cp-kafka:6.1.1
    container_name: kafkaWorker1
    depends_on:
      - zookeeper
    environment:
      KAFKA_BROKER_ID: 1
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafkaworker1:9092
      KAFKA_DEFAULT_REPLICATION_FACTOR: 2
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 2
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 2
      KAFKA_HEAP_OPTS: -Xmx512m -Xmx512m
    # mem_limit: 1g
    networks: 
        - fmap-int
        - fmap-com 
  

  #-------------------------------#
  # kafka web UI                  #
  #-------------------------------#

  webui:
    image: provectuslabs/kafka-ui:latest
    container_name: kafkaWebUI
    environment:
        KAFKA_CLUSTERS_0_NAME: my_cluster
        KAFKA_CLUSTERS_0_ZOOKEEPER: zookeeper:2181
        KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: kafkaserver:9092
        KAFKA_CLUSTERS_0_HEAP_OPTS: -Xmx512m -Xmx512m
    ports: 
        - 8080:8080
    depends_on:
        - kafkaserver
    # mem_limit: 1g
    networks: 
        - fmap-int


  #-------------------------------#
  # logstash                      #
  #-------------------------------#

  logstash:
    container_name: logstash
    image: docker.elastic.co/logstash/logstash:7.11.2
    volumes: 
        - $PWD/logstash/pipeline/:/usr/share/logstash/pipeline/
        - $PWD/logs/:/usr/share/logstash/logs/
    depends_on: 
        - kafkaserver
    environment: 
        - "LS_JAVA_OPTS=-Xms1g -Xmx1g"
    # mem_limit: 2g
    networks: 
      - fmap-ext 
      - fmap-com 
  

  #-------------------------------#
  # microcontrollers simulator    #
  #-------------------------------#

  simulator: 
    container_name: simulator
    image: python:3.9.5
    working_dir: /usr/src/sim
    command: bash -c "pip install pandas && python app.py"
    volumes: 
        - $PWD/logs/:/usr/src/logs/
        - $PWD/simulator/:/usr/src/sim
    networks: 
      - fmap-ext 
      - fmap-com 


  #-------------------------------#
  # Elasticsearch                 #
  #-------------------------------#

  elasticsearch:
    container_name: elasticsearch
    hostname: elasticsearch
    image: docker.elastic.co/elasticsearch/elasticsearch:7.10.2
    ports:
      - "9200:9200"
      - "9300:9300"      
    environment:
      - xpack.security.enabled=false
      - discovery.type=single-node
      - bootstrap.memory_lock=true
      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
      - cluster.routing.allocation.disk.threshold_enabled=false
    ulimits:
      memlock:
        soft: -1
        hard: -1
    # mem_limit: 1g
    networks: 
    - fmap-int 


  #-------------------------------#
  # kafka consumer to ES (python) #
  #-------------------------------#

  kafka-to-es: 
    container_name: kafka-to-es
    build: 
      context: kafka-to-es
    image: fmap:kafka-to-es
    depends_on:
      - kafkaserver
      - elasticsearch
    networks: 
      - fmap-int


  #-------------------------------#
  # Kibana                        #
  #-------------------------------#

  kibana:
    image: docker.elastic.co/kibana/kibana:7.10.2
    container_name: kibana
    hostname: kibana
    ports:
      - "5601:5601"
    depends_on: 
      - elasticsearch
    networks: 
      - fmap-int   
    

  #-------------------------------#
  # spark & spark streaming       #
  #-------------------------------#
  
  spark:
    build: 
        context: spark
    image: fmap:spark
    container_name: spark
    command: ["structured_streaming_example.py", "org.apache.spark:spark-sql-kafka-0-10_2.12:3.1.1,org.elasticsearch:elasticsearch-spark-30_2.12:7.12.1"]
    environment: 
        SPARK_ACTION: "spark-submit-python"
    ports: 
        - "4040:4040"
    depends_on: 
        - kafkaserver
        - elasticsearch
    networks: 
        - fmap-int
    deploy:
      resources:
        limits:
          cpus: '1.0'
          memory: 1g
        reservations:
          cpus: '0.5'
          memory: 500m

networks:

  fmap-int:
    name: fmap-int
    driver: bridge

  fmap-ext:
    name: fmap-ext
    driver: bridge
  
  fmap-com: 
    name: fmap-com
    driver: bridge