docker-compose.yml

version: '2'
services:
  kafka:
    image: wurstmeister/kafka
    ports:
      - "9092:9092"
    environment:
      KAFKA_ADVERTISED_HOST_NAME: kafka
      KAFKA_CREATE_TOPICS: "android:1:1,acceleration:1:1"
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
    volumes:
      - /var/run/docker.sock:/var/run/docker.sock

  zookeeper:
    image: wurstmeister/zookeeper
    ports:
      - "2181:2181"

  master:
    image: jairamc/spark-kafka-streaming
    hostname: master
    environment:
      SPARK_HOME: /usr/local/spark
      MASTER: spark://master:7077
      SPARK_CONF_DIR: /usr/local/spark/conf
      SPARK_PUBLIC_DNS: localhost
    ports:
      - 8080:8080
      - 7077:7077
      - 6066:6066
    command: /usr/local/spark/bin/spark-class org.apache.spark.deploy.master.Master -h master

  slave:
    image: jairamc/spark-kafka-streaming
    hostname: worker
    depends_on:
      - master
    environment:
      SPARK_HOME: /usr/local/spark
      SPARK_CONF_DIR: /usr/local/spark/conf
      SPARK_WORKER_CORES: 2
      SPARK_WORKER_MEMORY: 1g
      SPARK_WORKER_PORT: 8881
      SPARK_WORKER_WEBUI_PORT: 8081
      SPARK_PUBLIC_DNS: localhost
    ports:
      - 8081:8081
    command: /usr/local/spark/bin/spark-class org.apache.spark.deploy.worker.Worker spark://master:7077

  notebook:
    image: jairamc/spark-notebook
    environment:
      SPARK_OPTS: '--master=spark://master:7077'
    command: jupyter notebook --ip=0.0.0.0
    ports:
      - 8888:8888
    volumes:
      - ./notebooks:/notebooks