docker-compose.yml

# References:
# bitnami/spark: https://raw.githubusercontent.com/bitnami/containers/main/bitnami/spark/docker-compose.yml
# apache/hadoop: https://hub.docker.com/r/apache/hadoop
# apache/hive: https://hive.apache.org/developement/quickstart/

x-hadoop-common: &hadoop-common
  image: apache/hadoop:3
  env_file: ./containers/hadoop/hadoop.env

services:
  # Airflow
  airflow:
    build: 
      context: ./containers/airflow
    env_file: ./containers/airflow/airflow.env
    volumes:
      - ./src/dags:/opt/airflow/dags
      - ./src/jobs:/opt/airflow/jobs
      - ./src/config:/opt/airflow/config
      - ./logs/airflow:/opt/airflow/logs
      - ./data:/data
    ports:
      - "8080:8080"

  # Spark
  spark-master:
    image: bitnami/spark:3.5
    env_file: ./containers/spark/spark-master.env
    ports:
      - 8081:8080

  spark-worker:
    image: bitnami/spark:3.5
    env_file: ./containers/spark/spark-worker.env

  # HDFS
  hdfs-namenode:
    <<: *hadoop-common
    hostname: namenode
    command: ["hdfs", "namenode"]
    ports:
      - 9870:9870
    environment:
      - ENSURE_NAMENODE_DIR=/tmp/hadoop-hadoop/dfs/name
  
  hdfs-datanode:
    <<: *hadoop-common
    command: ["hdfs", "datanode"]

  # Hive
  hive-metastore:
    image: apache/hive:4.0.0-alpha-2
    depends_on:
      - hdfs-namenode
    hostname: hive-metastore
    environment:
      - HIVE_CUSTOM_CONF_DIR=/hive-custom-conf
      - SERVICE_NAME=metastore
    volumes:
      - ./containers/hive:/hive-custom-conf

  hive-server:
    image: apache/hive:4.0.0-alpha-2
    depends_on:
      - hive-metastore
    hostname: hive-server
    environment:
      - HIVE_SERVER2_THRIFT_PORT=10000
      - HIVE_CUSTOM_CONF_DIR=/hive-custom-conf
      - IS_RESUME=true
      - SERVICE_NAME=hiveserver2
    volumes:
      - ./logs/hiveserver:/tmp/hive/
      - ./containers/hive:/hive-custom-conf
    ports:
      - 10002:10002