docker-compose.yml

# NOTE: local development only
services:
  onetl:
    image: mtsrus/onetl:develop
    build:
      dockerfile: ./docker/Dockerfile
      context: .
      target: base
      args:
        SPARK_VERSION: 3.5.4
    env_file: .env.docker
    volumes:
      - ./:/app/
    networks:
      - onetl
      - default
    ulimits:
      # https://stackoverflow.com/a/56895801
      nofile:
        soft: 10000
        hard: 10000
    # no dependencies from other containers to allow running limited set of tests instead of all
    profiles: [onetl]

  greenplum:
    image: ${GREENPLUM_IMAGE:-andruche/greenplum:6}
    restart: unless-stopped
    environment:
      TZ: UTC
      POSTGRESQL_DATABASE: onetl
      POSTGRESQL_USERNAME: onetl
      POSTGRESQL_PASSWORD: 123UsedForTestOnly@!
    ports:
      - 5433:5432
    volumes:
      - ./docker/greenplum/custom_entrypoint.sh:/entrypoint.sh
      - ./docker/greenplum/initdb.d/:/container-entrypoint-initdb.d/
    networks:
      - onetl
    sysctls:
      - net.ipv6.conf.all.disable_ipv6=1
    healthcheck:
      test: ["CMD", "/usr/local/gpdb/bin/pg_isready"]
      interval: 10s
      timeout: 5s
      retries: 10
    profiles: [greenplum, db, all]

  pgbouncer-transaction-gp:
    image: ${PGBOUNCER_IMAGE:-bitnami/pgbouncer:latest}
    restart: unless-stopped
    environment:
      TZ: UTC
      PGBOUNCER_DATABASE: onetl
      PGBOUNCER_POOL_MODE: transaction
      POSTGRESQL_HOST: greenplum
      POSTGRESQL_PORT: 5432
      POSTGRESQL_DATABASE: onetl
      POSTGRESQL_USERNAME: onetl
      POSTGRESQL_PASSWORD: 123UsedForTestOnly@!
    ports:
      - 6433:6432
    networks:
      - onetl
    healthcheck:
      test: ["CMD-SHELL", "PGPASSWORD=$$POSTGRESQL_PASSWORD psql -h localhost -p 6432 -d onetl -U $$POSTGRESQL_USERNAME -tc 'SELECT 1;' | grep -q 1"]
      interval: 10s
      timeout: 5s
      retries: 10
    profiles: [greenplum, db, all]

  clickhouse:
    image: ${CLICKHOUSE_IMAGE:-clickhouse/clickhouse-server:latest-alpine}
    restart: unless-stopped
    environment:
      TZ: UTC
      # yandex/clickhouse-server doesn't read these options at all, so we using default values
      CLICKHOUSE_DB: default
      CLICKHOUSE_USER: default
      CLICKHOUSE_PASSWORD: 123UsedForTestOnly@!
      CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: 1
    ports:
      - 8123:8123
      - 9001:9000
    networks:
      - onetl
    sysctls:
      - net.ipv6.conf.all.disable_ipv6=1
    healthcheck:
      test: ["CMD-SHELL", 'wget --quiet --no-verbose --tries=1 --spider --header "X-ClickHouse-User: $$CLICKHOUSE_USER" --header "X-ClickHouse-Key: $$CLICKHOUSE_PASSWORD" "http://localhost:8123/?query=SELECT%201&database=$$CLICKHOUSE_DB" || exit 1']
      interval: 10s
      timeout: 5s
      retries: 10
    profiles: [clickhouse, jdbc, db, all]

  zookeeper:
    image: ${ZOOKEEPER_IMAGE:-bitnami/zookeeper:3.8}
    restart: unless-stopped
    ports:
      - 2181:2181
    networks:
      - onetl
    environment:
      TZ: UTC
      ALLOW_ANONYMOUS_LOGIN: 'yes'
    healthcheck:
      test: ["CMD-SHELL", "nc -z localhost 2181 || exit"]
      interval: 10s
      timeout: 5s
      retries: 5
    profiles: [kafka, db, all]

  kafka:
    image: ${KAFKA_IMAGE:-bitnami/kafka:latest}
    restart: unless-stopped
    ports:
      - 9093:9093
      - 9095:9095
    environment:
      TZ: UTC
      ALLOW_PLAINTEXT_LISTENER: 'yes'
      KAFKA_ENABLE_KRAFT: 'no'
      KAFKA_CLIENT_USERS: onetl
      KAFKA_CLIENT_PASSWORDS: 123UsedForTestOnly@!
      KAFKA_ZOOKEEPER_BOOTSTRAP_SCRAM_USERS: 'true'
      KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: 'true'
      KAFKA_CFG_ZOOKEEPER_CONNECT: zookeeper:2181
      KAFKA_CFG_INTER_BROKER_LISTENER_NAME: INTERNAL_PLAINTEXT_ANONYMOUS
      KAFKA_CFG_LISTENERS: INTERNAL_PLAINTEXT_ANONYMOUS://:9092,EXTERNAL_PLAINTEXT_ANONYMOUS://:9093,INTERNAL_PLAINTEXT_SASL://:9094,EXTERNAL_PLAINTEXT_SASL://:9095
      KAFKA_CFG_ADVERTISED_LISTENERS: INTERNAL_PLAINTEXT_ANONYMOUS://kafka:9092,EXTERNAL_PLAINTEXT_ANONYMOUS://localhost:9093,INTERNAL_PLAINTEXT_SASL://kafka:9094,EXTERNAL_PLAINTEXT_SASL://localhost:9095
      KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL_PLAINTEXT_ANONYMOUS:PLAINTEXT,EXTERNAL_PLAINTEXT_ANONYMOUS:PLAINTEXT,INTERNAL_PLAINTEXT_SASL:SASL_PLAINTEXT,EXTERNAL_PLAINTEXT_SASL:SASL_PLAINTEXT
      KAFKA_CFG_SASL_ENABLED_MECHANISMS: PLAIN,SCRAM-SHA-256,SCRAM-SHA-512
    networks:
      - onetl
    depends_on:
      zookeeper:
        condition: service_healthy
    healthcheck:
      test: ["CMD-SHELL", "kafka-topics.sh --bootstrap-server 127.0.0.1:9092 --list"]
      interval: 10s
      timeout: 5s
      retries: 5
    profiles: [kafka, db, all]

  mongodb:
    image: ${MONGODB_IMAGE:-mongo:latest}
    restart: unless-stopped
    ports:
      - 27017:27017
    environment:
      TZ: UTC
      MONGO_INITDB_ROOT_DB: admin
      MONGO_INITDB_ROOT_USERNAME: admin
      MONGO_INITDB_ROOT_PASSWORD: 123AdminUserForTests@!
    volumes:
      - ./docker/mongodb/:/docker-entrypoint-initdb.d/
    healthcheck:
      test: ["CMD-SHELL", echo 'db.runCommand("ping").ok' | $(which mongosh || which mongo) "$$MONGO_INITDB_ROOT_DB" --quiet]
      interval: 10s
      timeout: 5s
      retries: 5
    networks:
      - onetl
    profiles: [mongodb, db, all]

  mssql:
    image: ${MSSQL_IMAGE:-mcr.microsoft.com/mssql/server:latest}
    restart: unless-stopped
    environment:
      TZ: UTC
      ACCEPT_EULA: 'Y'
      MSSQL_SA_PASSWORD: 123AdminUserForTests@!
    ports:
      - 1433:1433
    volumes:
      - ./docker/mssql/:/usr/config/
    entrypoint: ["/usr/config/entrypoint.sh"]
    networks:
      - onetl
    platform: linux/amd64
    healthcheck:
      # Container is healthy only when database is created
      # TODO: replace with SELECT after fixing sqlcmd location: https://github.com/microsoft/mssql-docker/issues/892
      test: ["CMD-SHELL", "ls -lsah /var/opt/mssql/data/onetl.mdf"]
      interval: 10s
      timeout: 5s
      retries: 5
    profiles: [mssql, jdbc, db, all]

  mysql:
    image: ${MYSQL_IMAGE:-mysql:latest}
    restart: unless-stopped
    environment:
      TZ: UTC
      MYSQL_ROOT_PASSWORD: 123RootUserForTestsOnly@!
      MYSQL_DATABASE: onetl
      MYSQL_USER: onetl
      MYSQL_PASSWORD: 123UsedForTestOnly@!
    ports:
      - 3306:3306
    networks:
      - onetl
    platform: linux/amd64
    healthcheck:
      test: ["CMD-SHELL", "/usr/bin/mysql --user=root --password=\"$$MYSQL_ROOT_PASSWORD\" --execute \"SHOW DATABASES;\""]
      interval: 2s
      timeout: 20s
      retries: 10
    profiles: [mysql, jdbc, db, all]

  postgres:
    image: ${POSTGRES_IMAGE:-postgres:alpine}
    restart: unless-stopped
    environment:
      TZ: UTC
      POSTGRES_USER: onetl
      POSTGRES_DB: onetl
      POSTGRES_PASSWORD: 123UsedForTestOnly@!
    ports:
      - 5432:5432
    networks:
      - onetl
    healthcheck:
      test: ["CMD", "pg_isready"]
      interval: 10s
      timeout: 5s
      retries: 10
    profiles: [postgres, jdbc, db, all]

  pgbouncer-transaction-pg:
    image: ${PGBOUNCER_IMAGE:-bitnami/pgbouncer:latest}
    restart: unless-stopped
    environment:
      TZ: UTC
      PGBOUNCER_DATABASE: onetl
      PGBOUNCER_POOL_MODE: transaction
      POSTGRESQL_HOST: postgres
      POSTGRESQL_PORT: 5432
      POSTGRESQL_DATABASE: onetl
      POSTGRESQL_USERNAME: onetl
      POSTGRESQL_PASSWORD: 123UsedForTestOnly@!
    ports:
      - 6432:6432
    networks:
      - onetl
    healthcheck:
      test: ["CMD-SHELL", "PGPASSWORD=$$POSTGRESQL_PASSWORD psql -h localhost -p 6432 -d onetl -U $$POSTGRESQL_USERNAME -tc 'SELECT 1;' | grep -q 1"]
      interval: 10s
      timeout: 5s
      retries: 10
    profiles: [postgres, jdbc, db, all]

  hdfs:
    image: ${HDFS_IMAGE:-mtsrus/hadoop:hadoop2-hdfs}
    hostname: hdfs
    restart: unless-stopped
    environment:
      TZ: UTC
    ports:
      - 9820:9820 # HDFS IPC
      - 9870:9870 # WebHDFS
      - 9864:9864 # Datanode UI
    volumes:
      - ./docker/hdfs/conf/hadoop/:/var/hadoop/conf/
    networks:
      - onetl
    # healthcheck is defined in image
    profiles: [hdfs, file_df, file, all]

  oracle:
    image: ${ORACLE_IMAGE:-gvenzl/oracle-free:slim-faststart}
    restart: unless-stopped
    environment:
      TZ: UTC
      ORACLE_PASSWORD: 123AdminUserForTests@!
      APP_USER: onetl
      APP_USER_PASSWORD: 123UsedForTestOnly@!
    ports:
      - 1522:1521
    volumes:
      - ./docker/oracle/:/container-entrypoint-initdb.d
    networks:
      - onetl
    healthcheck:
      test: ["CMD", "healthcheck.sh"]
      interval: 10s
      timeout: 5s
      retries: 10
    platform: linux/amd64
    profiles: [oracle, jdbc, db, all]

  ftp:
    image: ${FTP_IMAGE:-chonjay21/ftps:latest}
    restart: unless-stopped
    environment:
      TZ: UTC
      APP_UID: 1000
      APP_GID: 1000
      APP_USER_NAME: onetl
      APP_USER_PASSWD: 123UsedForTestOnly@!
      USE_SSL: 'false'
      PASSV_MIN_PORT: 30000
      PASSV_MAX_PORT: 30010
    ports:
      - 2121:21
      - 30000-30010:30000-30010
    volumes:
      - ./docker/ftp/on_post_init.sh:/sources/ftps/eventscripts/on_post_init.sh
    networks:
      - onetl
    healthcheck:
      test: ["CMD-SHELL", "curl --silent --show-error --fail -u \"$$APP_USER_NAME:$$APP_USER_PASSWD\" ftp://localhost:21"]
      interval: 10s
      timeout: 5s
      retries: 10
    profiles: [ftp, file, all]

  ftps:
    image: ${FTPS_IMAGE:-chonjay21/ftps:latest}
    restart: unless-stopped
    environment:
      TZ: UTC
      APP_UID: 1000
      APP_GID: 1000
      APP_USER_NAME: onetl
      APP_USER_PASSWD: 123UsedForTestOnly@!
      USE_SSL: 'true'
      PASSV_MIN_PORT: 30020
      PASSV_MAX_PORT: 30030
    ports:
      - 2122:21
      - 30020-30030:30020-30030
    volumes:
      - ./docker/ftp/on_post_init.sh:/sources/ftps/eventscripts/on_post_init.sh
    networks:
      - onetl
    healthcheck:
      test: ["CMD-SHELL", "curl --silent --show-error --fail --ftp-ssl -k -u \"$$APP_USER_NAME:$$APP_USER_PASSWD\" ftp://localhost:21"]
      interval: 10s
      timeout: 5s
      retries: 10
    profiles: [ftps, file, all]

  samba:
    image: ${SAMBA_IMAGE:-elswork/samba}
    restart: unless-stopped
    environment:
      TZ: UTC
    ports:
      - "139:139"
      - "445:445"
    volumes:
      - ./docker/samba/custom_entrypoint.sh:/custom_entrypoint.sh
    entrypoint: ["/custom_entrypoint.sh"]
    networks:
      - onetl
    # healthcheck is defined in image
    profiles: [samba, file, all]

  s3:
    image: ${S3_IMAGE:-bitnami/minio:latest}
    restart: unless-stopped
    environment:
      TZ: UTC
      # MINIO_ACCESS_KEY and MINIO_SECRET_KEY are deprecated by new Minio version, but used by old version.
      # so we keeping both
      MINIO_ACCESS_KEY: onetl
      MINIO_ROOT_USER: onetl
      MINIO_SECRET_KEY: 123UsedForTestOnly@!
      MINIO_ROOT_PASSWORD: 123UsedForTestOnly@!
    ports:
      - 9010:9000
      - 9011:9001
    networks:
      - onetl
    healthcheck:
      test: ["CMD-SHELL", "curl --silent --show-error --fail http://localhost:9000/minio/health/live"]
      interval: 10s
      timeout: 5s
      retries: 10
    profiles: [s3, file_df, file, all]

  sftp:
    image: ${SFTP_IMAGE:-linuxserver/openssh-server}
    restart: unless-stopped
    environment:
      TZ: UTC
      PUID: 1000
      PGID: 1000
      USER_NAME: onetl
      PASSWORD_ACCESS: 'true'
      SUDO_ACCESS: 'true'
      USER_PASSWORD: 123UsedForTestOnly@!
    ports:
      - 2222:2222
    networks:
      - onetl
    # no sshpass in the image, so healthcheck is not available
    profiles: [sftp, file, all]

  webdav:
    image: ${WEBDAV_IMAGE:-chonjay21/webdav:latest}
    restart: unless-stopped
    environment:
      TZ: UTC
      APP_UID: 1000
      APP_GID: 1000
      APP_USER_NAME: onetl
      APP_USER_PASSWD: 123UsedForTestOnly@!
    ports:
      - 8000:80
    volumes:
    # Remove after https://github.com/chonjay21/docker-webdav/pull/3
      - ./docker/webdav/on_post_init.sh:/sources/webdav/eventscripts/on_post_init.sh
    networks:
      - onetl
    healthcheck:
      test: ["CMD-SHELL", "curl --silent --show-error --fail -u \"$$APP_USER_NAME:$$APP_USER_PASSWD\" -k http://localhost:80"]
      interval: 10s
      timeout: 5s
      retries: 10
    profiles: [webdav, file, all]

networks:
  onetl: