From e9cd4adf84b1633c518edbd4acf80b56a52e1811 Mon Sep 17 00:00:00 2001 From: Ivan Ermilov Date: Tue, 26 Dec 2017 11:29:33 +0100 Subject: [PATCH 1/5] bump spark version --- zeppelin/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zeppelin/Dockerfile b/zeppelin/Dockerfile index 8d61b58..53a01e4 100644 --- a/zeppelin/Dockerfile +++ b/zeppelin/Dockerfile @@ -1,7 +1,7 @@ -FROM bde2020/spark-base:2.1.0-hadoop2.8-hive-java8 +FROM bde2020/spark-base:2.2.0-hadoop2.8-hive-java8 MAINTAINER Ivan Ermilov -ENV APACHE_SPARK_VERSION 2.1.0 +ENV APACHE_SPARK_VERSION 2.2.0 ENV APACHE_HADOOP_VERSION 2.8.0 ENV ZEPPELIN_VERSION 0.7.2 From 745098a69767cf156275c5d999d15d09eddfb9e7 Mon Sep 17 00:00:00 2001 From: Ivan Ermilov Date: Tue, 26 Dec 2017 12:51:35 +0100 Subject: [PATCH 2/5] bump zeppelin version --- zeppelin/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zeppelin/Dockerfile b/zeppelin/Dockerfile index 53a01e4..e98d633 100644 --- a/zeppelin/Dockerfile +++ b/zeppelin/Dockerfile @@ -7,7 +7,7 @@ ENV ZEPPELIN_VERSION 0.7.2 RUN apt-get update && apt-get install wget RUN set -x \ - && curl -fSL "http://www-eu.apache.org/dist/zeppelin/zeppelin-0.7.2/zeppelin-0.7.2-bin-all.tgz" -o /tmp/zeppelin.tgz \ + && curl -fSL "http://www-eu.apache.org/dist/zeppelin/zeppelin-0.7.3/zeppelin-0.7.3-bin-all.tgz" -o /tmp/zeppelin.tgz \ && tar -xzvf /tmp/zeppelin.tgz -C /opt/ \ && mv /opt/zeppelin-* /opt/zeppelin \ && rm /tmp/zeppelin.tgz From e447d609de1cdec3c9155d0066379b7911d400ed Mon Sep 17 00:00:00 2001 From: Gezim Sejdiu Date: Tue, 11 Dec 2018 16:02:10 +0100 Subject: [PATCH 3/5] Add support for Zeppelin 0.8.0-hadoop-2.8.0-spark-2.4.0 --- Makefile | 4 ++-- zeppelin/Dockerfile | 11 ++++++----- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index 36f762c..ace9fd5 100644 --- a/Makefile +++ b/Makefile @@ -12,8 +12,8 @@ bash: run: docker build -t zeppelin ./zeppelin/. - docker run -it --rm --net spark-net -p 80:8080 -v $(shell pwd)/notebook:/opt/zeppelin/notebook -v $(shell pwd)/zeppelin-0.7.2-bin-all:/opt/zeppelin zeppelin /bin/bash + docker run -it --rm --net spark-net -p 80:8080 -v $(shell pwd)/notebook:/opt/zeppelin/notebook -v $(shell pwd)/zeppelin-0.8.0-bin-all:/opt/zeppelin zeppelin /bin/bash #docker run -it --rm --net spark-net -p 80:8080 -v $(shell pwd)/notebook:/opt/zeppelin/notebook zeppelin /opt/zeppelin/bin/zeppelin.sh build: - docker build -t earthquakesan/zeppelin:0.7.2 ./zeppelin/. + docker build -t bde2020/zeppelin:0.8.0 ./zeppelin/. diff --git a/zeppelin/Dockerfile b/zeppelin/Dockerfile index e98d633..71c18d6 100644 --- a/zeppelin/Dockerfile +++ b/zeppelin/Dockerfile @@ -1,18 +1,19 @@ -FROM bde2020/spark-base:2.2.0-hadoop2.8-hive-java8 +FROM bde2020/spark-base:2.4.0-hadoop2.8 MAINTAINER Ivan Ermilov +MAINTAINER Gezim Sejdiu -ENV APACHE_SPARK_VERSION 2.2.0 +ENV APACHE_SPARK_VERSION 2.4.0 ENV APACHE_HADOOP_VERSION 2.8.0 -ENV ZEPPELIN_VERSION 0.7.2 +ENV ZEPPELIN_VERSION 0.8.0 RUN apt-get update && apt-get install wget RUN set -x \ - && curl -fSL "http://www-eu.apache.org/dist/zeppelin/zeppelin-0.7.3/zeppelin-0.7.3-bin-all.tgz" -o /tmp/zeppelin.tgz \ + && curl -fSL "http://www-eu.apache.org/dist/zeppelin/zeppelin-${ZEPPELIN_VERSION}/zeppelin-${ZEPPELIN_VERSION}-bin-all.tgz" -o /tmp/zeppelin.tgz \ && tar -xzvf /tmp/zeppelin.tgz -C /opt/ \ && mv /opt/zeppelin-* /opt/zeppelin \ && rm /tmp/zeppelin.tgz -ENV SPARK_SUBMIT_OPTIONS "--jars /opt/zeppelin/sansa-examples-spark-2016-12.jar" +ENV SPARK_SUBMIT_OPTIONS "--jars /opt/zeppelin/sansa-examples-spark-2018-06.jar" WORKDIR /opt/zeppelin From abb3285af6fc6f97bdf39d4030d3aa94585059e5 Mon Sep 17 00:00:00 2001 From: Gezim Sejdiu Date: Tue, 11 Dec 2018 16:02:35 +0100 Subject: [PATCH 4/5] Upgrade Spark docker to the latest version (v2.4.0) --- docker-compose.yml | 33 +++++++++------------------------ 1 file changed, 9 insertions(+), 24 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 447d604..b2f2583 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,7 +11,7 @@ services: - CORE_CONF_fs_defaultFS=hdfs://namenode:8020 healthcheck: interval: 5s - retries: 100 + retries: 100 networks: - spark-net datanode: @@ -22,15 +22,11 @@ services: environment: - CORE_CONF_fs_defaultFS=hdfs://namenode:8020 depends_on: - namenode: - condition: service_healthy - healthcheck: - interval: 5s - retries: 100 + - namenode networks: - spark-net spark-master: - image: bde2020/spark-master:2.1.0-hadoop2.8-hive-java8 + image: bde2020/spark-master:2.4.0-hadoop2.8 container_name: spark-master ports: - "8080:8080" @@ -38,27 +34,18 @@ services: environment: - CORE_CONF_fs_defaultFS=hdfs://namenode:8020 depends_on: - namenode: - condition: service_healthy - datanode: - condition: service_healthy - healthcheck: - interval: 5s - retries: 100 + - namenode + - datanode networks: - spark-net spark-worker: - image: bde2020/spark-worker:2.1.0-hadoop2.8-hive-java8 + image: bde2020/spark-worker:2.4.0-hadoop2.8 environment: - "SPARK_MASTER=spark://spark-master:7077" environment: - CORE_CONF_fs_defaultFS=hdfs://namenode:8020 depends_on: - spark-master: - condition: service_healthy - healthcheck: - interval: 5s - retries: 100 + - spark-master networks: - spark-net zeppelin: @@ -73,10 +60,8 @@ services: MASTER: "spark://spark-master:7077" #SPARK_SUBMIT_OPTIONS: "--jars /opt/sansa-examples/jars/sansa-examples-spark-2016-12.jar" depends_on: - spark-master: - condition: service_healthy - namenode: - condition: service_healthy + - spark-master + - namenode networks: - spark-net From 7608192300984c2a7e4eb781188e1863cbbe488c Mon Sep 17 00:00:00 2001 From: Gezim Sejdiu Date: Mon, 17 Dec 2018 10:35:33 +0100 Subject: [PATCH 5/5] Add support for Zeppelin 0.8.0-hadoop-2.8.0-spark-2.3.1 --- Makefile | 2 +- docker-compose.yml | 4 ++-- zeppelin/Dockerfile | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index ace9fd5..dc44979 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ up: docker network create spark-net docker-compose build - docker-compose up + docker-compose up -d down: docker-compose down diff --git a/docker-compose.yml b/docker-compose.yml index b2f2583..51543ca 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -26,7 +26,7 @@ services: networks: - spark-net spark-master: - image: bde2020/spark-master:2.4.0-hadoop2.8 + image: bde2020/spark-master:2.3.1-hadoop2.8 container_name: spark-master ports: - "8080:8080" @@ -39,7 +39,7 @@ services: networks: - spark-net spark-worker: - image: bde2020/spark-worker:2.4.0-hadoop2.8 + image: bde2020/spark-worker:2.3.1-hadoop2.8 environment: - "SPARK_MASTER=spark://spark-master:7077" environment: diff --git a/zeppelin/Dockerfile b/zeppelin/Dockerfile index 71c18d6..978a995 100644 --- a/zeppelin/Dockerfile +++ b/zeppelin/Dockerfile @@ -1,8 +1,8 @@ -FROM bde2020/spark-base:2.4.0-hadoop2.8 +FROM bde2020/spark-base:2.3.1-hadoop2.8 MAINTAINER Ivan Ermilov MAINTAINER Gezim Sejdiu -ENV APACHE_SPARK_VERSION 2.4.0 +ENV APACHE_SPARK_VERSION 2.3.1 ENV APACHE_HADOOP_VERSION 2.8.0 ENV ZEPPELIN_VERSION 0.8.0