diff --git a/spark/.gitignore b/spark/.gitignore index 929d4cc..8b75448 100644 --- a/spark/.gitignore +++ b/spark/.gitignore @@ -1,5 +1,4 @@ .ipynb_checkpoints/ -live/ -live.html +live/live.html +live/live_mod.py _work.py - diff --git a/spark/live.ipynb b/spark/live.ipynb new file mode 100644 index 0000000..b6699bf --- /dev/null +++ b/spark/live.ipynb @@ -0,0 +1,42 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "hello world!\n" + ] + } + ], + "source": [ + "print(\"hello world!\")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.2" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/spark/live.py b/spark/live.py index b78ffa3..bab7041 100644 --- a/spark/live.py +++ b/spark/live.py @@ -3,4 +3,3 @@ spark = SparkSession.builder.master("local").appName("SQL").getOrCreate() df = spark.read.load('data/containers_tiny.parquet') df.select("ship_imo", "container_id", "net_weight").show() - diff --git a/spark/live.sh b/spark/live.sh deleted file mode 100755 index 97f3f0d..0000000 --- a/spark/live.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash -set -euo pipefail - -fswatch live.py | while read -r fpath; do \ - echo -e "\033[0;36mRELOAD\033[0m $fpath $(date +"%H%m%S")" - pygmentize -f html -O full,linenos=1 -o live.html live.py - scp live.html root@live.luisbelloch.es:/var/www/html/index.html -done - diff --git a/spark/live/live.sh b/spark/live/live.sh new file mode 100755 index 0000000..2eb7d7b --- /dev/null +++ b/spark/live/live.sh @@ -0,0 +1,13 @@ +#!/bin/bash +set -euo pipefail + +fswatch ../live.py | while read -r fpath; do \ + echo -e "\033[0;36mRELOAD\033[0m $fpath $(date +"%H%M%S")" + echo -e "# $(date +"%H:%M:%S")\n" | cat - ../live.py > live_mod.py + sed -e '/-python">/r./live_mod.py' live_template.html > live.html + gsutil cp live.html gs://bigdata.luisbelloch.es/live.html + + # echo -e "# $(date +"%H:%M:%S")\n" | cat - live.py | pygmentize -f html -O full,linenos=1 -o live.html + # scp live.html root@live.luisbelloch.es:/var/www/html/index.html +done + diff --git a/spark/live/live_jupyter.sh b/spark/live/live_jupyter.sh new file mode 100755 index 0000000..cc04b8b --- /dev/null +++ b/spark/live/live_jupyter.sh @@ -0,0 +1,10 @@ +#!/bin/bash +set -eou pipefail + +fswatch ../live.ipynb | while read -r fpath; do \ + echo -e "\033[0;36mRELOAD\033[0m $fpath $(date +"%H%m%S")" + jupyter nbconvert ../live.ipynb --to html --output-dir="$(pwd)" + gsutil cp live.html gs://bigdata.luisbelloch.es/live.html + # scp live/live.html root@live.luisbelloch.es:/var/www/html/index.html +done + diff --git a/spark/live/live_template.html b/spark/live/live_template.html new file mode 100644 index 0000000..038e96a --- /dev/null +++ b/spark/live/live_template.html @@ -0,0 +1,34 @@ + + + + + + + #bidataupv - live + + + + + +
+ 

+
+
+ + + + diff --git a/spark/reload.sh b/spark/reload.sh new file mode 100755 index 0000000..b3ddb73 --- /dev/null +++ b/spark/reload.sh @@ -0,0 +1,9 @@ +#!/bin/bash +set -euo pipefail + +fswatch live.py | while read -r fpath; do \ + clear + echo -e "\033[0;36mRELOAD\033[0m $fpath $(date +"%H%M%S")" + spark-submit live.py +done +