Modified scripts to produce live preview of code being run

luisbelloch · Feb 7, 2019 · 1e37851 · 1e37851
1 parent 9a75571
commit 1e37851
Show file tree

Hide file tree

Showing 8 changed files with 110 additions and 13 deletions.
diff --git a/spark/.gitignore b/spark/.gitignore
@@ -1,5 +1,4 @@
 .ipynb_checkpoints/
-live/
-live.html
+live/live.html
+live/live_mod.py
 _work.py
-
diff --git a/spark/live.ipynb b/spark/live.ipynb
@@ -0,0 +1,42 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "hello world!\n"
+     ]
+    }
+   ],
+   "source": [
+    "print(\"hello world!\")"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.7.2"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
diff --git a/spark/live.py b/spark/live.py
@@ -3,4 +3,3 @@
 spark = SparkSession.builder.master("local").appName("SQL").getOrCreate()
 df = spark.read.load('data/containers_tiny.parquet')
 df.select("ship_imo", "container_id", "net_weight").show()
-
diff --git a/spark/live.sh b/spark/live.sh
diff --git a/spark/live/live.sh b/spark/live/live.sh
@@ -0,0 +1,13 @@
+#!/bin/bash
+set -euo pipefail
+
+fswatch ../live.py | while read -r fpath; do \
+    echo -e "\033[0;36mRELOAD\033[0m $fpath $(date +"%H%M%S")"
+    echo -e "# $(date +"%H:%M:%S")\n" | cat - ../live.py > live_mod.py
+    sed -e '/-python">/r./live_mod.py' live_template.html > live.html
+    gsutil cp live.html gs://bigdata.luisbelloch.es/live.html
+
+    # echo -e "# $(date +"%H:%M:%S")\n" | cat - live.py | pygmentize -f html -O full,linenos=1 -o live.html
+    # scp live.html [email protected]:/var/www/html/index.html
+done
+
diff --git a/spark/live/live_jupyter.sh b/spark/live/live_jupyter.sh
@@ -0,0 +1,10 @@
+#!/bin/bash
+set -eou pipefail
+
+fswatch ../live.ipynb | while read -r fpath; do \
+  echo -e "\033[0;36mRELOAD\033[0m $fpath $(date +"%H%m%S")"
+  jupyter nbconvert ../live.ipynb --to html --output-dir="$(pwd)"
+  gsutil cp live.html gs://bigdata.luisbelloch.es/live.html
+  # scp live/live.html [email protected]:/var/www/html/index.html
+done
+
diff --git a/spark/live/live_template.html b/spark/live/live_template.html
@@ -0,0 +1,34 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <meta http-equiv="X-UA-Compatible" content="ie=edge" />
+    <title>#bidataupv - live</title>
+    <link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Fira+Mono:400,500,700|Roboto:300,400,500,700,900" />
+    <link rel="stylesheet" href="http://cdnjs.cloudflare.com/ajax/libs/highlight.js/9.14.2/styles/dracula.min.css" />
+    <style>
+      body {
+        background: #282a36;
+      }
+      pre {
+        font-size: 0.875rem;
+        line-height: 1.4;
+        overflow-x: auto;
+      }
+      code {
+        font-family: "Fira Mono", Menlo, Consolas, "Courier New", Courier, monospace;
+      }
+    </style>
+  </head>
+  <body>
+    <div>
+      <pre><code class="language-python">
+      </code></pre>
+    </div>
+    <script src="http://cdnjs.cloudflare.com/ajax/libs/highlight.js/9.14.2/highlight.min.js"></script>
+    <script>
+      hljs.initHighlightingOnLoad();
+    </script>
+  </body>
+</html>
diff --git a/spark/reload.sh b/spark/reload.sh
@@ -0,0 +1,9 @@
+#!/bin/bash
+set -euo pipefail
+
+fswatch live.py | while read -r fpath; do \
+    clear
+    echo -e "\033[0;36mRELOAD\033[0m $fpath $(date +"%H%M%S")"
+    spark-submit live.py
+done
+
Original file line number	Diff line number	Diff line change
Expand Up		@@ -3,4 +3,3 @@
		spark = SparkSession.builder.master("local").appName("SQL").getOrCreate()
		df = spark.read.load('data/containers_tiny.parquet')
		df.select("ship_imo", "container_id", "net_weight").show()