From 63643299f8b6fb54e561c12add05a536ca91b09d Mon Sep 17 00:00:00 2001 From: arthurgarmider-igz <69186266+arthurgarmider-igz@users.noreply.github.com> Date: Sun, 29 Nov 2020 20:06:17 +0200 Subject: [PATCH 1/4] Grafana with grafwiz --- .../grafana-grafwiz.ipynb | 266 ++++++++++++++++++ 1 file changed, 266 insertions(+) create mode 100644 data-ingestion-and-preparation/grafana-grafwiz.ipynb diff --git a/data-ingestion-and-preparation/grafana-grafwiz.ipynb b/data-ingestion-and-preparation/grafana-grafwiz.ipynb new file mode 100644 index 00000000..fb071f25 --- /dev/null +++ b/data-ingestion-and-preparation/grafana-grafwiz.ipynb @@ -0,0 +1,266 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Make sure grafwiz and v3io_frames are installed" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!pip install git+https://github.com/v3io/grafwiz --upgrade" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!pip install v3io_frames" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Add environment configurations and import the necessary libraries" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "grafana_url = 'http://grafana'\n", + "v3io_container = 'bigdata'\n", + "stocks_kv_table = 'stocks_kv_table'\n", + "stocks_tsdb_table = 'stocks_tsdb_table'" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from grafwiz import *\n", + "import os\n", + "import v3io_frames as v3f\n", + "import pandas as pd" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Generate data for the dataframe" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Date that will serve as an index for the time series and additional parameters" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import random\n", + "import datetime\n", + "def generate_date(rows):\n", + " datetimes = [datetime.datetime.today() + (random.random() * datetime.timedelta(days=1)) for i in range(rows)]\n", + " return datetimes\n", + "\n", + "def generate_volume_price(rows):\n", + " volume = [random.randint(0,10000) for i in range(rows)]\n", + " return volume\n", + "\n", + "rows = 345\n", + "time = generate_date(rows)\n", + "volume = generate_volume_price(rows)\n", + "price = generate_volume_price(rows)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Create a dataframe out of the generated index and add an index to the dataframe in order to make it accessible for tsdb" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "stocks_df = pd.DataFrame(\n", + " {'last_updated': time,\n", + " 'volume': volume,\n", + " 'price': price\n", + " })\n", + "stocks_df['symbol'] = 'GOOGL'\n", + "stocks_df = stocks_df.sort_values('last_updated')\n", + "\n", + "stocks_df" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "stocks_df = stocks_df.reset_index()\n", + "stocks_df = stocks_df.set_index(['last_updated', 'symbol'])\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Create a client for v3io_frames create table and persist the dataframe in tsdb" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "client = v3f.Client('framesd:8081',container='bigdata')\n", + "client.create(backend='tsdb', table=stocks_tsdb_table, rate='1/m', if_exists=1)\n", + "client.write(backend='tsdb', table=stocks_tsdb_table, dfs=stocks_df)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Update the kv table in order of rows arrival to simulate real time changing data" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "expr_template = \"symbol='{symbol}';price={price};volume={volume};last_updated='{last_updated}'\"\n", + "# update NoSQL table with stock data\n", + "for idx, record in stocks_df.iterrows():\n", + " stock = {'symbol': sym, 'price': record['price'], 'volume': record['volume'], 'last_updated': record['last_updated']}\n", + " expr = expr_template.format(**stock)\n", + " client.execute('kv', stocks_kv_table, 'update', args={'key': sym, 'expression': expr})" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Create a Datasource for grafana table and check that kv table exists and accessible" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Create datasources\n", + "DataSource(name='Iguazio').deploy(grafana_url, use_auth=True)\n", + "\n", + "# Verify the KV table can be shown\n", + "client = v3f.Client('framesd:8081', container=v3io_container)\n", + "client.execute(backend='kv', table=stocks_kv_table, command='infer')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Create a Dashboard define its datasource and then create a dashboard template" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Create grafana dashboard\n", + "dash = Dashboard(\"stocks\", start='now-7d', dataSource='Iguazio')\n", + "\n", + "# Add a symbol combo box (template) with data from the stocks table\n", + "dash.template(name=\"SYMBOL\", label=\"Symbol\", query=\"fields=symbol;table=stocks/stocks_kv;backend=kv;container=bigdata\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Create a table and 2 charts" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Create a table and log viewer in one row\n", + "tbl = Table('Current Stocks Value', span=12).source(table=stocks_kv_table,fields=['symbol','volume', 'price', 'last_updated'],container=v3io_container)\n", + "dash.row([tbl])\n", + "\n", + "# Create 2 charts on the second row\n", + "metrics_row = [Graph(metric).series(table=stocks_tsdb_table, fields=[metric], filter='symbol==\"$SYMBOL\"',container=v3io_container) for metric in ['price','volume']]\n", + "metrics_row.append(Graph('price').series(table=stocks_tsdb_table, fields=['price'], filter='symbol==\"$SYMBOL\"', container=v3io_container))\n", + "dash.row(metrics_row)\n", + "\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Deploy to Grafana\n", + "dash.deploy(grafana_url)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python [conda env:root] *", + "language": "python", + "name": "conda-root-py" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.6" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} From 18e0fe323501417096071544e86cbc5fa617d768 Mon Sep 17 00:00:00 2001 From: arthurgarmider-igz <69186266+arthurgarmider-igz@users.noreply.github.com> Date: Sun, 6 Dec 2020 23:22:25 +0200 Subject: [PATCH 2/4] Update grafana file --- .../grafana-grafwiz.ipynb | 29 +++++++------------ 1 file changed, 10 insertions(+), 19 deletions(-) diff --git a/data-ingestion-and-preparation/grafana-grafwiz.ipynb b/data-ingestion-and-preparation/grafana-grafwiz.ipynb index fb071f25..01646634 100644 --- a/data-ingestion-and-preparation/grafana-grafwiz.ipynb +++ b/data-ingestion-and-preparation/grafana-grafwiz.ipynb @@ -4,7 +4,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### Make sure grafwiz and v3io_frames are installed" + "### Make sure grafwiz is installed" ] }, { @@ -16,15 +16,6 @@ "!pip install git+https://github.com/v3io/grafwiz --upgrade" ] }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!pip install v3io_frames" - ] - }, { "cell_type": "markdown", "metadata": {}, @@ -39,9 +30,10 @@ "outputs": [], "source": [ "grafana_url = 'http://grafana'\n", - "v3io_container = 'bigdata'\n", + "v3io_container = 'users'\n", "stocks_kv_table = 'stocks_kv_table'\n", - "stocks_tsdb_table = 'stocks_tsdb_table'" + "stocks_tsdb_table = 'stocks_tsdb_table'\n", + "rows = 345" ] }, { @@ -51,7 +43,6 @@ "outputs": [], "source": [ "from grafwiz import *\n", - "import os\n", "import v3io_frames as v3f\n", "import pandas as pd" ] @@ -86,7 +77,7 @@ " volume = [random.randint(0,10000) for i in range(rows)]\n", " return volume\n", "\n", - "rows = 345\n", + "\n", "time = generate_date(rows)\n", "volume = generate_volume_price(rows)\n", "price = generate_volume_price(rows)" @@ -123,7 +114,7 @@ "outputs": [], "source": [ "stocks_df = stocks_df.reset_index()\n", - "stocks_df = stocks_df.set_index(['last_updated', 'symbol'])\n" + "stocks_df = stocks_df.set_index(['last_updated'])\n" ] }, { @@ -139,7 +130,7 @@ "metadata": {}, "outputs": [], "source": [ - "client = v3f.Client('framesd:8081',container='bigdata')\n", + "client = v3f.Client('framesd:8081',container=v3io_container)\n", "client.create(backend='tsdb', table=stocks_tsdb_table, rate='1/m', if_exists=1)\n", "client.write(backend='tsdb', table=stocks_tsdb_table, dfs=stocks_df)" ] @@ -160,7 +151,7 @@ "expr_template = \"symbol='{symbol}';price={price};volume={volume};last_updated='{last_updated}'\"\n", "# update NoSQL table with stock data\n", "for idx, record in stocks_df.iterrows():\n", - " stock = {'symbol': sym, 'price': record['price'], 'volume': record['volume'], 'last_updated': record['last_updated']}\n", + " stock = {'symbol': sym, 'price': record['price'], 'volume': record['volume'], 'last_updated': record.index[0]}\n", " expr = expr_template.format(**stock)\n", " client.execute('kv', stocks_kv_table, 'update', args={'key': sym, 'expression': expr})" ] @@ -200,10 +191,10 @@ "outputs": [], "source": [ "# Create grafana dashboard\n", - "dash = Dashboard(\"stocks\", start='now-7d', dataSource='Iguazio')\n", + "dash = Dashboard(\"stocks\", start='now-1d', dataSource='Iguazio')\n", "\n", "# Add a symbol combo box (template) with data from the stocks table\n", - "dash.template(name=\"SYMBOL\", label=\"Symbol\", query=\"fields=symbol;table=stocks/stocks_kv;backend=kv;container=bigdata\")" + "dash.template(name=\"SYMBOL\", label=\"Symbol\", query=f'fields=symbol;table=stocks/stocks_kv;backend=kv;container={v3io_container}')" ] }, { From 65888f97e893e1ae6b28649df1a3a533a3731dcd Mon Sep 17 00:00:00 2001 From: arthurgarmider-igz <69186266+arthurgarmider-igz@users.noreply.github.com> Date: Sun, 13 Dec 2020 20:55:13 +0200 Subject: [PATCH 3/4] Modification to a local user --- .../grafana-grafwiz.ipynb | 50 ++++++++++--------- 1 file changed, 26 insertions(+), 24 deletions(-) diff --git a/data-ingestion-and-preparation/grafana-grafwiz.ipynb b/data-ingestion-and-preparation/grafana-grafwiz.ipynb index 01646634..80e09699 100644 --- a/data-ingestion-and-preparation/grafana-grafwiz.ipynb +++ b/data-ingestion-and-preparation/grafana-grafwiz.ipynb @@ -20,7 +20,9 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "### Add environment configurations and import the necessary libraries" + "### Add environment configurations and import the necessary libraries\n", + "* Please make sure you have a Grafana service, in order to create one, please go to Iguazio dashboard -> Service, Create new service\n", + "* To retrieve the url of Grafana service please use \"API\" tab, in the services dashboard" ] }, { @@ -29,11 +31,14 @@ "metadata": {}, "outputs": [], "source": [ - "grafana_url = 'http://grafana'\n", + "import os\n", + "\n", + "grafana_url = 'http://grafana' \n", "v3io_container = 'users'\n", - "stocks_kv_table = 'stocks_kv_table'\n", - "stocks_tsdb_table = 'stocks_tsdb_table'\n", - "rows = 345" + "stocks_kv_table = os.path.join(os.getenv(\"V3IO_USERNAME\"),'stocks_kv_table')\n", + "stocks_tsdb_table = os.path.join(os.getenv(\"V3IO_USERNAME\"),'stocks_tsdb_table')\n", + "sym = 'XYZ'\n", + "rows = 3450\n" ] }, { @@ -70,17 +75,16 @@ "import random\n", "import datetime\n", "def generate_date(rows):\n", - " datetimes = [datetime.datetime.today() + (random.random() * datetime.timedelta(days=1)) for i in range(rows)]\n", + " datetimes = [datetime.datetime.today() - (random.random() * datetime.timedelta(minutes=15)) for i in range(rows)]\n", " return datetimes\n", "\n", "def generate_volume_price(rows):\n", " volume = [random.randint(0,10000) for i in range(rows)]\n", " return volume\n", "\n", - "\n", "time = generate_date(rows)\n", "volume = generate_volume_price(rows)\n", - "price = generate_volume_price(rows)" + "price = generate_volume_price(rows)\n" ] }, { @@ -101,7 +105,7 @@ " 'volume': volume,\n", " 'price': price\n", " })\n", - "stocks_df['symbol'] = 'GOOGL'\n", + "stocks_df['symbol'] = sym\n", "stocks_df = stocks_df.sort_values('last_updated')\n", "\n", "stocks_df" @@ -113,8 +117,9 @@ "metadata": {}, "outputs": [], "source": [ - "stocks_df = stocks_df.reset_index()\n", - "stocks_df = stocks_df.set_index(['last_updated'])\n" + "stocks_df_tsdb = stocks_df\n", + "stocks_df_tsdb = stocks_df.reset_index()\n", + "stocks_df_tsdb = stocks_df.set_index(['last_updated'])\n" ] }, { @@ -132,7 +137,7 @@ "source": [ "client = v3f.Client('framesd:8081',container=v3io_container)\n", "client.create(backend='tsdb', table=stocks_tsdb_table, rate='1/m', if_exists=1)\n", - "client.write(backend='tsdb', table=stocks_tsdb_table, dfs=stocks_df)" + "client.write(backend='tsdb', table=stocks_tsdb_table, dfs=stocks_df_tsdb)" ] }, { @@ -148,10 +153,10 @@ "metadata": {}, "outputs": [], "source": [ - "expr_template = \"symbol='{symbol}';price={price};volume={volume};last_updated='{last_updated}'\"\n", + "expr_template = \"symbol='{symbol}';price='{price}';volume='{volume}';last_updated='{last_updated}'\"\n", "# update NoSQL table with stock data\n", "for idx, record in stocks_df.iterrows():\n", - " stock = {'symbol': sym, 'price': record['price'], 'volume': record['volume'], 'last_updated': record.index[0]}\n", + " stock = {'symbol': sym, 'price': record['price'], 'volume': record['volume'], 'last_updated': record['last_updated']}\n", " expr = expr_template.format(**stock)\n", " client.execute('kv', stocks_kv_table, 'update', args={'key': sym, 'expression': expr})" ] @@ -181,7 +186,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "#### Create a Dashboard define its datasource and then create a dashboard template" + "#### Create a Dashboard define its datasource" ] }, { @@ -191,17 +196,15 @@ "outputs": [], "source": [ "# Create grafana dashboard\n", - "dash = Dashboard(\"stocks\", start='now-1d', dataSource='Iguazio')\n", - "\n", - "# Add a symbol combo box (template) with data from the stocks table\n", - "dash.template(name=\"SYMBOL\", label=\"Symbol\", query=f'fields=symbol;table=stocks/stocks_kv;backend=kv;container={v3io_container}')" + "dash = Dashboard(\"stocks\", start='now', dataSource='Iguazio')\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "#### Create a table and 2 charts" + "#### Create a table and 2 graphs\n", + "Please note that you have to wait for couple of minutes to see the graphs being updated" ] }, { @@ -214,11 +217,10 @@ "tbl = Table('Current Stocks Value', span=12).source(table=stocks_kv_table,fields=['symbol','volume', 'price', 'last_updated'],container=v3io_container)\n", "dash.row([tbl])\n", "\n", - "# Create 2 charts on the second row\n", - "metrics_row = [Graph(metric).series(table=stocks_tsdb_table, fields=[metric], filter='symbol==\"$SYMBOL\"',container=v3io_container) for metric in ['price','volume']]\n", - "metrics_row.append(Graph('price').series(table=stocks_tsdb_table, fields=['price'], filter='symbol==\"$SYMBOL\"', container=v3io_container))\n", - "dash.row(metrics_row)\n", + "# Metrics row will\n", + "metrics_row = [Graph(metric).series(table=stocks_tsdb_table, fields=[metric], container=v3io_container) for metric in ['price','volume']]\n", "\n", + "dash.row(metrics_row)\n", "\n" ] }, From 02eb4d4850ab9a94de7181f2e5888f6a444d1a7c Mon Sep 17 00:00:00 2001 From: arthurgarmider-igz <69186266+arthurgarmider-igz@users.noreply.github.com> Date: Sun, 20 Dec 2020 18:18:05 +0200 Subject: [PATCH 4/4] modify to work numpy to evenly distribute values --- .../grafana-grafwiz.ipynb | 33 +++++++++++-------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/data-ingestion-and-preparation/grafana-grafwiz.ipynb b/data-ingestion-and-preparation/grafana-grafwiz.ipynb index 80e09699..8cf4ebf4 100644 --- a/data-ingestion-and-preparation/grafana-grafwiz.ipynb +++ b/data-ingestion-and-preparation/grafana-grafwiz.ipynb @@ -38,7 +38,7 @@ "stocks_kv_table = os.path.join(os.getenv(\"V3IO_USERNAME\"),'stocks_kv_table')\n", "stocks_tsdb_table = os.path.join(os.getenv(\"V3IO_USERNAME\"),'stocks_tsdb_table')\n", "sym = 'XYZ'\n", - "rows = 3450\n" + "rows = 3450" ] }, { @@ -74,17 +74,16 @@ "source": [ "import random\n", "import datetime\n", + "import numpy as np\n", + "\n", "def generate_date(rows):\n", + " \n", " datetimes = [datetime.datetime.today() - (random.random() * datetime.timedelta(minutes=15)) for i in range(rows)]\n", " return datetimes\n", "\n", - "def generate_volume_price(rows):\n", - " volume = [random.randint(0,10000) for i in range(rows)]\n", - " return volume\n", - "\n", - "time = generate_date(rows)\n", - "volume = generate_volume_price(rows)\n", - "price = generate_volume_price(rows)\n" + "time = sorted(generate_date(rows))\n", + "volume = np.random.randint(low=100, high=10000, size=rows)\n", + "price = np.cumsum([0.0001] * rows + np.random.random(rows))\n" ] }, { @@ -107,7 +106,6 @@ " })\n", "stocks_df['symbol'] = sym\n", "stocks_df = stocks_df.sort_values('last_updated')\n", - "\n", "stocks_df" ] }, @@ -119,7 +117,7 @@ "source": [ "stocks_df_tsdb = stocks_df\n", "stocks_df_tsdb = stocks_df.reset_index()\n", - "stocks_df_tsdb = stocks_df.set_index(['last_updated'])\n" + "stocks_df_tsdb = stocks_df.set_index(['last_updated'])" ] }, { @@ -196,7 +194,7 @@ "outputs": [], "source": [ "# Create grafana dashboard\n", - "dash = Dashboard(\"stocks\", start='now', dataSource='Iguazio')\n" + "dash = Dashboard(\"stocks\", start='now-15m', dataSource='Iguazio', end='now')\n" ] }, { @@ -220,7 +218,7 @@ "# Metrics row will\n", "metrics_row = [Graph(metric).series(table=stocks_tsdb_table, fields=[metric], container=v3io_container) for metric in ['price','volume']]\n", "\n", - "dash.row(metrics_row)\n", + "dashboard = dash.row(metrics_row)\n", "\n" ] }, @@ -233,13 +231,20 @@ "# Deploy to Grafana\n", "dash.deploy(grafana_url)" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { "kernelspec": { - "display_name": "Python [conda env:root] *", + "display_name": "Python 3", "language": "python", - "name": "conda-root-py" + "name": "python3" }, "language_info": { "codemirror_mode": {