-
Notifications
You must be signed in to change notification settings - Fork 42
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #304 from v3io/development
Development
- Loading branch information
Showing
3 changed files
with
290 additions
and
2 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,264 @@ | ||
| { | ||
| "cells": [ | ||
| { | ||
| "cell_type": "markdown", | ||
| "metadata": {}, | ||
| "source": [ | ||
| "### Make sure grafwiz is installed" | ||
| ] | ||
| }, | ||
| { | ||
| "cell_type": "code", | ||
| "execution_count": null, | ||
| "metadata": {}, | ||
| "outputs": [], | ||
| "source": [ | ||
| "!pip install git+https://github.com/v3io/grafwiz --upgrade" | ||
| ] | ||
| }, | ||
| { | ||
| "cell_type": "markdown", | ||
| "metadata": {}, | ||
| "source": [ | ||
| "### Add environment configurations and import the necessary libraries\n", | ||
| "* Please make sure you have a Grafana service, in order to create one, please go to Iguazio dashboard -> Service, Create new service\n", | ||
| "* To retrieve the url of Grafana service please use \"API\" tab, in the services dashboard" | ||
| ] | ||
| }, | ||
| { | ||
| "cell_type": "code", | ||
| "execution_count": null, | ||
| "metadata": {}, | ||
| "outputs": [], | ||
| "source": [ | ||
| "import os\n", | ||
| "\n", | ||
| "grafana_url = 'http://grafana' \n", | ||
| "v3io_container = 'users'\n", | ||
| "stocks_kv_table = os.path.join(os.getenv(\"V3IO_USERNAME\"),'stocks_kv_table')\n", | ||
| "stocks_tsdb_table = os.path.join(os.getenv(\"V3IO_USERNAME\"),'stocks_tsdb_table')\n", | ||
| "sym = 'XYZ'\n", | ||
| "rows = 3450" | ||
| ] | ||
| }, | ||
| { | ||
| "cell_type": "code", | ||
| "execution_count": null, | ||
| "metadata": {}, | ||
| "outputs": [], | ||
| "source": [ | ||
| "from grafwiz import *\n", | ||
| "import v3io_frames as v3f\n", | ||
| "import pandas as pd" | ||
| ] | ||
| }, | ||
| { | ||
| "cell_type": "markdown", | ||
| "metadata": {}, | ||
| "source": [ | ||
| "### Generate data for the dataframe" | ||
| ] | ||
| }, | ||
| { | ||
| "cell_type": "markdown", | ||
| "metadata": {}, | ||
| "source": [ | ||
| "#### Date that will serve as an index for the time series and additional parameters" | ||
| ] | ||
| }, | ||
| { | ||
| "cell_type": "code", | ||
| "execution_count": null, | ||
| "metadata": {}, | ||
| "outputs": [], | ||
| "source": [ | ||
| "import random\n", | ||
| "import datetime\n", | ||
| "import numpy as np\n", | ||
| "\n", | ||
| "def generate_date(rows):\n", | ||
| " \n", | ||
| " datetimes = [datetime.datetime.today() - (random.random() * datetime.timedelta(minutes=15)) for i in range(rows)]\n", | ||
| " return datetimes\n", | ||
| "\n", | ||
| "time = sorted(generate_date(rows))\n", | ||
| "volume = np.random.randint(low=100, high=10000, size=rows)\n", | ||
| "price = np.cumsum([0.0001] * rows + np.random.random(rows))\n" | ||
| ] | ||
| }, | ||
| { | ||
| "cell_type": "markdown", | ||
| "metadata": {}, | ||
| "source": [ | ||
| "#### Create a dataframe out of the generated index and add an index to the dataframe in order to make it accessible for tsdb" | ||
| ] | ||
| }, | ||
| { | ||
| "cell_type": "code", | ||
| "execution_count": null, | ||
| "metadata": {}, | ||
| "outputs": [], | ||
| "source": [ | ||
| "stocks_df = pd.DataFrame(\n", | ||
| " {'last_updated': time,\n", | ||
| " 'volume': volume,\n", | ||
| " 'price': price\n", | ||
| " })\n", | ||
| "stocks_df['symbol'] = sym\n", | ||
| "stocks_df = stocks_df.sort_values('last_updated')\n", | ||
| "stocks_df" | ||
| ] | ||
| }, | ||
| { | ||
| "cell_type": "code", | ||
| "execution_count": null, | ||
| "metadata": {}, | ||
| "outputs": [], | ||
| "source": [ | ||
| "stocks_df_tsdb = stocks_df\n", | ||
| "stocks_df_tsdb = stocks_df.reset_index()\n", | ||
| "stocks_df_tsdb = stocks_df.set_index(['last_updated'])" | ||
| ] | ||
| }, | ||
| { | ||
| "cell_type": "markdown", | ||
| "metadata": {}, | ||
| "source": [ | ||
| "#### Create a client for v3io_frames create table and persist the dataframe in tsdb" | ||
| ] | ||
| }, | ||
| { | ||
| "cell_type": "code", | ||
| "execution_count": null, | ||
| "metadata": {}, | ||
| "outputs": [], | ||
| "source": [ | ||
| "client = v3f.Client('framesd:8081',container=v3io_container)\n", | ||
| "client.create(backend='tsdb', table=stocks_tsdb_table, rate='1/m', if_exists=1)\n", | ||
| "client.write(backend='tsdb', table=stocks_tsdb_table, dfs=stocks_df_tsdb)" | ||
| ] | ||
| }, | ||
| { | ||
| "cell_type": "markdown", | ||
| "metadata": {}, | ||
| "source": [ | ||
| "#### Update the kv table in order of rows arrival to simulate real time changing data" | ||
| ] | ||
| }, | ||
| { | ||
| "cell_type": "code", | ||
| "execution_count": null, | ||
| "metadata": {}, | ||
| "outputs": [], | ||
| "source": [ | ||
| "expr_template = \"symbol='{symbol}';price='{price}';volume='{volume}';last_updated='{last_updated}'\"\n", | ||
| "# update NoSQL table with stock data\n", | ||
| "for idx, record in stocks_df.iterrows():\n", | ||
| " stock = {'symbol': sym, 'price': record['price'], 'volume': record['volume'], 'last_updated': record['last_updated']}\n", | ||
| " expr = expr_template.format(**stock)\n", | ||
| " client.execute('kv', stocks_kv_table, 'update', args={'key': sym, 'expression': expr})" | ||
| ] | ||
| }, | ||
| { | ||
| "cell_type": "markdown", | ||
| "metadata": {}, | ||
| "source": [ | ||
| "#### Create a Datasource for grafana table and check that kv table exists and accessible" | ||
| ] | ||
| }, | ||
| { | ||
| "cell_type": "code", | ||
| "execution_count": null, | ||
| "metadata": {}, | ||
| "outputs": [], | ||
| "source": [ | ||
| "# Create datasources\n", | ||
| "DataSource(name='Iguazio').deploy(grafana_url, use_auth=True)\n", | ||
| "\n", | ||
| "# Verify the KV table can be shown\n", | ||
| "client = v3f.Client('framesd:8081', container=v3io_container)\n", | ||
| "client.execute(backend='kv', table=stocks_kv_table, command='infer')" | ||
| ] | ||
| }, | ||
| { | ||
| "cell_type": "markdown", | ||
| "metadata": {}, | ||
| "source": [ | ||
| "#### Create a Dashboard define its datasource" | ||
| ] | ||
| }, | ||
| { | ||
| "cell_type": "code", | ||
| "execution_count": null, | ||
| "metadata": {}, | ||
| "outputs": [], | ||
| "source": [ | ||
| "# Create grafana dashboard\n", | ||
| "dash = Dashboard(\"stocks\", start='now-15m', dataSource='Iguazio', end='now')\n" | ||
| ] | ||
| }, | ||
| { | ||
| "cell_type": "markdown", | ||
| "metadata": {}, | ||
| "source": [ | ||
| "#### Create a table and 2 graphs\n", | ||
| "Please note that you have to wait for couple of minutes to see the graphs being updated" | ||
| ] | ||
| }, | ||
| { | ||
| "cell_type": "code", | ||
| "execution_count": null, | ||
| "metadata": {}, | ||
| "outputs": [], | ||
| "source": [ | ||
| "# Create a table and log viewer in one row\n", | ||
| "tbl = Table('Current Stocks Value', span=12).source(table=stocks_kv_table,fields=['symbol','volume', 'price', 'last_updated'],container=v3io_container)\n", | ||
| "dash.row([tbl])\n", | ||
| "\n", | ||
| "# Metrics row will\n", | ||
| "metrics_row = [Graph(metric).series(table=stocks_tsdb_table, fields=[metric], container=v3io_container) for metric in ['price','volume']]\n", | ||
| "\n", | ||
| "dashboard = dash.row(metrics_row)\n", | ||
| "\n" | ||
| ] | ||
| }, | ||
| { | ||
| "cell_type": "code", | ||
| "execution_count": null, | ||
| "metadata": {}, | ||
| "outputs": [], | ||
| "source": [ | ||
| "# Deploy to Grafana\n", | ||
| "dash.deploy(grafana_url)" | ||
| ] | ||
| }, | ||
| { | ||
| "cell_type": "code", | ||
| "execution_count": null, | ||
| "metadata": {}, | ||
| "outputs": [], | ||
| "source": [] | ||
| } | ||
| ], | ||
| "metadata": { | ||
| "kernelspec": { | ||
| "display_name": "Python 3", | ||
| "language": "python", | ||
| "name": "python3" | ||
| }, | ||
| "language_info": { | ||
| "codemirror_mode": { | ||
| "name": "ipython", | ||
| "version": 3 | ||
| }, | ||
| "file_extension": ".py", | ||
| "mimetype": "text/x-python", | ||
| "name": "python", | ||
| "nbconvert_exporter": "python", | ||
| "pygments_lexer": "ipython3", | ||
| "version": "3.7.6" | ||
| } | ||
| }, | ||
| "nbformat": 4, | ||
| "nbformat_minor": 4 | ||
| } |