metrics.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "a15752bd-54c6-49d7-8b73-516214bed8f8",
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "import zmq\n",
    "import json\n",
    "import pandas as pd\n",
    "import os\n",
    "from pandas_gbq import read_gbq\n",
    "import plotly.express as px\n",
    "import plotly.offline as pyo\n",
    "import plotly.io as pio\n",
    "import re\n",
    "\n",
    "pio.renderers.default = 'notebook'  # or 'jupyterlab' if using JupyterLab\n",
    "import warnings\n",
    "\n",
    "# Suppress specific deprecation warnings\n",
    "warnings.filterwarnings(\"ignore\", category=FutureWarning)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b9ba6081-3df0-4f07-9e98-37362f9d5c8b",
   "metadata": {},
   "source": [
    "## Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "33853d5b-09af-4b16-9b86-5344745f4759",
   "metadata": {},
   "outputs": [],
   "source": [
    "NODE_DOMAIN_MAP = {\n",
    "\t\"GABMKJM6I25XI4K7U6XWMULOUQIQ27BCTMLS6BYYSOWKTBUXVRJSXHYQ\": \"Stellar Development Foundation\",\n",
    "\t\"GCGB2S2KGYARPVIA37HYZXVRM2YZUEXA6S33ZU5BUDC6THSB62LZSTYH\": \"Stellar Development Foundation\",\n",
    "\t\"GCM6QMP3DLRPTAZW2UZPCPX2LF3SXWXKPMP3GKFZBDSF3QZGV2G5QSTK\": \"Stellar Development Foundation\",\n",
    "\t\"GAK6Z5UVGUVSEK6PEOCAYJISTT5EJBB34PN3NOLEQG2SUKXRVV2F6HZY\": \"SatoshiPay\",\n",
    "\t\"GBJQUIXUO4XSNPAUT6ODLZUJRV2NPXYASKUBY4G5MYP3M47PCVI55MNT\": \"SatoshiPay\",\n",
    "\t\"GC5SXLNAM3C4NMGK2PXK4R34B5GNZ47FYQ24ZIBFDFOCU6D4KBN4POAE\": \"SatoshiPay\",\n",
    "\t\"GCFONE23AB7Y6C5YZOMKUKGETPIAJA4QOYLS5VNS4JHBGKRZCPYHDLW7\": \"LOBSTR\",\n",
    "\t\"GCB2VSADESRV2DDTIVTFLBDI562K6KE3KMKILBHUHUWFXCUBHGQDI7VL\": \"LOBSTR\",\n",
    "\t\"GD5QWEVV4GZZTQP46BRXV5CUMMMLP4JTGFD7FWYJJWRL54CELY6JGQ63\": \"LOBSTR\",\n",
    "\t\"GA7TEPCBDQKI7JQLQ34ZURRMK44DVYCIGVXQQWNSWAEQR6KB4FMCBT7J\": \"LOBSTR\",\n",
    "\t\"GA5STBMV6QDXFDGD62MEHLLHZTPDI77U3PFOD2SELU5RJDHQWBR5NNK7\": \"LOBSTR\",\n",
    "\t\"GAAV2GCVFLNN522ORUYFV33E76VPC22E72S75AQ6MBR5V45Z5DWVPWEU\": \"Blockdaemon Inc.\",\n",
    "\t\"GAVXB7SBJRYHSG6KSQHY74N7JAFRL4PFVZCNWW2ARI6ZEKNBJSMSKW7C\": \"Blockdaemon Inc.\",\n",
    "\t\"GAYXZ4PZ7P6QOX7EBHPIZXNWY4KCOBYWJCA4WKWRKC7XIUS3UJPT6EZ4\": \"Blockdaemon Inc.\",\n",
    "\t\"GBLJNN3AVZZPG2FYAYTYQKECNWTQYYUUY2KVFN2OUKZKBULXIXBZ4FCT\": \"Public Node\",\n",
    "\t\"GCIXVKNFPKWVMKJKVK2V4NK7D4TC6W3BUMXSIJ365QUAXWBRPPJXIR2Z\": \"Public Node\",\n",
    "\t\"GCVJ4Z6TI6Z2SOGENSPXDQ2U4RKH3CNQKYUHNSSPYFPNWTLGS6EBH7I2\": \"Public Node\",\n",
    "\t\"GA7DV63PBUUWNUFAF4GAZVXU2OZMYRATDLKTC7VTCG7AU4XUPN5VRX4A\": \"Franklin Templeton\",\n",
    "\t\"GARYGQ5F2IJEBCZJCBNPWNWVDOFK7IBOHLJKKSG2TMHDQKEEC6P4PE4V\": \"Franklin Templeton\",\n",
    "\t\"GCMSM2VFZGRPTZKPH5OABHGH4F3AVS6XTNJXDGCZ3MKCOSUBH3FL6DOB\": \"Franklin Templeton\",\n",
    "\t\"GD6SZQV3WEJUH352NTVLKEV2JM2RH266VPEM7EH5QLLI7ZZAALMLNUVN\": \"Whalestack LLC\",\n",
    "\t\"GADLA6BJK6VK33EM2IDQM37L5KGVCY5MSHSHVJA4SCNGNUIEOTCR6J5T\": \"Whalestack LLC\",\n",
    "\t\"GAZ437J46SCFPZEDLVGDMKZPLFO77XJ4QVAURSJVRZK2T5S7XUFHXI2Z\": \"Whalestack LLC\",\n",
    "}\n",
    "\n",
    "def filter_recent_data(df, minutes=5, timestamp_field=\"close_at\"):\n",
    "    df[timestamp_field] = pd.to_datetime(df[timestamp_field], utc=True)\n",
    "    current_time = pd.Timestamp.utcnow()\n",
    "    five_minutes_ago = current_time - pd.Timedelta(minutes=minutes)\n",
    "    df = df[df[timestamp_field] >= five_minutes_ago]\n",
    "    return df\n",
    "\n",
    "\n",
    "def get_longest_leaders(df, filter_last_five_min=True):\n",
    "    # Step 1: Filter last five minute worth of data if arg set to true\n",
    "    if filter_last_five_min:\n",
    "        filtered_df = filter_recent_data(df)\n",
    "    else:\n",
    "        filtered_df = df\n",
    "\n",
    "    # Step 2: Assign row numbers based on closed_at\n",
    "    filtered_df = filtered_df.sort_values('sequence_number')\n",
    "    filtered_df['rn'] = range(1, len(filtered_df) + 1)\n",
    "\n",
    "    # Step 3: Create a grouping identifier (grp)\n",
    "    filtered_df['grp'] = filtered_df['rn'] - filtered_df.groupby('node_id')['rn'].transform(lambda x: x.rank(method='first'))\n",
    "\n",
    "    # Step 4: Get start and end time for each group\n",
    "    windowed_data = filtered_df.groupby(['node_id', 'grp']).agg(start_ledger=('sequence_number', 'min'),\n",
    "                                                           end_ledger=('sequence_number', 'max')).reset_index()\n",
    "\n",
    "    # Step 5: Calculate continuous_time and validator_frequency\n",
    "    windowed_data['continuous_ledger'] = (windowed_data['end_ledger'] - windowed_data['start_ledger'])\n",
    "    result = windowed_data.groupby(['continuous_ledger', 'node_id']).size().reset_index(name='validator_frequency')\n",
    "\n",
    "    # Step 6: Sort the results\n",
    "    result = result.sort_values(by='continuous_ledger', ascending=False)\n",
    "    result['home_domain'] = result['node_id'].map(NODE_DOMAIN_MAP)\n",
    "    result[\"continuous_ledger\"] = result[\"continuous_ledger\"] +  1\n",
    "    return result\n",
    "\n",
    "def get_ledger_close_count_by_home_domain(df, filter_last_five_min=True):\n",
    "    if filter_last_five_min:\n",
    "        filtered_df = filter_recent_data(df)\n",
    "    else:\n",
    "        filtered_df = df\n",
    "\n",
    "    mapped_df = filtered_df.copy()\n",
    "    mapped_df['home_domain'] = filtered_df['node_id'].map(NODE_DOMAIN_MAP)\n",
    "    if filter_last_five_min:\n",
    "        mapped_df['minute'] = mapped_df['close_at'].dt.floor('min')\n",
    "        mapped_df['minute'] = mapped_df['minute'].dt.strftime('%Y-%m-%d %H:%M')\n",
    "        result_df = mapped_df.groupby(['home_domain', 'minute']).size().reset_index(name='count')\n",
    "    else:\n",
    "        mapped_df['day'] = mapped_df['close_at'].dt.date\n",
    "        result_df = mapped_df.groupby(['home_domain', 'day']).size().reset_index(name='count')\n",
    "    return result_df\n",
    "\n",
    "\n",
    "def add_operation_type_cols(df):\n",
    "    new_rows = []\n",
    "    \n",
    "    for _, row in df.iterrows():\n",
    "        parsed = json.loads(row['operations'])\n",
    "\n",
    "        for operation in ['claimable_balances', 'account_creation', 'payments', \n",
    "                          'offers_and_AMMs', 'trust', 'sponsorship', \n",
    "                          'soroban', 'other']:\n",
    "            new_rows.append({\n",
    "                'close_at': row['close_at'],\n",
    "                'node_id': row['node_id'],\n",
    "                'sequence_number': row['sequence_number'],\n",
    "                'operation_type': operation,\n",
    "                'count': int(parsed[\"categories\"].get(operation, 0))\n",
    "            })\n",
    "\n",
    "    new_df = pd.DataFrame(new_rows)\n",
    "    return new_df\n",
    "\n",
    "\n",
    "def get_operation_count_by_home_domain(df, filter_last_five_min=True):\n",
    "    if filter_last_five_min:\n",
    "        filtered_df = filter_recent_data(df)\n",
    "    else:\n",
    "        filtered_df = df\n",
    "\n",
    "    mapped_df = filtered_df.copy()\n",
    "    mapped_df['home_domain'] = filtered_df['node_id'].map(NODE_DOMAIN_MAP)\n",
    "    if filter_last_five_min:\n",
    "        mapped_df['minute'] = mapped_df['close_at'].dt.floor('min')\n",
    "        mapped_df['minute'] = mapped_df['minute'].dt.strftime('%Y-%m-%d %H:%M')\n",
    "        result_df = mapped_df.groupby(['home_domain', 'minute', 'operation_type']).agg({\n",
    "            'count': 'sum'\n",
    "        }).reset_index()\n",
    "    else:\n",
    "        mapped_df['day'] = mapped_df['close_at'].dt.date\n",
    "        result_df = mapped_df.groupby(['home_domain', 'day', 'operation_type']).agg({\n",
    "            'count': 'sum'\n",
    "        }).reset_index()\n",
    "    return result_df\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "859b7ac9-c29f-465d-83c9-069c04d5bd6c",
   "metadata": {},
   "source": [
    "## Plots"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "f0c079e0-dabe-4f69-b2de-bb3b6ce61668",
   "metadata": {},
   "outputs": [],
   "source": [
    "def write_plot_to_html(fig, filename, append=False):\n",
    "    html_string = pyo.plot(fig, include_plotlyjs='cdn', output_type='div')\n",
    "    \n",
    "    if os.path.exists(f\"{filename}.html\") and append == True:\n",
    "        with open(f\"{filename}.html\", \"a\") as f:\n",
    "            f.write(html_string)\n",
    "    else:\n",
    "        with open(f\"{filename}.html\", \"w\") as f:\n",
    "            f.write(html_string)\n",
    "\n",
    "def plot_continuous_leader_chart(df, title, filename, append=False):\n",
    "    fig = px.bar(\n",
    "        df,\n",
    "        x='continuous_ledger',\n",
    "        y='validator_frequency',\n",
    "        color='node_id',\n",
    "        title=title,\n",
    "        labels={'continuous_ledger': 'Continuous Ledger count', 'validator_frequency': 'Frequency'},\n",
    "        text='home_domain'\n",
    "    )\n",
    "\n",
    "    # Update layout for grouping\n",
    "    fig.update_layout(barmode='group')\n",
    "    return fig\n",
    "\n",
    "\n",
    "def plot_validator_frequency_chart(df, title, filename, x='', y='',labels={}, append=False):\n",
    "    fig = px.bar(\n",
    "        df,\n",
    "        x=x,\n",
    "        y=y,\n",
    "        color='home_domain',\n",
    "        title=title,\n",
    "        labels=labels,\n",
    "        barmode='stack',\n",
    "        text='home_domain'\n",
    "    )\n",
    "\n",
    "    if x == \"minute\":\n",
    "        fig.update_xaxes(tickmode='array', tickvals=df['minute'], ticktext=df['minute'])\n",
    "    else:\n",
    "        fig.update_xaxes(tickmode='array', tickvals=df['day'], ticktext=df['day'])\n",
    "\n",
    "    return fig\n",
    "\n",
    "def plot_operation_type_frequency_chart(df, title, filename, x='', y='',labels={}, append=False):\n",
    "    fig = px.bar(\n",
    "        df,\n",
    "        x=x,\n",
    "        y=y,\n",
    "        color='operation_type',\n",
    "        title=title,\n",
    "        labels=labels,\n",
    "        barmode='stack',\n",
    "        text='operation_type'\n",
    "    )\n",
    "\n",
    "    if x == \"minute\":\n",
    "        fig.update_xaxes(tickmode='array', tickvals=df['minute'], ticktext=df['minute'])\n",
    "    else:\n",
    "        fig.update_xaxes(tickmode='array', tickvals=df['day'], ticktext=df['day'])\n",
    "\n",
    "    return fig"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "279249dd-aba5-46b8-b26b-1b21690d0be8",
   "metadata": {},
   "source": [
    "## Data Sources"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "add2eb29-d04f-4f20-94c0-9da22e76320a",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_full_data_from_hubble():\n",
    "    nodes = NODE_DOMAIN_MAP.keys()\n",
    "    nodes_string = \",\".join(f\"'{item}'\" for item in nodes)\n",
    "    query = f\"\"\"\n",
    "      SELECT\n",
    "        hl.node_id as node_id,\n",
    "        hl.closed_at AS close_at\n",
    "      FROM crypto-stellar.crypto_stellar.history_ledgers AS hl\n",
    "      WHERE hl.closed_at BETWEEN '2024-01-01 00:00:00 UTC' AND '2025-01-01 00:00:00 UTC'\n",
    "      AND hl.node_id in ({nodes_string})\n",
    "    \"\"\"\n",
    "    full_df = read_gbq(query, project_id='crypto-stellar')\n",
    "    full_df['close_at'] = full_df['close_at'].dt.tz_localize(None)\n",
    "    full_df['close_at'] = pd.to_datetime(full_df['close_at'])\n",
    "    return full_df\n",
    "\n",
    "def get_full_data_from_CSV_files():\n",
    "    df = pd.read_csv(\"history_data_latest.csv\")\n",
    "    df.rename(columns={'close_time': 'close_at'}, inplace=True)\n",
    "    df['close_at'] = pd.to_datetime(df['close_at'], unit='s')\n",
    "    return df\n",
    "\n",
    "def get_live_data_socket_from_stream():\n",
    "    #  Socket to talk to server\n",
    "    context = zmq.Context()\n",
    "    socket = context.socket(zmq.SUB)\n",
    "    \n",
    "    print(\"Collecting validator info from pipeline ...\")\n",
    "    socket.connect(\"tcp://127.0.0.1:5555\")\n",
    "    socket.subscribe(\"\")\n",
    "    return socket"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7293bf78-48cb-4102-9f53-07eb0ab6906a",
   "metadata": {},
   "source": [
    "## Full history Analytics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "a57adc11-8848-40d1-8a79-1f4709744e53",
   "metadata": {},
   "outputs": [],
   "source": [
    "full_df = get_full_data_from_CSV_files()\n",
    "full_data_figs = []\n",
    "longest_leaders_result_df = get_longest_leaders(full_df, filter_last_five_min=False)\n",
    "full_data_figs.append(plot_continuous_leader_chart(longest_leaders_result_df[longest_leaders_result_df[\"continuous_ledger\"] >= 3], title=\"Full History: Nodes leading for greater than equal to 3 continuous ledgers\", filename=\"graph\", append=False))\n",
    "\n",
    "validator_count_by_home_domain_df = get_ledger_close_count_by_home_domain(full_df, filter_last_five_min=False)\n",
    "full_data_figs.append(plot_validator_frequency_chart(validator_count_by_home_domain_df, title=\"Full History: Validator count by day\", filename=\"graph\", x=\"day\", y=\"count\",labels={\"day\": \"Timestamp by day\", \"count\": \"Closing validator count\"}, append=True))\n",
    "\n",
    "operation_count_by_home_domain_df = get_operation_count_by_home_domain(add_operation_type_cols(full_df), filter_last_five_min=False)\n",
    "full_data_figs.append(plot_operation_type_frequency_chart(operation_count_by_home_domain_df[operation_count_by_home_domain_df[\"home_domain\"] == \"Stellar Development Foundation\"], title=\"Full History: Operation count by day for SDF validators\", filename=\"graph\", x=\"day\", y=\"count\",labels={\"day\": \"Timestamp by day\", \"count\": \"Operation type count\"}, append=True))\n",
    "full_data_figs.append(plot_operation_type_frequency_chart(operation_count_by_home_domain_df[operation_count_by_home_domain_df[\"home_domain\"] == \"LOBSTR\"], title=\"Full History: Operation count by day for LOBSTR validators\", filename=\"graph\", x=\"day\", y=\"count\",labels={\"day\": \"Timestamp by day\", \"count\": \"Operation type count\"}, append=True))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cc0b55d6-ad3d-4204-b95b-c79a872d3276",
   "metadata": {},
   "source": [
    "## Live data Analytics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "82952364-0584-4b8a-9b1e-29e3584b3bdc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting validator info from pipeline ...\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[8], line 6\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[38;5;28;01mwhile\u001b[39;00m \u001b[38;5;28;01mTrue\u001b[39;00m:\n\u001b[1;32m      5\u001b[0m     live_data_figs \u001b[38;5;241m=\u001b[39m []\n\u001b[0;32m----> 6\u001b[0m     message \u001b[38;5;241m=\u001b[39m \u001b[43msocket\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrecv\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m      7\u001b[0m     json_object \u001b[38;5;241m=\u001b[39m json\u001b[38;5;241m.\u001b[39mloads(message)\n\u001b[1;32m      8\u001b[0m     json_formatted_str \u001b[38;5;241m=\u001b[39m json\u001b[38;5;241m.\u001b[39mdumps(json_object, indent\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m2\u001b[39m)\n",
      "File \u001b[0;32m_zmq.py:1156\u001b[0m, in \u001b[0;36mzmq.backend.cython._zmq.Socket.recv\u001b[0;34m()\u001b[0m\n",
      "File \u001b[0;32m_zmq.py:1191\u001b[0m, in \u001b[0;36mzmq.backend.cython._zmq.Socket.recv\u001b[0;34m()\u001b[0m\n",
      "File \u001b[0;32m_zmq.py:1278\u001b[0m, in \u001b[0;36mzmq.backend.cython._zmq._recv_copy\u001b[0;34m()\u001b[0m\n",
      "File \u001b[0;32m_zmq.py:160\u001b[0m, in \u001b[0;36mzmq.backend.cython._zmq._check_rc\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "cur_data_vals = []\n",
    "socket = get_live_data_socket_from_stream()\n",
    "\n",
    "while True:\n",
    "    live_data_figs = []\n",
    "    message = socket.recv()\n",
    "    json_object = json.loads(message)\n",
    "    json_formatted_str = json.dumps(json_object, indent=2)\n",
    "\n",
    "    cur_data_vals.append(json_object)\n",
    "    cur_df = pd.DataFrame(cur_data_vals)\n",
    "    cur_df.rename(columns={'close_time': 'close_at'}, inplace=True)\n",
    "    cur_df['close_at'] = pd.to_datetime(cur_df['close_at'], unit='s')\n",
    "    cur_df['sequence_number'] = cur_df['sequence_number'].astype(int)\n",
    "\n",
    "    cur_res_df = get_longest_leaders(cur_df)\n",
    "    live_data_figs.append(plot_continuous_leader_chart(cur_res_df, title=\"Live data: Nodes leading for last 5 minutes\", filename=\"graph\", append=True))\n",
    "\n",
    "    cur_res_df = get_ledger_close_count_by_home_domain(cur_df)\n",
    "    live_data_figs.append(plot_validator_frequency_chart(cur_res_df, title=\"Live data: Validator count by minute\", filename=\"graph\", x=\"minute\", y=\"count\",labels={\"minute\": \"Timestamp by minutes\", \"count\": \"Closing validator count\"}, append=True))\n",
    "\n",
    "    append = False\n",
    "    for fig in live_data_figs + full_data_figs:\n",
    "        write_plot_to_html(fig, filename=\"graph\", append=append)\n",
    "        append = True\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a01fd389-b602-4297-860d-fc44525793da",
   "metadata": {},
   "source": [
    "## Chi Square tests"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bb63fca4-4a55-4bef-9c7e-b2a807ccc69f",
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import division\n",
    "\n",
    "'''\n",
    "This test and code was motivated by \n",
    "http://stats.stackexchange.com/questions/100838/autocorrelation-of-discrete-time-series\n",
    "and\n",
    "http://stats.stackexchange.com/questions/73084/analysis-of-temporal-patterns/73170#73170\n",
    "as described in\n",
    "P. C. O'Brien and P. J. Dyck. A runs test based on run lengths. Biometrics, pages 237-244, 1985.\n",
    "'''\n",
    "\n",
    "from collections import Counter, defaultdict\n",
    "import sys\n",
    "import math\n",
    "from scipy.stats import chi2\n",
    "\n",
    "def weighted_variance(counts):\n",
    "    avg = 0\n",
    "\n",
    "    for length, count in counts.items():\n",
    "        avg += count * length\n",
    "\n",
    "    counts_only = counts.values()\n",
    "\n",
    "    avg /= sum(counts_only)\n",
    "\n",
    "    var = 0\n",
    "\n",
    "    for length, count in counts.items():\n",
    "        var += count * math.pow((length - avg),2)\n",
    "\n",
    "    try:\n",
    "        var /= sum(counts_only) - 1\n",
    "    except:\n",
    "        #var = 0\n",
    "        raise Exception(\"Division by zero due to too few counts!\")\n",
    "\n",
    "    return var\n",
    "\n",
    "def runs_test(input, path = True):\n",
    "    '''\n",
    "    You can pass a path or a dictionary of runs lengths\n",
    "    path_passed = True states that you pass a path\n",
    "    '''\n",
    "\n",
    "    if path == True:\n",
    "        counter = 1\n",
    "        same = True\n",
    "        cats = defaultdict(lambda : defaultdict(int))\n",
    "\n",
    "        for i, elem in enumerate(input):\n",
    "            #print elem, i\n",
    "            if i == len(input) - 1:\n",
    "                cats[elem][counter] += 1\n",
    "                break\n",
    "\n",
    "            if input[i+1] == elem:\n",
    "                same = True\n",
    "                counter += 1\n",
    "            else:\n",
    "                cats[elem][counter] += 1\n",
    "                counter = 1\n",
    "    else:\n",
    "        cats = input\n",
    "\n",
    "    x2 = 0\n",
    "    df = 0\n",
    "    nr_elem = len(cats.keys())\n",
    "    fail_cnt = 0\n",
    "\n",
    "    for elem in cats.keys():\n",
    "        ns = sum([x*y for x,y in cats[elem].items()])\n",
    "        rs = sum(cats[elem].values())\n",
    "\n",
    "        #at the moment elements that have the following limitations get ignored\n",
    "        #one could also think about throwing an exception here and stopping the calculation\n",
    "        if len(cats[elem].keys()) == 1 or rs == 1 or (ns-rs) == 1:\n",
    "            #print \"Category '%s' has only one run length or only one run or ns-rs equals one! Sorry I will ignore it!\" % elem\n",
    "            fail_cnt += 1\n",
    "            continue\n",
    "\n",
    "        #print rs\n",
    "        ss = weighted_variance(cats[elem])\n",
    "        #print ss\n",
    "        cs = (pow(rs,2)-1)*(rs+2)*(rs+3) / (2*rs*(ns-rs-1)*(ns+1))\n",
    "        #print cs\n",
    "        vs = cs * ns * (ns-rs) / (rs*(rs+1))\n",
    "\n",
    "        x2 += ss * cs\n",
    "        #print x2\n",
    "        df += vs\n",
    "\n",
    "    #note that this is kind-of a hack, you can adapt this as wanted\n",
    "    if nr_elem - fail_cnt < 2:\n",
    "        raise Exception(\"I ignored too many categories of this sequences! Sorry can't do the test!\")\n",
    "\n",
    "    if x2 == 0 or df == 0:\n",
    "        raise Exception(\"x2 or df are zero, this really shouldn't happen!\")\n",
    "    pval = chi2.sf(x2,df)\n",
    "    return pval\n",
    "\n",
    "test_df = full_df.copy()\n",
    "test_df['home_domain'] = test_df['node_id'].map(NODE_DOMAIN_MAP)\n",
    "pval = runs_test(test_df.home_domain.values)\n",
    "print(pval)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e839140b-6f75-48de-9b86-ffa933988fb9",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}