WIP Story #340: Replay Widget on Binder

Dockerfile

@@ -0,0 +1,54 @@
+FROM ubuntu:18.04
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update && apt-get upgrade -yq
+RUN apt-get install -yq apt-utils apt-transport-https tzdata sudo curl python3-pip python3-dev git
+RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 10
+RUN apt-get -yq install postgresql-10
+
+#=========================================================================================
+# from https://docs.docker.com/engine/examples/postgresql_service/
+# ubuntu 18.04 contains postgres 10.3
+USER postgres
+# Create a PostgreSQL role named ``dallinger`` with ``dallinger`` as the password and
+# then create a database `dallinger` and 'dallinger-import' owned by the ``dallinger`` role.
+# Note: here we use ``&&\`` to run commands one after the other - the ``\``
+#       allows the RUN command to span multiple lines.
+RUN    /etc/init.d/postgresql start &&\
+    psql --command "CREATE USER dallinger WITH SUPERUSER PASSWORD 'dallinger';" &&\
+    createdb -O dallinger dallinger &&\
+    createdb -O dallinger dallinger-import
+
+# Adjust PostgreSQL configuration so that remote connections to the
+# database are possible.
+USER root
+RUN echo "host    all             all              0.0.0.0/0              trust" >> /etc/postgresql/10/main/pg_hba.conf
+# And add ``listen_addresses`` to ``/etc/postgresql/10/main/postgresql.conf``
+RUN echo "listen_addresses='*'" >> /etc/postgresql/10/main/postgresql.conf
+#=========================================================================================
+RUN sed /etc/postgresql/10/main/pg_hba.conf -e 's/md5/trust/g' --in-place
+
+# Redis
+RUN apt-get -yq install redis-server
+
+RUN sudo pip3 install matplotlib==2.1.0
+RUN sudo pip3 install -e git+https://github.com/Dallinger/Dallinger.git@stories/341-bot-data#egg=dallinger[data,jupyter]
+RUN sudo pip3 install -e git+https://github.com/Dallinger/Griduniverse.git@stories/342-realtime-playback#egg=dlgr-griduniverse
+
+# My binder specifics as per:
+# https://mybinder.readthedocs.io/en/latest/dockerfile.html#preparing-your-dockerfile
+ENV NB_USER jovyan
+ENV NB_UID 1000
+ENV HOME /home/${NB_USER}
+RUN adduser --disabled-password \
+    --gecos "Default user" \
+    --uid ${NB_UID} \
+    ${NB_USER}
+COPY . ${HOME}
+RUN chown -R ${NB_UID} ${HOME}
+RUN echo "jovyan ALL=(ALL) NOPASSWD: /usr/sbin/service" >> /etc/sudoers
+
+USER ${NB_USER}
+WORKDIR ${HOME}
+
+CMD ["jupyter", "notebook", "--ip", "0.0.0.0"]

README.md

@@ -1,3 +1,7 @@
 # NGS2 Cycle 2 pilot data
 
-[![Binder](https://mybinder.org/badge.svg)](https://mybinder.org/v2/gh/Dallinger/ngs2-cycle2-pilot-data/master?filepath=analyses.ipynb)
+[![Binder](https://mybinder.org/badge.svg)](https://mybinder.org/v2/gh/Dallinger/ngs2-cycle2-pilot-data/stories%2F341-bot-data?filepath=analyses.ipynb)
+
+# Bot Run Data
+
+[![Binder](https://mybinder.org/badge.svg)](https://mybinder.org/v2/gh/Dallinger/ngs2-cycle2-pilot-data/stories%2F341-bot-data?filepath=bot_runs%2Fbot_collector.ipynb)

binder_helpers.py

@@ -0,0 +1,10 @@
+import os
+import subprocess
+
+
+def start_services():
+    if os.path.exists('/usr/sbin/service'):
+        if subprocess.call("sudo /usr/sbin/service postgresql status", shell=True):
+            subprocess.call("sudo /usr/sbin/service postgresql start", shell=True)
+        if subprocess.call("sudo /usr/sbin/service redis-server status", shell=True):
+            subprocess.call("sudo /usr/sbin/service redis-server start", shell=True)

bot_runs/bot_collector.ipynb

@@ -0,0 +1,282 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import sys\n",
+    "import csv\n",
+    "import dallinger\n",
+    "from dallinger.experiments import Griduniverse\n",
+    "\n",
+    "\n",
+    "sys.path.append('..')\n",
+    "import binder_helpers\n",
+    "binder_helpers.start_services()\n",
+    "\n",
+    "ROWS = 49\n",
+    "COLS = 49\n",
+    "ORIG_CSV_LIMIT = csv.field_size_limit(ROWS*COLS*1024)\n",
+    "\n",
+    "BASE_ID = \"{}01daa{}-f7ed-43fa-ad6b-9928aa51f8e1\"\n",
+    "PARTICIPANTS = 9\n",
+    "NUM_AS_EXPERIMENTS = 3\n",
+    "NUM_RANDOM_EXPERIMENTS = 3\n",
+    "\n",
+    "EXP_CONFIG = {\n",
+    "    \"recruiter\": \"bots\",\n",
+    "    \"max_participants\": PARTICIPANTS,\n",
+    "}\n",
+    "\n",
+    "exp = Griduniverse()\n",
+    "data = {}"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Collecting 3 experiments with 9 AdvantageSeekingBot players\n",
+      ">>>> Retrieve: Data found for experiment ab01daa0-f7ed-43fa-ad6b-9928aa51f8e1, retrieving.\n",
+      ">>>> Retrieve: Data found for experiment ab01daa1-f7ed-43fa-ad6b-9928aa51f8e1, retrieving.\n",
+      ">>>> Retrieve: Data found for experiment ab01daa2-f7ed-43fa-ad6b-9928aa51f8e1, retrieving.\n",
+      "Data retrieval complete\n"
+     ]
+    }
+   ],
+   "source": [
+    "print('Collecting {} experiments with {} AdvantageSeekingBot players'.format(\n",
+    "    NUM_AS_EXPERIMENTS, PARTICIPANTS\n",
+    "))\n",
+    "\n",
+    "for count in range(NUM_AS_EXPERIMENTS):\n",
+    "    exp_id = BASE_ID.format('ab', count)\n",
+    "    config = EXP_CONFIG.copy()\n",
+    "    config['bot_policy'] = 'AdvantageSeekingBot'\n",
+    "    data[exp_id] = {\n",
+    "        'title': '{} Experiment #{} ({})'.format(config['bot_policy'], count + 1, exp_id),\n",
+    "        'data': exp.collect(exp_id, exp_config=config)\n",
+    "    }\n",
+    "    \n",
+    "print('Data retrieval complete.')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Collecting 3 experiments with 9 RandomBot players\n",
+      ">>>> Retrieve: Data found for experiment ad01daa0-f7ed-43fa-ad6b-9928aa51f8e1, retrieving.\n",
+      ">>>> Retrieve: Data found for experiment ad01daa1-f7ed-43fa-ad6b-9928aa51f8e1, retrieving.\n",
+      ">>>> Retrieve: Data found for experiment ad01daa2-f7ed-43fa-ad6b-9928aa51f8e1, retrieving.\n",
+      "Data retrieval complete.\n"
+     ]
+    }
+   ],
+   "source": [
+    "print('Collecting {} experiments with {} RandomBot players'.format(\n",
+    "    NUM_RANDOM_EXPERIMENTS, PARTICIPANTS\n",
+    "))\n",
+    "\n",
+    "for count in range(NUM_RANDOM_EXPERIMENTS):\n",
+    "    exp_id = BASE_ID.format('ad', count)\n",
+    "    config = EXP_CONFIG.copy()\n",
+    "    config['bot_policy'] = 'RandomBot'\n",
+    "    data[exp_id] = {\n",
+    "        'title': '{} Experiment #{} ({})'.format(config['bot_policy'], count + 1, exp_id),\n",
+    "        'data': exp.collect(exp_id, exp_config=config)\n",
+    "    }\n",
+    "\n",
+    "print('Data retrieval complete.')    "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {
+    "scrolled": false
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Successfully collected data from 6 bot experiments. Rendering replay widgets:\n",
+      "Ingesting dataset from ab01daa0-f7ed-43fa-ad6b-9928aa51f8e1-data.zip...\n"
+     ]
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "38f06c4e23f44271ac6f6875fdd9066a",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "VBox(children=(ExperimentWidget(children=(HTML(value='\\n<h2>AdvantageSeekingBot Experiment #1 (ab01daa0-f7ed-4…"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Ingesting dataset from ab01daa1-f7ed-43fa-ad6b-9928aa51f8e1-data.zip...\n"
+     ]
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "db5a419157a64f2fbc53fb6d8b3cb8da",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "VBox(children=(ExperimentWidget(children=(HTML(value='\\n<h2>AdvantageSeekingBot Experiment #2 (ab01daa1-f7ed-4…"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Ingesting dataset from ab01daa2-f7ed-43fa-ad6b-9928aa51f8e1-data.zip...\n"
+     ]
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "23c4951777ab4e37af3d8deffb690e29",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "VBox(children=(ExperimentWidget(children=(HTML(value='\\n<h2>AdvantageSeekingBot Experiment #3 (ab01daa2-f7ed-4…"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Ingesting dataset from ad01daa0-f7ed-43fa-ad6b-9928aa51f8e1-data.zip...\n"
+     ]
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "058a6f9ff1144200989455a030dca0a8",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "VBox(children=(ExperimentWidget(children=(HTML(value='\\n<h2>RandomBot Experiment #1 (ad01daa0-f7ed-43fa-ad6b-9…"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Ingesting dataset from ad01daa1-f7ed-43fa-ad6b-9928aa51f8e1-data.zip...\n"
+     ]
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "8f32522f6e264e3f8cea262aebdf14b5",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "VBox(children=(ExperimentWidget(children=(HTML(value='\\n<h2>RandomBot Experiment #2 (ad01daa1-f7ed-43fa-ad6b-9…"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Ingesting dataset from ad01daa2-f7ed-43fa-ad6b-9928aa51f8e1-data.zip...\n"
+     ]
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "0dc3e866a56a4200bd6028ec6a73d27b",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "VBox(children=(ExperimentWidget(children=(HTML(value='\\n<h2>RandomBot Experiment #3 (ad01daa2-f7ed-43fa-ad6b-9…"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "print('Successfully collected data from {} bot experiments. Rendering replay widgets:'.format(len(data)))\n",
+    "\n",
+    "for exp_id in data:\n",
+    "    replay_exp = Griduniverse()\n",
+    "    replay_exp.task = data[exp_id]['title']\n",
+    "    replay_exp.jupyter_replay(\n",
+    "        app_id=exp_id,\n",
+    "        session=dallinger.db.init_db(drop_all=True),\n",
+    "        rows=ROWS, columns=COLS\n",
+    "    )"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 2",
+   "language": "python",
+   "name": "python2"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.6.5"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}