my2048-rl-theano-n-tuple-Copy7.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from random import randint, shuffle, seed\n",
    "import sys\n",
    "from selenium import webdriver\n",
    "from selenium.webdriver.common.keys import Keys"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from IPython.display import clear_output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from c2048 import Game, push"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using gpu device 0: GeForce GTX 965M (CNMeM is disabled, cuDNN 4007)\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import theano\n",
    "import theano.tensor as T\n",
    "import lasagne\n",
    "from lasagne.layers import DenseLayer, InputLayer, batch_norm, DropoutLayer\n",
    "from lasagne.layers import  MergeLayer, ReshapeLayer, FlattenLayer, ConcatLayer\n",
    "floatX = theano.config.floatX\n",
    "from lasagne.nonlinearities import rectify, elu, softmax, sigmoid\n",
    "from lasagne.init import Constant, Sparse\n",
    "floatX = theano.config.floatX"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from lasagne.layers.dnn import Conv2DDNNLayer\n",
    "from lasagne.regularization import regularize_network_params, l1, l2, regularize_layer_params_weighted"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'float32'"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "floatX"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def Winit(shape):\n",
    "    rtn = np.random.normal(size=shape).astype(floatX)\n",
    "    rtn[np.random.uniform(size=shape) < 0.9] *= 0.01\n",
    "    return rtn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "input_var = T.tensor4()\n",
    "target_var = T.vector()\n",
    "N_FILTERS = 512\n",
    "N_FILTERS2 = 4096\n",
    "\n",
    "_ = InputLayer(shape=(None, 16, 4, 4), input_var=input_var)\n",
    "\n",
    "conv_a =  Conv2DDNNLayer(_, N_FILTERS, (2,1), pad='valid')#, W=Winit((N_FILTERS, 16, 2, 1)))\n",
    "conv_b =  Conv2DDNNLayer(_, N_FILTERS, (1,2), pad='valid')#, W=Winit((N_FILTERS, 16, 1, 2)))\n",
    "\n",
    "conv_aa =  Conv2DDNNLayer(conv_a, N_FILTERS2, (2,1), pad='valid')#, W=Winit((N_FILTERS2, N_FILTERS, 2, 1)))\n",
    "conv_ab =  Conv2DDNNLayer(conv_a, N_FILTERS2, (1,2), pad='valid')#, W=Winit((N_FILTERS2, N_FILTERS, 1, 2)))\n",
    "\n",
    "conv_ba =  Conv2DDNNLayer(conv_b, N_FILTERS2, (2,1), pad='valid')#, W=Winit((N_FILTERS2, N_FILTERS, 2, 1)))\n",
    "conv_bb =  Conv2DDNNLayer(conv_b, N_FILTERS2, (1,2), pad='valid')#, W=Winit((N_FILTERS2, N_FILTERS, 1, 2)))\n",
    "\n",
    "_ = ConcatLayer([FlattenLayer(x) for x in [conv_aa, conv_ab, conv_ba, conv_bb, conv_a, conv_b]])\n",
    "l_out = DenseLayer(_, num_units=1,  nonlinearity=None)\n",
    "\n",
    "prediction = lasagne.layers.get_output(l_out)\n",
    "P = theano.function([input_var], prediction)\n",
    "loss = lasagne.objectives.squared_error(prediction, target_var).mean()/2\n",
    "#layers = {conv1: 0.5, conv2: 0.5}\n",
    "#l1_penalty = regularize_layer_params_weighted(layers, l1)\n",
    "#loss = loss + 1e-4 * l1_penalty\n",
    "accuracy = lasagne.objectives.squared_error(prediction, target_var).mean()\n",
    "params = lasagne.layers.get_all_params(l_out, trainable=True)\n",
    "#params = [l_out.W]\n",
    "updates = lasagne.updates.adam(loss, params, beta1=0.5)\n",
    "#updates = lasagne.updates.sgd(loss, params, learning_rate=α)\n",
    "#updates = lasagne.updates.adamax(loss, params)\n",
    "\n",
    "\n",
    "train_fn = theano.function([input_var, target_var], loss, updates=updates)\n",
    "loss_fn = theano.function([input_var, target_var], loss)\n",
    "accuracy_fn =theano.function([input_var, target_var], accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from random import randint\n",
    "table ={2**i:i for i in range(1,16)}\n",
    "table[0]=0\n",
    "def make_input(grid):\n",
    "    g0 = grid\n",
    "    r = np.zeros(shape=(16, 4, 4), dtype=floatX)\n",
    "    for i in range(4):\n",
    "        for j in range(4):\n",
    "            v = g0[i, j]\n",
    "            r[table[v],i, j]=1\n",
    "    return r"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "logf=open(\"logf-rl-theano-n-tuple-6\", \"w\")\n",
    "def printx(*a, **kw):\n",
    "    print(*a, file=logf, flush=True, **kw)\n",
    "    print(*a, flush=True, **kw)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from random import random, randint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_grid(driver):\n",
    "    grid = np.zeros(shape=(4,4), dtype='uint16')\n",
    "    for x in driver.find_elements_by_class_name('tile'):\n",
    "        cl = x.get_attribute('class').split()\n",
    "        for t in cl:\n",
    "            if t.startswith('tile-position-'):\n",
    "                pos = int(t[14])-1, int(t[16])-1\n",
    "            elif t.startswith('tile-') and t[5].isdigit():\n",
    "                v = int(t[5:])\n",
    "        grid[pos[1], pos[0]] = v\n",
    "    return grid"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "matplotlib.rcParams['figure.figsize'] = (12.0, 8.0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import math\n",
    "import time\n",
    "from collections import defaultdict\n",
    "  \n",
    "def Vchange(grid, v):\n",
    "    g0 = grid\n",
    "    g1 = g0[:,::-1,:]\n",
    "    g2 = g0[:,:,::-1]\n",
    "    g3 = g2[:,::-1,:]\n",
    "    r0 = grid.swapaxes(1,2)\n",
    "    r1 = r0[:,::-1,:]\n",
    "    r2 = r0[:,:,::-1]\n",
    "    r3 = r2[:,::-1,:]\n",
    "    xtrain = np.array([g0,g1,g2,g3,r0,r1,r2,r3], dtype=floatX)\n",
    "    ytrain = np.array([v]*8, dtype=floatX)\n",
    "    train_fn(xtrain, ytrain)\n",
    "\n",
    "arrow=[Keys.ARROW_LEFT, Keys.ARROW_UP, Keys.ARROW_RIGHT, Keys.ARROW_DOWN]\n",
    "def gen_sample_and_learn(driver):\n",
    "    body = driver.find_element_by_tag_name('body')\n",
    "    game_len = 0\n",
    "    game_score = 0\n",
    "    last_grid = None\n",
    "    keep_playing =False\n",
    "    while True:\n",
    "        try:\n",
    "            grid_array = get_grid(driver)\n",
    "        except:\n",
    "            grid_array = None\n",
    "        board_list = []\n",
    "        if grid_array is not None:\n",
    "            if not keep_playing and grid_array.max()==2048:\n",
    "                driver.find_element_by_class_name('keep-playing-button').click()\n",
    "                keep_playing = True\n",
    "                time.sleep(1)\n",
    "            for m in range(4):\n",
    "                g = grid_array.copy()\n",
    "                s = push(g, m%4)\n",
    "                if s >= 0:\n",
    "                    board_list.append( (g, m, s) )\n",
    "        if board_list:\n",
    "            boards = np.array([make_input(g) for g,m,s in board_list], dtype=floatX)\n",
    "            p = P(boards).flatten()        \n",
    "            game_len+=1\n",
    "            best_move = -1\n",
    "            best_v = None\n",
    "            for i, (g,m,s) in enumerate(board_list):\n",
    "                v = 2*s + p[i]\n",
    "                if best_v is None or v > best_v:\n",
    "                    best_v = v\n",
    "                    best_move = m\n",
    "                    best_score = 2*s\n",
    "                    best_grid = boards[i]\n",
    "            body.send_keys(arrow[best_move])\n",
    "            game_score += best_score\n",
    "        else:\n",
    "            best_v = 0\n",
    "            best_grid = None\n",
    "        if last_grid is not None:\n",
    "            Vchange(last_grid, best_v)       \n",
    "        last_grid = best_grid\n",
    "        if not board_list:\n",
    "            break\n",
    "        plt.pause(0.05)\n",
    "    return game_len, grid_array.max(), game_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/lib/python3/dist-packages/matplotlib/backend_bases.py:2437: MatplotlibDeprecationWarning: Using default event loop until function specific to this GUI is implemented\n",
      "  warnings.warn(str, mplDeprecation)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 (136, 64, 1136)\n",
      "1 (385, 512, 5396)\n",
      "2 (212, 128, 2240)\n",
      "3 (275, 256, 3352)\n",
      "4 (262, 256, 3160)\n",
      "5 (209, 256, 2528)\n",
      "6 (177, 128, 1752)\n",
      "7 (309, 256, 3756)\n",
      "8 (428, 512, 6060)\n",
      "9 (288, 256, 3428)\n",
      "10 (221, 256, 2624)\n",
      "11 (270, 256, 3228)\n",
      "12 (320, 256, 3984)\n",
      "13 (257, 256, 2972)\n",
      "14 (279, 256, 3312)\n",
      "15 (467, 512, 6756)\n",
      "16 (513, 512, 7380)\n",
      "17 (378, 256, 4544)\n",
      "18 (531, 512, 7536)\n",
      "19 (190, 128, 1864)\n",
      "20 (792, 1024, 13344)\n",
      "21 (851, 1024, 14476)\n",
      "22 (826, 512, 12776)\n",
      "23 (1075, 1024, 18444)\n",
      "24 (736, 512, 11348)\n",
      "25 (894, 1024, 14984)\n",
      "26 (277, 256, 3412)\n",
      "27 (1343, 2048, 25976)\n"
     ]
    }
   ],
   "source": [
    "results = []\n",
    "driver = webdriver.Firefox()\n",
    "graph = plt.plot([], [], 'b')[0]\n",
    "dots256 = plt.plot([],[], 'ro')[0]\n",
    "dots512 = plt.plot([],[], 'yo')[0]\n",
    "dots1024 = plt.plot([],[], 'go')[0]\n",
    "plt.xlim((0,100))\n",
    "plt.ylim((0,25000))\n",
    "for j in range(200):\n",
    "    driver.get(\"https://gabrielecirulli.github.io/2048/\")\n",
    "    time.sleep(2)\n",
    "    result = gen_sample_and_learn(driver)\n",
    "    print(j, result)\n",
    "    results.append(result)\n",
    "    graph.set_data(np.arange(len(results)), np.array(results)[:, 2])\n",
    "    dots_data =[[],[],[]]\n",
    "    for i, d in enumerate(results):\n",
    "        c = 0 if d[1]<=256 else (1 if d[1]==512 else 2)\n",
    "        dots_data[c].append([i, d[2]])\n",
    "    dots_graph = [dots256, dots512, dots1024]\n",
    "    for i in range(3):\n",
    "        if dots_data[i]:\n",
    "            xy = np.array(dots_data[i])\n",
    "            dots_graph[i].set_data(xy[:, 0], xy[:,1])\n",
    "    plt.title(\"Game #%d\"%j, fontsize=64)\n",
    "    plt.draw()\n",
    "    plt.pause(3)\n",
    "    if result[1] >= 2048:\n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}