diff --git a/.gitignore b/.gitignore index 372c13e..b259ed8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,2 @@ -__pycache__/ - +**/__pycache__ +Durivaux/.ipynb_checkpoints diff --git a/Durivaux/FlappyAgent.py b/Durivaux/FlappyAgent.py new file mode 100644 index 0000000..3ff6e08 --- /dev/null +++ b/Durivaux/FlappyAgent.py @@ -0,0 +1,35 @@ +import numpy as np +from collections import deque +from skimage import color, transform +from keras.models import load_model + +stackedX = [] +call = 0 +actions = [119, None] +dqn = load_model('dqn-925k.h5') +# Choose a new action every REPEAT call +REPEAT = 2 +lastAction = None + +def processScreen(screen): + """ Resize and gray-ify screen """ + return 255*transform.resize(color.rgb2gray(screen[60:,25:310,:]),(80,80)) + +def FlappyPolicy(state, screen): + global stackedX, call, actions, dqn, lastAction + + screenX = processScreen(screen) + + if call == 0: + stackedX = deque([screenX]*4, maxlen=4) + x = np.stack(stackedX, axis=-1) + else: + stackedX.append(screenX) + x = np.stack(stackedX, axis=-1) + + Q = dqn.predict(np.array([x])) + + if call % REPEAT == 0 or REPEAT == 1: + lastAction = actions[np.argmax(Q)] + call += 1 + return lastAction diff --git a/Durivaux/Learning_curves.ipynb b/Durivaux/Learning_curves.ipynb new file mode 100644 index 0000000..7722360 --- /dev/null +++ b/Durivaux/Learning_curves.ipynb @@ -0,0 +1,87 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "import re\n", + "import matplotlib.pyplot as plt\n", + "%matplotlib inline" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "regex = re.compile(r\"Min: (?P-?\\d+)\\tMean: (?P-?\\d+(\\.\\d+)?)\\tMax: (?P-?\\d+)\")\n", + "\n", + "scoreOffset = 6\n", + "mins = []\n", + "means = []\n", + "maxs = []\n", + "with open('training-1000k.log') as logFile:\n", + " for line in logFile.readlines():\n", + " m = regex.search(line)\n", + " if m is not None:\n", + " mins.append(int(m.group('min'))+scoreOffset)\n", + " means.append(float(m.group('mean'))+scoreOffset)\n", + " maxs.append(int(m.group('max'))+scoreOffset)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAA0YAAAHwCAYAAACCKH9ZAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzs3Xd0FdXax/HvTuhVAggIAqGHDgLS\nRUB670WqehEvCigoil3xRUVEBBUUBERAinRUQEV6CRqQ3nsPvQWS7PePSXJDh+ScTMrvs1aWOTN7\n9n5mkoXzZDdjrUVERERERCQp83E7ABEREREREbcpMRIRERERkSRPiZGIiIiIiCR5SoxERERERCTJ\nU2IkIiIiIiJJnhIjERERERFJ8pQYiYjEM8aYvMYYa4xJ5nYs8ZUxpqsxZrlLbb9hjPnOjbZFRMR7\n9D9dERGXGGP2AdmAsGiHC7kTzZ0ZY5YApYDs1tqQOGrTAgWttbvior0HYa39yO0YRETE89RjJCLi\nrsbW2nTRvo64HVB0xpi8QDXAAk1cDSYOJIZeusRwDyIiblBiJCISzxljuhljthpjLhhj9hhjekQ7\nV8MYcyhieNcpY8w+Y0zHaOfHGWO+McYsirj+L2NMnohzI40xn93U1lxjTJ9ohzoDq4FxQJebymaO\nKH/eGLPOGPNh9OFtxpgiEe2eNsZsN8a0uSmukcaY+RFxrTHG5I84tzSi2AZjzEVjTNv7eEZ3a6uh\nMeafiDgPGmPejXYuctjiM8aYA8Af0Y51McYciHiuA6Nd864xZuJN19+pbGpjzHhjzJmIn+GrxphD\nd7mPYtHu47gx5o1oz+vDaOVqRK8n4uf+mjFmI3DJGPOmMWb6TXV/YYwZHvF9RmPMGGPMUWPM4Yif\nnW/EuQIRvyfnIu7np3s9fxGRxECJkYhI/HcCaARkALoBnxtjykY7nx3IAuTESV5GG2MKRzvfEfgg\nokwQ8GPE8fFAe2OMD4AxJgtQC5gc7drOEeV/BOoaY7JFOzcSuBTRfheiJU7GmLTAImAS8DDQHvjK\nGFMs2vXtgfeATMAuYBCAtbZ6xPlSEb1od30xv4+2LkXcx0NAQ6CnMabZTdU8AQQAdaMdqwoUjngm\nbxtjAu4Sxp3KvgPkBfIBTwFP3+U+0gOLgV+BR4ACwO93afNm7XHu7yHgB6CBMSZDRN2+QBucZwTO\nzz40oo0yQB3g2YhzHwALcX4uuYAvHyAGEZEES4mRiIi7ZhljzkZ8zbpdAWvtfGvtbuv4C+eltdpN\nxd6y1oZEnJ+P8xIcab61dmnE/KCBQCVjzKPW2rXAOZyXeYB2wBJr7XEAY0xVIA8w1Vq7HtgNdIg4\n5wu0BN6x1l621m7BedmO1AjYZ6393lobaq39G5gBtIpW5mdr7VprbShO4lX6/h/bDe7alrV2ibX2\nX2ttuLV2I07i98RNdbxrrb1krb0S7dh71tor1toNwAaceVZ3cqeybYCPrLVnrLWHgOH3uI9j1trP\nrLVXrbUXrLVr7vMZAAy31h6MiGM/8DcQmQDWBC5ba1dHJLf1gT4R93wC+Bzn5w9wHefn/khEHK4s\nciEiEteUGImIuKuZtfahiK+bezEAMMbUN8asjhhedRZogNP7E+mMtfZStM/7cXocIh2M/MZaexE4\nHe38eP7Xi/E0Tk9DpC7AQmvtqYjPk/hfr1BWnAV8DkYrH/37PMDj0ZK+szg9V9mjlTkW7fvLQLpb\nbv7+3LUtY8zjxpg/jTEnjTHngOe58fndHHtM4rtT2Ue48zO62aM4yWdM3Vz3JJxeJHAS2sjeojxA\ncuBotOc1Cqe3DeBVwABrjTGbjTHdYxGTiEiCoQmaIiLxmDEmJU7vR2dgtrX2ekTPkolWLJMxJm20\n5Cg3sCna+Uej1ZcO8AMiF3mYCGwyxpTCGUo2K6JcapzeDl9jTORLf0rgoYiym3CGYuUCdtzcDs5L\n+l/W2qdifPP3715tTQJGAPWttVeNMcO4NTGyXortKM4z2hLx+dG7lD3I/xKZm10C0kT7nP02ZW6+\nh2nAZ8aYXEBzoFK0dkKALBG9dTdWYu0x4DmI6jVcbIxZGh9XCBQR8ST1GImIxG8pcBKSk0CoMaY+\nznyQm71njElhjKmGMyRrWrRzDYwxVY0xKXDmj6yx1h4EiBjetQ6np2hGtKFkzXCWES+KM8StNE7i\ntAzobK0NA34G3jXGpDHGFMFJ3iLNAwoZYzoZY5JHfJW/xzyd6I7jzMu5H/dqKz1wOiIpqkDEcMA4\nMhV43RiTyRiTE+h1l7LzgOzGmD7GmJTGmPTGmMcjzgXh/Bz9jDHZgT53rsZhrT0JLAG+B/Zaa7dG\nHD+KMxzzM2NMBmOMjzEmvzHmCQBjTOuIZArgDE7CFXZLAyIiiYwSIxGReMxaewF4CecF+wzOS/2c\nm4odizh3BGeuzvPW2m3Rzk/CWQTgNPAYzjCz6MYDJbh1GN331toD1tpjkV84PS8djbMkdC8gY0T7\nP+DM3QmJFncdnHkrRyLKfIyT5N2Pd4HxEUO92tyt4H209QLwvjHmAvA2zrOMK+8Dh4C9OAsrTCfi\nGd0s4j6eAhrj3MNO4MmI0z/gzF3ah5PU3O9KcZOA2vxvGF2kzjhJ9xac353pQI6Ic+WBNcaYizi/\na72ttXvvsz0RkQTLWOut0QMiIuJtxpgawERrba47nB8HHLLWvnmXOqrjDKnLa60Nj0UsH+NsAtvl\nnoWTKGNMT6CdtfbmxR9ERMRl6jESEUnCjDHJgd7Adw+aFBln76CSxlEBeAaY6Y04EypjTA5jTJWI\n4WqFgVfQMxIRiZe0+IKISBIVMQcnEGeIVrcYVJEeZ/jcIzh7LX0GzPZYgIlDCpwV3/yBs8AU4CtX\nIxIRkdvSUDoREREREUnyNJRORERERESSPCVGIiIiIiKS5CXoOUZZsmSxefPmdTsMERERERGJp9av\nX3/KWpv1XuUSdGKUN29eAgMD3Q5DRERERETiKWPM/vspp6F0IiIiIiKS5CkxEhERERGRJE+JkYiI\niIiIJHkJeo7R7Vy/fp1Dhw5x9epVt0NJFFKlSkWuXLlInjy526GIiIiIiHhNokuMDh06RPr06cmb\nNy/GGLfDSdCstQQHB3Po0CH8/f3dDkdERERExGsS3VC6q1evkjlzZiVFHmCMIXPmzOp9ExEREZFE\nL9ElRoCSIg/SsxQRERGRpCBRJkZuM8bQqVOnqM+hoaFkzZqVRo0axai+OXPmMHjwYE+FJyIiIiIi\nN0l0c4zig7Rp07Jp0yauXLlC6tSpWbRoETlz5oxxfU2aNKFJkyYejFBERERERKJTj5GX1K9fn/nz\n5wMwefJk2rdvH3Vu7dq1VK5cmTJlylC5cmW2b98OwNChQ+nevTsA//77L8WLF+fy5cuMGzeOXr16\nAdC1a1d69uzJk08+Sb58+fjrr7/o3r07AQEBdO3aNaqNdOnSRX0/ffr0qHP3e72IiIiISFKSqHuM\n+vzah6BjQR6ts3T20gyrN+ye5dq1a8f7779Po0aN2LhxI927d2fZsmUAFClShKVLl5IsWTIWL17M\nG2+8wYwZM+jTpw81atRg5syZDBo0iFGjRpEmTZpb6j5z5gx//PEHc+bMoXHjxqxYsYLvvvuO8uXL\nExQUROnSpe8aW2yvFxERERFJbBJ1YuSmkiVLsm/fPiZPnkyDBg1uOHfu3Dm6dOnCzp07McZw/fp1\nAHx8fBg3bhwlS5akR48eVKlS5bZ1N27cGGMMJUqUIFu2bJQoUQKAYsWKsW/fvnsmNrG9XkREREQk\nsUnUidH99Ox4U5MmTejXrx9LliwhODg46vhbb73Fk08+ycyZM9m3bx81atSIOrdz507SpUvHkSNH\n7lhvypQpASeRivw+8nNoaChw42pyNy+3fT/Xi4iIiIgkJZpj5EXdu3fn7bffjuqRiXTu3LmoxRjG\njRt3w/HevXuzdOlSgoODmT59eozbzpYtG1u3biU8PJyZM2fGuB4RERERkaRAiZEX5cqVi969e99y\n/NVXX+X111+nSpUqhIWFRR3v27cvL7zwAoUKFWLMmDEMGDCAEydOxKjtwYMH06hRI2rWrEmOHDli\nfA8iIiIiIkmBsda6HUOMlStXzgYGBt5wbOvWrQQEBLgUUeKkZyoiIiIiCZUxZr21tty9yqnHSERE\nREREkjwlRiIiIiJ3MGDxAMqMKkNCHmEjIvdHiZGIiIjIbYSFh/F90PcEHQti4/GNbocjIl4WbxIj\nY0yAMeYbY8x0Y0xPt+MRERGRpG3p/qWcuOQsgjRzm1Z4FUnsvJoYGWPGGmNOGGM23XS8njFmuzFm\nlzFmAIC1dqu19nmgDXDPyVEiIiIi3jRtyzRSJ0vNYzkeY9a2WW6HIyJe5u0eo3FAvegHjDG+wEig\nPlAUaG+MKRpxrgmwHPjdy3GJiIiI3FFYeBg/b/2ZhoUa0qFEBzYc38DeM3vdDktEvMiriZG1dilw\n+qbDFYBd1to91tprwBSgaUT5OdbaykDHO9VpjPmPMSbQGBN48uRJb4UeazNnzsQYw7Zt29wORURE\nRB7QsgPLOH7pOK2LtqZ5keaAhtOJJHZuzDHKCRyM9vkQkNMYU8MYM9wYMwpYcKeLrbWjrbXlrLXl\nsmbN6u1YY2zy5MlUrVqVKVOmxLqu6JvAioiIiPdN2+wMo2tYsCH+mfwpla2UEiORRM6NxMjc5pi1\n1i6x1r5kre1hrR0Z51F50MWLF1mxYgVjxoyJSozatm3LggX/y/e6du3KjBkzCAsLo3///pQvX56S\nJUsyatQoAJYsWcKTTz5Jhw4dKFGiBADNmjXjscceo1ixYowePTqqrjFjxlCoUCFq1KjBc889R69e\nvQA4efIkLVu2pHz58pQvX54VK1bE1SMQERFJsMLCw5ixdQYNCjYgbYq0ADQr0owVB1ZELcYgIolP\nMhfaPAQ8Gu1zLuCINxrq0weCgjxbZ+nSMGzY3cvMmjWLevXqUahQIfz8/Pj7779p164dP/30Ew0a\nNODatWv8/vvvfP3114wZM4aMGTOybt06QkJCqFKlCnXq1AFg7dq1bNq0CX9/fwDGjh2Ln58fV65c\noXz58rRs2ZKQkBA++OAD/v77b9KnT0/NmjUpVaoUAL1796Zv375UrVqVAwcOULduXbZu3erZByIi\nIpLILD+wnOOXjtOqaKuoY82LNOe9v95jzvY5PFv2WRejExFvcSMxWgcUNMb4A4eBdkAHF+LwmsmT\nJ9OnTx8A2rVrx+TJk/nggw946aWXCAkJ4ddff6V69eqkTp2ahQsXsnHjRqZPnw7AuXPn2LlzJylS\npKBChQpRSRHA8OHDmTnT6cY/ePAgO3fu5NixYzzxxBP4+fkB0Lp1a3bs2AHA4sWL2bJlS9T158+f\n58KFC6RPnz5OnoOIiEhCNH3LdFIlS0WjQo2ijpXMVhL/h/yZuW2mEiORRMqriZExZjJQA8hijDkE\nvGOtHWOM6QX8BvgCY621m73R/r16drwhODiYP/74g02bNmGMISwsDGMMn3zyCTVq1OC3337jp59+\non379gBYa/nyyy+pW7fuDfUsWbKEtGnT3vB58eLFrFq1ijRp0lCjRg2uXr161524w8PDWbVqFalT\np/bOzYqIiCQy4TacGVtnUL9AfdKlSBd13BhD8yLNGbFuBOdDzpMhZQYXoxQRb/D2qnTtrbU5rLXJ\nrbW5rLVjIo4vsNYWstbmt9YOetB6jTGNjTGjz5075/mgY2n69Ol07tyZ/fv3s2/fPg4ePIi/vz/L\nly+nXbt2fP/99yxbtiwqEapbty5ff/01169fB2DHjh1cunTplnrPnTtHpkyZSJMmDdu2bWP16tUA\nVKhQgb/++oszZ84QGhrKjBkzoq6pU6cOI0aMiPoc5OlxhSIiIonMigMrOHrxKK2Ltr7lXPOA5lwL\nu8avu351ITIR8TY3Fl+INWvtXGvtfzJmzOh2KLeYPHkyzZs3v+FYy5YtmTRpEnXq1GHp0qXUrl2b\nFClSAPDss89StGhRypYtS/HixenRowehoaG31FuvXj1CQ0MpWbIkb731FhUrVgQgZ86cvPHGGzz+\n+OPUrl2bokWLEvlchg8fTmBgICVLlqRo0aJ88803Xr57ERGRhG3almmk9E15wzC6SJVyVSJrmqxa\nnU4kkTJ3G4oV35UrV84GBgbecGzr1q0EBAS4FJE7Ll68SLp06QgNDaV58+Z07979luQsNpLiMxUR\nkaQn3Ibz6OePUiFnBWa2vX3y89yc5/hp80+c7H+SlMlSxnGEIhITxpj11tpy9yqXIHuM5Ebvvvsu\npUuXpnjx4vj7+9OsWTO3QxIREUlwVh5cyZELR247jC5S84DmXLh2gT/2/hGHkYlIXHBjVTrxsCFD\nhrgdgoiISII3bbMzjK5xocZ3LFPLvxbpUqRj5raZ1C9YPw6jExFvU4+RiIiIJHnhNpzpW6dTr0A9\n0qe887YWKZOlpEHBBszePpuw8LA4jFBEvE2JkYiIiCR5qw6uuucwukjNizTnxKUTrD60Og4iE5G4\nkiATo/i8XLeIiIgkPJGr0TUufOdhdJEaFGxACt8UWp1OJJFJkIlRfF6uW0RERBKWcBvO9C3TqVug\n7n1t3JohZQZq+ddi5raZd91oXUQSlgSZGMV3xhg6deoU9Tk0NJSsWbPSqJGzJ8KcOXMYPHjwXes4\ncuQIrVq18mqcIiIiAmsOreHwhcP3NYwuUrMizdhzZg+bTmzyYmQiEpeUGHlB2rRp2bRpE1euXAFg\n0aJF5MyZM+p8kyZNGDBgwF3reOSRR5g+fbpX4xQRERFnGF0K3xR3XY3uZk0LN8VgNJxOJBFRYuQl\n9evXZ/78+QBMnjyZ9u3bR50bN24cvXr1AqBr16689NJLVK5cmXz58kUlQ/v27aN48eJR5Zs1a0bj\nxo3x9/dnxIgRDB06lDJlylCxYkVOnz4NQI0aNYjc8PbUqVPkzZv3ga4XERFJaiKH0dXJX4eMqe5/\niH62dNmo/GhlJUYiiUji3seoTx8ICvJsnaVLw7Bh9yzWrl073n//fRo1asTGjRvp3r07y5Ytu23Z\no0ePsnz5crZt20aTJk1uO4Ru06ZN/PPPP1y9epUCBQrw8ccf888//9C3b18mTJhAnz597hpPbK8X\nERFJjNYeXsvB8wf5sOaHD3xt8yLN6beoH3vP7MU/k78XohORuKQeIy8pWbIk+/btY/LkyTRo0OCu\nZZs1a4aPjw9Fixbl+PHjty3z5JNPkj59erJmzUrGjBlp3Njp7i9RogT79u27ZzyxvV5ERCQxmrZ5\nGsl9ktOkcJMHvrZZkWYAzN4+29NhiYgLEneP0X307HhTkyZN6NevH0uWLCE4OPiO5VKmTBn1/Z1W\nt4lexsfHJ+qzj48PoaGhACRLlozw8HAArl69+sDXi4iIJCXWWqZvdYbRPZTqoQe+Pr9ffko8XIKZ\n22bSp6JGXogkdAmyxyih7GPUvXt33n77bUqUKBEn7eXNm5f169cDaOEGERGRe1h7eC0Hzh14oNXo\nbta8SHOWH1jOyUsnPRiZiLghQSZGCWUfo1y5ctG7d+84a69fv358/fXXVK5cmVOnTsVZuyIiIgnR\ntC3OMLqmRZrGuI7mAc0Jt+HM2T7Hg5GJiBtMQt6YrFy5cjZyFbZIW7duJSAgwKWIEic9UxERSWys\ntfh/4U+xh4sxv8P8WNWTb3g+imUtxrwO8zwYoYh4ijFmvbW23L3KJcgeIxEREZHYWHdkHfvP7Y/V\nMDpwNnVvVrgZi/cs5kLIBQ9FJyJuUGIkIiIiSU7kanRNC8d8GF2k5gHNCQkL4dddv3ogMhFxixIj\nERERSVIiV6Orna82mVJninV9VR6tQtY0WbXZq0gCp8RIREREkpT1R9ez7+y+WA+ji+Tr40uTwk2Y\nv3M+18KueaROEYl7SoxEREQkSZm2eRrJfJLFajW6mzUr0ozzIef5c++fHqtTROJWgkyMEso+RiIi\nIhK/WGuZtmUatfxr4Zfaz2P11s5Xm3Qp0mk4nUgCliATo4Swj9GhQ4do2rQpBQsWJH/+/PTu3Ztr\n164xbtw4evXqddtrKleuHKO2Zs2axZYtW6I+v/322yxevDhGdYmIiCRmfx/9m71n93psGF2kVMlS\nUb9AfWZvn024Dfdo3SISNxJkYhTfWWtp0aIFzZo1Y+fOnezYsYOLFy8ycODAu163cuXKGLV3c2L0\n/vvvU7t27RjVJSIikphN2zINX+NLsyLNPF538yLNOXbxGKsPrfZ43SLifUqMvOCPP/4gVapUdOvW\nDQBfX18+//xzxo4dy+XLlzl48CD16tWjcOHCvPfee1HXpUuXLur7Tz/9lPLly1OyZEneeeedqOMT\nJkygZMmSlCpVik6dOrFy5UrmzJlD//79KV26NLt376Zr165Mnz6dX375hTZt2kRdu2TJEho3bgzA\nwoULqVSpEmXLlqV169ZcvHjR249FRETEVVHD6PLVInOazB6vv0HBBiT3Sc6sbbM8XrdIQrL15NYE\n2XOazO0AvKlPnz4EBQV5tM7SpUszbNiwu5bZvHkzjz322A3HMmTIQO7cuQkNDWXt2rVs2rSJNGnS\nUL58eRo2bEi5cv/bjHfhwoXs3LmTtWvXYq2lSZMmLF26lMyZMzNo0CBWrFhBlixZOH36NH5+fjRp\n0oRGjRrRqlWrG9p86qmn6NGjB5cuXSJt2rT89NNPtG3bllOnTvHhhx+yePFi0qZNy8cff8zQoUN5\n++23PfegRERE4pl/jv3DnjN7eL3q616pP2OqjNT0r8nMbTP5uPbHGGO80o5IfLb60GpqT6hN34p9\n+aDmB26H80DUY+QF1trb/mMYefypp54ic+bMpE6dmhYtWrB8+fIbyi1cuJCFCxdSpkwZypYty7Zt\n29i5cyd//PEHrVq1IkuWLAD4+d190miyZMmoV68ec+fOJTQ0lPnz59O0aVNWr17Nli1bqFKlCqVL\nl2b8+PHs37/fcw9AREQkHpq22XvD6CI1L9KcXad3sfnkZq+1IRJfbTy+kfo/1idH+hy8UP4Ft8N5\nYIm6x+hePTveUqxYMWbMmHHDsfPnz3Pw4EF8fX1vSZpu/myt5fXXX6dHjx43HB8+fPgD//Wpbdu2\njBw5Ej8/P8qXL0/69Omx1vLUU08xefLkB6pLREQkoYocRlfTvyZZ0mTxWjtNizSl5/yezNw6k+IP\nF/daOyLxzY7gHTz1w1OkS5GOxZ0WkyN9DrdDemDqMfKCWrVqcfnyZSZMmABAWFgYr7zyCl27diVN\nmjQsWrSI06dPc+XKFWbNmkWVKlVuuL5u3bqMHTs2at7P4cOHOXHiBLVq1WLq1KkEBwcDcPr0aQDS\np0/PhQsXbhtLjRo1+Pvvv/n2229p27YtABUrVmTFihXs2rULgMuXL7Njxw7PPwgREZF4IuhYELvP\n7Pb4anQ3y54uO5UeraRluyVOrD+yni/XfOn6xsIHzh2g9oTaWGtZ3GkxeR7K42o8MaXEyAuMMcyc\nOZNp06ZRsGBBChUqRKpUqfjoo48AqFq1Kp06daJ06dK0bNkyan5RZG9QnTp16NChA5UqVaJEiRK0\natWKCxcuUKxYMQYOHMgTTzxBqVKlePnllwFo164dn376KWXKlGH37t03xOLr60ujRo345ZdfaNSo\nEQBZs2Zl3LhxtG/fnpIlS1KxYkW2bdsWV49HREQkzk3fMh1f40vzgOZeb6tZ4Wb8c+wf9p/VMHXx\njtWHVtNwUkPKfVuOl359ifo/1ufs1bOuxHL84nFqT6jN+ZDzLOy0kMJZCrsShycYa63bMcRYuXLl\nbGBg4A3Htm7dSkBAgEsRxVxwcDBly5aNl3N9EuozFRERAWcYXeERhcnzUB4WdVrk9fZ2nd5FwS8L\nMqzuMHpX7O319iTpWH5gOe//9T6L9iwic+rMvFzpZbKmycp/F/yXgpkLsqDDgjjtrTl95TQ1xtVg\nz5k9LOq0iEqPVoqzth+EMWa9tbbcvcqpxygeOHLkCJUqVaJfv35uhyIiIpLobDy+kZ2nd3p9GF2k\nAn4FKP5wcQ2nE4+w1vLn3j95cvyTVPu+GkHHgvik9ifs67OPN6q9wXOPPcdvT//G4fOHqTimIoFH\nAu9dqQdcCLlAgx8bsD14O7PazYq3SdGDSJCJkTGmsTFm9Llz59wOxSMeeeQRduzYwYsvvuh2KCIi\nIonOtC3T8DE+NC/i/WF0kZoXac6yA8s4eelknLUpiYu1lkW7F1F9XHVqTqjJtlPbGFpnKPv67KN/\nlf6kS/G//S+f9H+Slc+sJKVvSp4Y9wTzdszzamxXQ6/SdEpTAo8EMrXVVGrnq+3V9uJKgkyMrLVz\nrbX/yZgxo9uhiIiISDwWuRpdjbw1yJo2a5y126xIM8JtuNdfUCXxsdayYOcCKo2pRJ2Jddh3dh9f\n1v+SPS/toW+lvqRJnua21xXNWpTVz66maNaiNJ3SlK/WfeWV+K6HXafNtDYs2beE8c3G07RIU6+0\n44YEmRjdS0KeNxXf6FmKiEhC9u+Jf9kRvCPOhtFFKpO9DHky5tFwuhg4c+UMfX/ty8FzB90OJU5Z\na5m9bTblvy1Pw0kNOXbxGN80/IZdL+6iV4VepE6e+p51ZE+XnSVdltCwYEP+u+C/9FvYj3Ab7rEY\nw8LD6DKrC3N3zOWrhl/RsWRHj9UdHyS6fYxSpUpFcHAwmTNn1o7TsWStJTg4mFSpUrkdioiISIxM\n2+wMo2sR0CJO2zXG0KxIM74J/IaL1y7eMOxJ7izchtNpZifm75zP2ZCzfN/0e7dD8rpwG87PW3/m\nw6UfsuH4BvJlyseYJmPoVLITyX2TP3B9aVOkZWbbmfT5tQ+frfqMfWf38UPzH+4rsbobay095/dk\n8qbJfFz7Y54v93ys6ouPEl1ilCtXLg4dOsTJkxrT6wmpUqUiV65cbochIiLywCKH0T2R5wkeTvtw\nnLffvEhzvljzBb/u+pVWRVvFefsJ0eDlg5m/cz5FshRh4saJvF/jfR7N+KjbYXlFWHgYUzdPZdCy\nQWw+uZlCmQsxodkE2pdoTzKf2L2i+/r4Mrz+cPL75efl317m8IXDzGk3J8bDSa219F/Un2///paB\n1QbyapVXYxVffJXoEqPkyZM5NiEFAAAgAElEQVTj7+/vdhgiIiLisk0nNrE9eDu9H3dnyewquauQ\nJU0WZm2bpcToPvy+53fe+vMt2hVvx+BagynwZQGGrhrK5/U+dzs0j5v872Te++s9tgdvp2jWokxq\nMYk2xdrg6+PrsTaMMfSp2IfcGXPT8eeOVBxTkV86/kKhzIUeuK4Pl37IZ6s+48UKL/LBkx94LMb4\nJlHOMRIRERGJXI0urofRRUrmk4zGhRozb8c8roVdcyWGhOLw+cO0n9GewpkL823jb8nzUB7aF2/P\nt39/S/DlYLfD86hRgaPo8HMHUiZLybTW0/i357+0L9Heo0lRdC0CWrCkyxIuhFyg0phKLNu/7IGu\n/2L1F7y95G26lOrCsHrDEvVUFSVGIiIikuhEDqOrnqc62dJlcy2O5kWacy7kHEv2LXEthvjueth1\n2k5vy+Xrl5nRZkbUfKxXq7zKpeuXGLlupMsRes7S/Uvp9Usv6heoz9//+ZtWRVvhY7z/Ov54rsdZ\n9cwqsqTJQu0favPTpp/u67qx/4ylz299aBnQku+afBcnsbopcd+diIiIJElbTm5h26ltcb4a3c2e\nyv8UaZOnZeZWrU53J68tfo0VB1cwpskYArIGRB0v/nBxGhdqzPA1w7l07ZKLEXrGvrP7aDm1Jfkz\n5Wdyy8le6yG6k/x++Vn1zCoez/k47Wa04+PlH9919eFpm6fx3NznqJu/Lj+2+DHW854SAiVGIiIi\nkuhM2zINg3FtGF2kVMlSUb9gfWZvn+3RZZMTi+lbpvP56s95scKLtC3e9pbzA6oOIPhKMGP+GeNC\ndJ5z8dpFmk5pyvWw68xpP4eMqdzZi9MvtR+LOi2iffH2DPh9AM/Pe57Q8NBbyi3YuYCOP3ek8qOV\n+bntz6RMltKFaOOeEiMRERFJdCKH0WVPl93tUGhWuBlHLx5l7eG1bocSr2w/tZ3us7tTMVdFhtQZ\nctsylR+tTLXc1fhs1WdcD7sexxF6RrgNp+usrmw6sYmfWv0Uo8UPPCllspRMbDGRN6q+wei/R9N4\ncmMuhFyIOv/Xvr9oObUlJbKVYF77eXfcUDYxSvx9YiIiIhLvWWvp82sfdpzeEeu6QsND2XJyCyPq\nj/BAZLHXsFBDkvkkY+bWmVTMVdHtcOKFS9cu0XJqS1ImS8nUVlNJ4ZvijmUHVB1Aw0kNmbJpCp1K\ndYrDKD3jg78+YMbWGQytM5S6Beq6HQ4APsaHQbUG4Z/Jn+fnPU+176sxv8N8jlw4QuPJjfF/yJ/f\nnv7NtZ4tt5i7jS2Mr4wxjYHGBQoUeG7nzp1uhyMiIiKxtPbwWh7/7nGKZClChpQZYl1fhpQZmNJy\nCpnTZPZAdLFXd2Jd9p7Zy/Ze2xP1ql73w1pL51md+XHjj/z29G88lf+pe5Yv9U0pwm04G3tuTFAL\nAMzYMoNW01rRtXRXxjYZGy9/9r/t+o3W01qTIWUGroReIWPKjCzrtoycGXK6HZrHGGPWW2vL3atc\nguwxstbOBeaWK1fuObdjERERkdgbHzSeVMlSsfqZ1Ynyr9TNizSn5/yebDm5hWIPF3M7HFeNWj+K\niRsn8l6N9+6ZFIGzH89rVV7j6ZlPM3/HfBoXbhwHUcbehmMb6DyrMxVzVeSbht/Ey6QIoG6Buizr\ntoyGkxqS0jclizsvTlRJ0YNIkD1GkcqVK2cDAwPdDkNERERiISQ0hByf5aBegXpMajnJ7XC84siF\nI+QcmpOmhZsyvP5wcmfM7XZIAFGrksXVS3vgkUCqjK1CTf+azO8w/757f0LDQyn4ZUFypMvBiu4r\n4m2SEenkpZOU/7Y8oeGhrHtuHTnS53A7pHu6EHKBMBvGQ6kecjsUj7vfHqOE0xcpIiIiidLcHXM5\nc/UMXUt3dTsUr3kk/SO8We1NFuxcQIHhBfjP3P+w98xe1+IJCQ3h+3++p8TXJcjyaRa+CfzG66vm\nBV8OptXUVmRPl52JzSc+0JC4ZD7J6FepH6sOrWL5geVejDL2roVdo9W0Vhy/dJxZ7WYliKQIIH3K\n9IkyKXoQSoxERETEVeM3jCdn+pzU8q/ldihe9UHND9j10i6eK/sc4zeMp+CXBek2uxs7g+NuvnTw\n5WAGLR1E3i/y0n1Od3x9fCn+cHF6zu9J1bFV+ff4v15pN9yG02lmJ45ePMr01tNjNPerW5luZE2T\nlcErBnshQs+w1vLighdZun8pY5uMpdwj9+ykkHhEiZGIiIi45vjF4/yy8xeeLvl0nG946YbcGXMz\nsuFI9ry0h14VejFl0xSKjCzC0z8/zdaTW73W7u7Tu+m1oBe5h+XmzT/fpHT20izqtIigHkEs6bKE\nCc0msPP0TsqMKsNri17z+IaqHy37iF92/cKwusMon7N8jOpIkzwNvR/vzYKdC9h4fKNH4/OUrwO/\nZvTfo3m96uu0L9He7XDkAWmOkYiIiLhm6KqhvLLwFba8sIWArAFuhxPnjl08xmcrP+OrwK+4cv0K\nbYq14c3qb1L84eIeqX/lwZUMWTmEWdtmkdw3OR1LdOTlSi/ftv7gy8G8tvg1xvwzhrwP5WVkg5E0\nKNgg1jEs3rOYOj/UoUOJDvzQ/IdYzQ86c+UMuYflpmnhpkxsMTHWsXnSH3v/oM4PdWhQsAGz2s1K\nUKvnJXb3O8dIiZGIiIi4ptQ3pUiVLBVrnl3jdiiuOnnpJJ+v/pwv137JxWsXaRHQgreqv0Xp7KUf\nuK6w8DBmbpvJZ6s+Y/Wh1WRKlYme5XrSq0Kv+5rvsmz/MnrM68HWU1tpXbQ1w+oN45H0j8Tktjh0\n/hBlRpUhW9psrHl2DWlTpI1RPdH1W9iPYauHsfPFnfhn8o91fZ6w58weyn9bnuzpsrPqmVUeWXJe\nPEeLL4iIiEi8FnQsiI3HN9KlVBe3Q3Fd1rRZ+ajWR+zvs5+3q7/N73t+p8yoMjSZ3IR1h9fdVx0X\nr13kyzVfUvDLgrSe1poTl04wov4IDvY9yKBag+57EYBqeaoR9HwQHz75IXO2zyFgZAAj144kLDzs\nge7pWtg12kxrw9XQq8xoM8MjSRFA34p98TE+fLbqM4/UF1sXQi7QdEpTrLXMbjdbSVECpsRIRERE\nXDEuaBwpfFPQrng7t0OJN/xS+/Hek++xr88+PnjyA5YfWE6F7ypQ/8f6rDq46rbXHLlwhNcXv86j\nnz/KS7++RI70OZjRZgY7eu3gvxX+G6OEJIVvCgZWH8imFzbxeM7H6fVLLyqPrUzQsaD7rqP/wv6s\nOrSKsU3GUjhL4QeO4U5yZshJ51KdGfPPGE5cOuGxemMiclGJrSe3Mq31NAr4FXA1HokdJUYiIiIS\n566HXWfSv5NoUrgJfqn93A4n3nko1UO8Wf1N9vfZz+Bagwk8EkjlsZWpPaE2S/cvBWDj8Y10ndWV\nvMPy8snKT6jlX4uV3VeyovsKWgS08MhiFgX8CvDb078xqcUk9p3dR7nR5ei3sB8Xr12863U/bfqJ\n4WuH0/vx3rQu1jrWcdysf+X+hISGMHzNcI/X/SDe+fMdZm+fzed1P6dWvsS9qmJSoDlGIiIiEufm\nbJ9D0ylNmdt+Lo0KNXI7nHjv0rVLjFo/ik9WfMLxS8cp4FeAXad3kSZ5GrqX7k6fin3I75ffqzGc\nuXKGAYsHMPrv0eTOmJsR9UfQuHDjW8ptO7WN8t+Wp8TDJVjSdQkpfFN4JZ5WU1vx+97fOdDnAOlT\npvdKG3fz06afaDejHc+WeZbRjUfH+01nkzLNMRIREZFbnLp8ihcXvEjAyAAOnz/sWhzjgsbxcNqH\nqZu/rmsxJCRpU6Tl5Uovs7f3Xr6o9wU50uXgo5ofcbDvQb5s8KXXkyKATKkzMarxKJZ3W06GlBlo\nMqUJLae25ND5Q1FlLl67SMupLUmdLDVTW0/1WlIE8FqV1zh79Syj14/2Wht38vfRv+k2uxtVc1dl\nZMORSooSCfUYiYiIJAGRw44GLRvEhWsXMBh6luvJlw2+jPNYgi8Hk+OzHPSq0IuhdYfGefsSe9fD\nrjN01VDe++s9fH18GVRzEP8t/186z+rMlE1TWPj0wjgZWlZrQi22ndrGnpf2kDJZSq+3B87eW+W/\ndfZiCvxPIA+nfThO2pWYU4+RiIiIYK1l6uapBIwM4NXFr1L50cpsfH4j3Up3Y/Tfo13pNZq8aTLX\nw6/TtXTXOG9bPCO5b3Jeq/oam17YRNXcVen9a28KfFmASf9O4v0a78fZfJsBVQZw5MIRJm6Mmz2N\nQkJDaDG1Bacun2J2u9lKihIZJUYiIiKJ1KqDq6gytgptp7clXYp0LHx6IQs6LqDYw8V4o9obhNtw\nBi8fHOdxjd8wntLZS1MyW8k4b1s8K1+mfCzosICfWv1ESGgITQs35fVqr8dZ+7Xz1aZsjrJ8svKT\nB15O/EFZa3lh/gusPLiS8c3GUyZHGa+2J3EvQSZGxpjGxpjR586dczsUERGReGfvmb20nd6WymMr\ns/fsXr5r/B3/9PiHp/I/FVXGP5M/XUt15du/v43TXqPNJzYTeCRQexclIsYY2hRrw8G+B/m57c/4\nmLh7vTTGMKDKAHYE72DWtllea8day6crP2Vs0Fjeqv6WV1baE/clyMTIWjvXWvufjBkzuh2KiIhI\nvHH26lleXfQqRUYWYe72ubxV/S12vriTZ8o+c9ulm9+o9gZhNoyPV3wcZzGO3zCeZD7J6FCiQ5y1\nKXHD18c3TpOiSC0CWlDArwAfr/gYb8ydPx9yng4/d+C1xa/Rqmgr3q3xrsfbkPghQSZGIiIi8j/X\nw64zYu0ICgwvwJCVQ2hfvD07XtzB+0++T7oU6e54nX8mf7qU6sLo9aM5cuGI1+MMDQ9l4saJNCjY\nQHMzxGN8fXzpX7k/646s4899f3q07sAjgZQdVZZpm6cxqOYgprSc4kryJ3FDP1kREREvmrl1JgMW\nD2DChgkEHgnk0rVLHqvbWsuc7XMo8XUJXvzlRUpmK8n6/6xnXLNx5MqQ677qiOo1Wu79XqNFuxdx\n9OJRDaMTj+tcqjPZ02X32Jw5ay3DVg+j8pjKXAu7xpKuS3ij2hse2TRX4q9kbgcgIiKSWO0+vZsO\nP3fgaujVG47nfSgvxbIWo2jWolH/DcgacNfenZv9c/QfXln4Cn/u+5PCmQszp90cGhVq9MD7qeTL\nlI/OJTszav0oXqv6Go+kf+SBrn8Q4zeMxy+1Hw0LNvRaG5I0pUqWir4V+/La4tdYf2Q9jz3yWIzr\nOnX5FN1md2Pejnk0LdyUsU3H4pfaz4PRSnylfYxERES8wFpL/R/rs/LgSv7t+S9XQ6+y+eRmtpzc\nEvXfbae2cS3sWtQ1eTLmuSFZKvZwMQKyBJA+ZfqoMofPH2bgHwOZsGECfqn9eK/Ge/znsf+Q3Dd5\njGPdc2YPhb4sxH/L/5cv6n8Rq/u+k7NXz5J9SHaeK/ucK3snSeJ3PuQ8uT/PTZ38dZjaemqM6li6\nfykdZnTg5OWTDHlqCL0q9NLmrYnA/e5jpB4jERERL5i6eSq/7f6NL+p9QZ6H8gBQOEthWgS0iCoT\nGh7KnjN72HzixoTpj71/EBIWElUud8bcFM1alBzpcjBl0xTCbBj9KvfjjWpv8FCqh2Ida75M+ehS\nqotXe42mbp5KSFgIXUprGJ14R4aUGXih/AsMXj6YncE7KZi54H1fGxYexqBlg3jvr/fInyk/q55Z\nRdkcZb0YrcRH6jESERHxsLNXzxIwMoCc6XOy5tk1DzwvITQ8lL1n9t7Sw7T79G4aFGzA/9X6P/wz\n+Xs05sheo14VejGs3jCP1g1QeUxlzoWcY1PPTfoLvHjN8YvHyTMsj5PoNx51X9ccuXCEjj93ZMm+\nJTxd8mm+avDVDb20kvCpx0hERMQlA38fyIlLJ5jXfl6MJmsn80lGwcwFKZi5IM2KNPNChLfKlykf\nnUtFzDWq8ho50ufwWN07gnew6tAqPq79sZIi8aps6bLRrXQ3xgaN5d0a797z9/iXnb/QeVZnLl+/\nzPdNv6dLqS76HU3CtCqdiIiIB609vJavA7+mV/lesZoA7oaB1QZyPey6x/c1mrBhAj7Gh6dLPu3R\nekVup1/lfoSGhzJs9Z17Pq+FXaP/wv40mNSAHOlyEPhcIF1Ld1VSlMQpMRIREfGQ0PBQeszrQY70\nOfig5gduh/PA8vvlj+o1OnrhqEfqDLfhTNgwgTr563h1xTuRSPn98tOmWBu+Dvyas1fP3nJ+z5k9\nVPu+GkNWDaFnuZ6seXYNAVkDXIhU4hslRiIiIh7y5ZovCToWxPB6w8mQMoPb4cRIZK/RJys+8Uh9\nf+79k4PnD2rvIolTr1V5jQvXLvD1uq9vOD5t8zTKjCrD9lPbmd56Ol81/IrUyVO7FKXEN0qMRERE\nPODguYO89edbNCzY8IaV5xKa/H756VSqE9+s/8YjvUbjN4wnY8qMNC3c1APRidyf0tlLU69APYat\nGcaV61e4cv0Kz897njbT2xCQJYCg54NoWbSl22FKPKPESERExANe+vUlwm04IxqMSPDzFCJ7jT5d\n+Wms6rkQcoEZW2fQtlhb/VVe4tyAKgM4cekEb//5No9/93jUwiLLui0j70N53Q5P4iElRiIiIrE0\nZ/scZm2bxTtPvJMoXrgK+BXg6ZJP83Xg1xy7eCzG9UzfMp3L1y9r7yJxRfU81Xk85+MMWTWEYxeP\n8WvHXxlce3CsNkOWxE2JkYiISCxcvHaRXgt6Ufzh4rxc6WW3w/GYN6u/Geu5RuM3jKegX0Eq5ark\nwchE7o8xhuH1h/NsmWfZ8PwG6hao63ZIEs8pMRIREYmFd5e8y8HzBxnVaFSi+kt0bHuN9p7Zy1/7\n/6Jzqc4JfmihJFwVclbg2ybfenRfLkm8lBiJiIjE0IZjGxi2ehjPlX2Oyo9WdjscjxtYbSDXwq7x\n6YoHn2v0w8YfMBg6l+rshchERDxPiZGIiEgMhIWH0WNeD/xS+zG49mC3w/GKgpkLxqjXyFrL+A3j\nedL/SXJnzO3FCEVEPEeJkYiISAyMXj+aNYfXMLTuUPxS+7kdjte8We1NQsJCHqjXaPmB5ew5s0d7\nF4lIgqLESERE5AEdu3iM139/nZr+NelYoqPb4XhV9F6j4xeP39c14zeMJ12KdLQM0D4xIpJwKDES\nERF5QC//9jJXQq/wVYOvksTCAlG9Rvexr9Hl65eZunkqrYq2Im2KtHEQnYiIZygxEhEReQALdy9k\n8qbJvF71dQpnKex2OHGiYOaCdCzRka/WfXXPXqOZW2dy4doFDaMTkQQnQSZGxpjGxpjR586dczsU\nERFJQq5cv8IL81+goF9BBlQd4HY4cerN6k6v0ZCVQ+5abvyG8eTJmIfqearHUWQiIp6RIBMja+1c\na+1/MmbM6HYoIiKShHy07CN2n9nNN42+IVWyVG6HE6cKZS5EhxIdGLluJCcunbhtmUPnD7F4z2K6\nlOqCj0mQrxgikoTpXy0REZH7sPXkVj5e8TFPl3yamv413Q7HFfdaoe6HDT9gsdq7SEQSJCVGIiIi\n92Ctpef8nqRLkY7P6nzmdjiuKZylMB1KdOCrwK9u6TWK3Luoau6q5PfL71KEIiIxp8RIRETkHsZv\nGM9f+//i49of83Dah90Ox1VvVnuTq6FXb5lrtPbwWrYHb9eiCyKSYCkxEhERuYtTl0/Rb2E/Kj9a\nmWfKPuN2OK4rnKUw7Yu3v2Wu0bigcaROlprWRVu7GJ2ISMwpMRIREbmLVxe9yrmQc3zT8BstKBDh\nzeo39hpdDb3KlM1TaB7QnIyptDCSiCRM+hdeRETkDpbuX8r3Qd/zcsWXKZGthNvhxBtFshSJ6jU6\neekkc7fP5ezVsxpGJyIJmhIjERGR27gWdo3n5z1Pnox5ePuJt90OJ955s/qbXLl+hSErhzB+w3hy\nps9JLf9aboclIhJjydwOQEREJD4asnIIW09tZV77eaRNkdbtcOKdIlmK0L5Ee0asG0FIaAj9K/fH\n18fX7bBERGJMPUYiIiI32RG8gw+WfkDLgJY0LNTQ7XDirbeqv8WV61cIs2F0Ka1hdCKSsKnHSERE\nJJqgY0HU/7E+qZOl5ot6X7gdTrxWJEsRni/3PAfOHaBIliJuhyMiEitKjERERCIs2r2IllNb8lCq\nh1jcbTE5M+R0O6R476uGX7kdgoiIR2gonYiICPDDhh9oMKkBeR/Ky6pnVlHs4WJuhyQiInFIiZGI\niCRp1loGLx9M51mdqZa7Gsu6LVNPkYhIEqShdCIikmSFhYfR+9fejFw3kvbF2/N90+9JmSyl22GJ\niIgL1GMkIiJJ0pXrV2g9rTUj142kf+X+TGwxUUmRiCQYYWEwbBjUqQNHjrgdTeKgHiMREUlygi8H\n02RKE1YdXMWwusPoXbG32yGJiNy3f/+FZ5+FtWvBxwcaNYKlSyFdOrcjS9jUYyQiIknK/rP7qfp9\nVdYfWc/U1lOVFIlIghESAm+9BWXLwp49MGkSzJ0LGzdC27YQGup2hAmbEiMREUkygo4FUWlMJY5d\nPMbCTgtpVbSV2yGJiNyXFSugdGn48ENo3x62bnX+26ABjBwJCxZAr15grduRJlxKjERExFXvLXmP\nLrO6sGj3IsLCw7zWzuI9i6n+fXV8fXxZ3m051fNU91pbIiKecv48/Pe/ULUqXLkCv/wCEyZAliz/\nK9OjBwwYAKNGwSefuBdrpPBwtyOIGSVGIiLimoPnDvL+0veZuHEidSbWIe8XeXl98etsO7XNo+1M\n3DiR+j/W1x5FIpKgzJ8PxYrB119D796waRPUq3f7soMGQbt2ToI0ZUrcxhnd8eNQpQrMnu1eDDGl\nxEhERFzz3d/fYa1l8wubmdpqKqWyleLTlZ8SMDKACt9WYOTakQRfDo5x/dZaPl7+MZ1mdoraoyhX\nhlwevAMREc87ccIZJteoEWTMCCtXOivQ3W1xBR8fGDcOqlWDLl1g2bI4CzfK9u1QqRJs2ADGxH37\nsWVsAh6IWK5cORsYGOh2GCIiEgPXw66TZ1geyuQow/wO86OOH794nEn/TmL8hvFsOL6B5D7JaVSo\nEV1KdaF+wfqk8E1xX/WHhYfR59c+jFg3gnbF2zGu6Tgtxy0i8Zq1MHEi9OkDFy7Am286PUAp7u+f\nPQBOn4bKlZ3katUqKFzYe/FGt3w5NG0Kvr4wbx5UqBA37d4PY8x6a225e5VTj5GIiLhizvY5HL14\nlOcfe/6G49nSZaNvpb4EPR9EUI8gXqzwIisPrqTZT83IOTQnvX/pzfoj67nbH/Yi9ygasW4E/Sr1\n48cWPyopEpF4bf9+qF8fOnd2kpmgIHj77QdLigD8/JyFGJIlc+o7ftw78UY3bRrUru3Me1q9On4l\nRQ9CPUYiIuKK2hNqs/P0Tva8tAdfH9+7lg0ND+W3Xb8xfsN4Zm+fzbWwaxTLWowupbrQsWRHHkn/\nSFTZ01dO02RyE1YeXMnndT/XctwiEq+FhcGIETBwoPP5//4PXnjB6XmJjbVroUYNKFEC/vwT0qSJ\ndai3sBY++wz69//fvKLMmT3fTmzdb4+REiMREYlzO4J3UHhEYT588kMGVh/4QNeeuXKGqZunMn7D\neFYdWoWP8eGpfE/RpVQXSmcvTYupLdhzZg8Tm0+kdbHWXroDEZHY27wZnnkG1qxxene++QZy5/Zc\n/bNnQ/Pm0KQJzJgR+2QrurAwZ8jfiBHQurWzUl6qVJ6r35M0lE5EROKtUYGjSOaTjGfKPvPA12ZK\nnYke5Xqw8pmV7Oi1gzeqvsHWU1vp8HMHin5V1Nmj6OmFSook1kJCoHt3yJcPnnsOZs2CixfdjkoS\ng5AQeOcdKFMGdu1y5hXNn+/ZpAicOT9ffOEkSC+/7Ll6L1+Gli2dpOiVV5xV8OJrUvQg1GMkIiJx\n6sr1K+QcmpOn8j/FT61+8kid4TacpfuX8vue32lfoj1Fsxb1SL2SdJ0+7fylfelSZ+7E2rXOfjIp\nUsATT0DDhs6KYfnzux2pJDQ7dzq/W5s3Q8eO8PnnkDWrd9t8+WWnnc8/d3p5YuPECWjcGNatc5Ku\nF1/0TIzedL89RsniIhgREZFIUzdP5czVM7csuhAbPsaHGnlrUCNvDY/VKUnXvn3QoAHs3g2TJjnL\nJl+75qy6NX++89Wnj/NVuLCTJDVs6GzA+aAT5SVpWbIEWrRwhrTNn+/8nsWFIUOcxR1efhkefdTp\n7YmJHTucIX9Hj8LPP0OzZp6N023qMRIRkThV8buKnAs5x5YXtmAS4kYXkqitX+/0BF296gyde+KJ\n25fbtet/SdJffzmJU4YMUKeOkyTVrw/ZssVt7BK/ffcd9OwJhQrB3LnOEM24dOUK1KzprHb3xx/O\nfkMPYsUKZ66Sr68T/+OPeydOb9AcIxERiXf+OfoPaw6v4fnHnldSJPHOggVOIpQypbOh5p2SIoAC\nBaB3b1i4EIKDYeZMaNPGeXns1g1y5HBeHN9/30m2wsPj7j4kfgkLg379nHlqtWo5v1txnRQBpE4N\nc+ZAzpxOgrNr1/1fO326E3vmzM7eSAkpKXoQSoxERCTOfBP4DamTpaZL6S5uhyJyg9GjnZfFwoWd\nF7+AgPu/Nl06Z0jRt9/C4cNOIvTee2AMvPsulCvnvIw+8wxs2eK1W5B46MIF53fjs8+cuTjz5kHG\njO7FkzWr8weA8HBnGN+pU3cvby0MHeok/WXLOkldYp5Xp8RIRETixPmQ8/z474+0L96eh1I95HY4\nIoDz4jdwIPTo4QyD++svp7cnpoxxXiDfesvZ6PLYMRg3DqpVczbBrFHDmachid+BA868s19+gZEj\nYfhwZ9NVtxUq5PQcHTjgJG1Xr96+XORy3K+84iwW8fvvzgauiZkSIxERiRM/bPiBS9cv8Xw5zy26\nIBIb165Bp07w0Ufw7LPOy2K6dJ5t4+GHoUsXmDoVAgOdRKxOHadnSW7PWmdI4okTbkcSc6tXQ4UK\nzoIHCxY4G7bGJ1WqwPq6C00AACAASURBVA8/OM+5c+dbh3pevuzsTTR8OPTt6/z+pk7tTqxxSYmR\niIh4nbWWrwO/5rEcj1E+Z3m3wxHh7FmoVw9+/BEGDXKG0nn7r/mFCsGvvzpzkurWdZYEl1t9+qnT\n05I9u7NAwKBBsHGjkzAlBFOmOD2DadM6wzLr1HE7ottr3dp51tOmwWuv/e/4yZPOIg2zZsGwYc5Q\nOk9uDBufxavEyBjTzBjzrTFmtjEmnv4aiYjIg1pxcAWbT26mZ7mebociwsGDzov38uXOX83feMMZ\nAhcXHnvM2Wxz505n9btLl+Km3YRi+nTnJb1FC2eeVlgYvPkmlCoFefI4PS+//HLn4V9ustaZU9a+\nvdNbtGbNg81Vc8MrrzjPdMgQ+Oor5/eyUiXYsMH5WfTu7XaEccvry3UbY8YCjYAT1tri0Y7XA74A\nfIHvrLWDo53LBAyx1t51S3Qt1y0ikjB0/Lkj83fM5/DLh0mbIq3b4UgSFhTkLKd98aKzklzNmu7E\nMWOGM6G9bl0nUUqe3J04/vjD2by2f3/3ewXWrHF6WsqUceJKlco5fuyYMxxt7lxYtMhJJtP8P3t3\nHmdj/f5x/PWZYexrIZGQIpRtRDEhKi2WEhGylC0pSUV9S0QqEllCZOyRpZBQGfs6iFB2UrZsYzfb\n/fvj0/xQljMz58x9zsz7+Xicx8ycOfd9XzNpzrnO5/pcV2Z4+GGbXD7xRPL2hXnD+fO2G+GUKdCy\nJQwfbrsbBoLYWLuHaO5c2xgiKMj+rhPbztuf+VO77nCg9uV3GGOCgaHAY0BJoIkx5vIx5f/75/si\nIhLg/j77N9O2TuP5Ms8rKRJXzZ9vmyAEBdm9FW4lRWAHbA4fblc/WrZM+XbejgODBtnkont3u8fK\nzZbie/faroC33moTxYSkCGxJXevWNpE9etT+zlq1gg0bbAvsW2+1nf969rQdAVO65O7QIZvQTZ0K\nH38MX30VOEkR2BLSr7+GihVtc4WVK1NXUpQYPk+MHMdZAvy7ivY+YKfjOLsdx4kGvgbqGetj4AfH\ncdb7OjYREfG9rzZ8RXRctJouiKu++squLBQtajfGly5942N8rU0bu39m0iS7wT2lXtBHR0Pbtrbj\nWN26tpQwPBzat3cnOTp50v63iY62A3Pz5Ln2YzNmtHvDhgyxydSmTbZ5RkiITYxCQ6FgQfvzzZrl\n+1LFjRtt2dzmzTBjBrz5ZsqVZXpTliz2zYKtW+HOO92Oxj1uNQ0sAOy/7Os/gUpAJ6AWkMMYU8xx\nnOH/PtAY0xZoC1CoUKEUCFVERJIq3olnxLoRVLu9GiXzlLzxASJe5jj2BXPPnnZ1ZNo0yJ7d7agu\n6d7dbnYfONAmBP/7n2+vd/SoXa1assS2Ke/V69IL+Q8/tCV9Q4ak3Iv7mBh45hnbwnzBAihRwvNj\njYF77rG3hN/jDz/YMrCvv7ZzpTJmhAcftKsh5cvb2+23e+fnmz3b7ifKmdPuVytXLvnndJPbpZT+\nwK3E6Gr/HB3HcT4HPr/egY7jjARGgt1j5IPYRETESxbsWsCek3voW7Ov26FIGhQTY1cOwsNt6dWI\nEe7t5bkWY+zwz6NH7eyjm2+2Kze+sHmzXSE6cMB243vuuUvf693brtj0729/R5995vvkyHGgQwc7\nH2fMGKhRI3nny5PHtp5+/nn7syxdapOXhQvho49sIweAXLkuJUkJt2LFbImlp3F/+qldHUpopnHr\nrcmLXfyDW4nRn8Btl31dEDjgUiwiImlGvBNPkEm5hqRfRH5B3ix5eerup1LsmiIAp07ZlYgff7Sd\nwt57z39LnIKCbKnfiRO2Q9hNN9lWyt40Z45d3cia1a4W3Xffld83Bj75xCaTgwbZ0rSPP/bt7+yT\nT2D0aLty1bKld88dEgI1a9ob2C52v/4K69dfug0aZBMosL+XcuWuTJZKlPhvC/foaPvfaPRo+98o\nPNw2gpDUwa3EaC1wpzGmCPAX0Bh47vqHiIhIcqz9ay21J9bm00c+pWXZlj6/3h9RfzBn+xzeqvIW\nIcEhPr+eSIJDh2y3t61b7UqEt190+0L69Hbz/iOPQNOmdlWjVq3kn9dx7CrQW2/ZF/7ffWf34FyN\nMXalKCbGzrdJn96uJPkiOfrmG+jWDRo3tuV8vpYxoy2nq3jZGLXoaPtv5PJkaeRI22EO7EDTMmUu\nJUolS9qSvcWL7ere++97vsokgSEl2nVPBqoDNwOHgR6O44w2xjwODMS26/7KcZw+iT232nWLiHgm\n3omn8qjKrD2wlpDgEBa1WMT9t/m27dC7C9+lz9I+7H51N4VzFvbptUQSHDliO4T98YfdDO+vwzWv\n5cQJqFYNdu+GiIgrX8gn1sWL0K4djB2buNWN+Hhbzvfll/bFf48eSY/halatsmVz5cvbMrrLO9C5\nLS4Otm27Mllavx5On7bfDwmxq3tNm7obpySOp+26fZ4Y+ZISIxERz4zZMIbWs1oz8NGBDF4zmDPR\nZ4hsG0nB7Nd46ziZYuJiKDSwEBXyV2DOc3N8cg2Rfzt61Lbg3rnTbsKvVs3tiJLmwAGoUsW+GF+2\nLHENCRIcPmxn06xcmbRSwvh4eOEFm0z16WM713nDnj1QqRJky2YTpOt1oPMX8fE2Ud2wwa4alSrl\ndkSSWP40x8jrjDF1jDEjo6Ki3A5FRMTvRV2IotvP3bi/4P10qtSJWU1mcTbmLE9NeYrzMed9cs3v\ntn3HoTOH6BDawSfnl+RxHJtEpCbHj9uuczt22P00gZoUgd3I/+OPtkvYI4/A/v03PuZyCS2kf/nF\nluf16JH4crigIBg1yq6MvPOOLcdLroS23LGxdphoICRFYH8XxYrZVTclRalbQCZGjuPMdhynbY4c\nOdwORUTE7/Vc3JO/z/7N4McGE2SCKJmnJBOfnsi6A+toM7sNvqgc+CLyC27PcTu1i9W+8YMlxY0c\nCfnz23fAU4OTJ20CsXWr3UPj5uBWbylWDObNg6gou1/q2DHPjps5Ex54wJaELV2avCYOwcF2xejZ\nZ+GNN2yzgqSKjrZtwnfutCWOxYsn/VwivhKQiZGIiHhm699bGbxmMC+Wf5EKt1b4//vrFq/LBzU+\nYOKvE/l05adevea2o9tYuGchbSu0JThIgzH8zdmzdgUhNtaWVwW6hMRh0yabFATanqLrKVfODind\nvRsefxzOnLn2Yx3HziF6+mk7vHbtWttKOrnSpYPx4+15O3eGYcMSf46EttwLF9p9S9WrJz8uEV9Q\nYiQikko5jsOr814la0hW+jz03/42b4e9TcOSDXnrp7eYt3Oe1647PHI46YPS80K5F7x2TvGewYPt\n/pOnn7YlZ6tWuR1R0p0+bROG9evt4NbHH3c7Iu+rVs0OK42MtCsuCe2lL3f+/KWSt+eeg0WL7Iqg\nt6RPD5MnQ5060LGjTW4S4+OPbcOC//0PWrTwXlwi3qbESEQklfr292/5afdP9KreizxZ/lvMb4xh\nTL0x3JvvXhpPa8y2o9uSfc3zMecJ3xjO03c/Tb6s+ZJ9PvGuEyfsi9Qnn7SdyvLmtS9WA9HZs3a/\nyurVNnGoW9ftiHynfn2bjCxYYIeXJgwqBduooVo1m7h8+CFMmGDbTHtbSIhtsf3447bTXXi4Z8dN\nnWpbXDdpkjJtuUWSQ4mRiEgqdD7mPF0WdKF03tJ0qHjtBghZQrLw7bPfkj44PfW+rkfUheQ1tZmy\nZQonL5xU0wU/1b+/3Y/Tp48daNm9u22XHBHhdmSJc+6cXb1YvhwmTrQrKald69Y2qZ0yBV55xZan\nrVtnmyxs3WrLCLt39+1A1gwZYPp0O1+pdWubhF3PypU2katSxa4Y+euAXZEEAZkYqSudiMj19VvR\nj70n9/J57c9JF3T9Wd6357yd6Y2ms+vELp6b8Rxx8XHXffz1fBH5BXfffDcP3v5gks8hvnHoEAwc\naN+5v/dee1/79lCggB1WGSjTOy5csCsoixbBuHG2MUBa8eab0LWr3efz7LMQFmYbJCxfbn8nKSFj\nRvj2W7tPqEULm6hdze7dUK+eHSY7c6Z/zSoSuZaATIzUlU5E5Nr2ndxH32V9aViyITWK1PDomAdv\nf5DBjw1m7o65/G9h0mqr1h9cz5q/1tA+tD1Gbw37nT597P6Uy8uZMma0pXTLl9sOaP7u4kU7m+en\nn9LukM1PPoGWLW1ZW7lysGYNlCmTsjFkzgyzZ9uVoKZNbZe5y504cakt9/ffB05bbpGATIxEROTa\nuv7YFYOh/yOJGzzSPrQ97Su056PlHzH518mJvu7wyOFkTp+Z58s8n+hjxbf27oURI+zAzmLFrvxe\n69ZQpIhNkPx51Sg6Gp55xiZwI0fa5CAtMsbuN/rhB9vlLZ9LW/myZLFJT6VKdvVq1ix7f8J/p127\n7EqR2nJLIFFiJCKSiizcs5BpW6fRvWp3CuUolOjjBz02iLBCYbSe1Zp1B9Z5fFzUhSgm/jqRJqWb\nkDNjzkRfV3zr/fdtydW77/73eyEhtn33+vW2RMofxcRA48a2i96wYfDii25H5K506aB2bbvnx03Z\nstlBreXL22Ro7lxbnrlwoR0OG8hDdiVtMr4Y7JdSQkNDncjISLfDEBHxCzFxMZQbUY5zMefY2nEr\nGdMlraj/yNkjVPyyIvFOPJFtIj3qLjdkzRA6/dCJyDaRV8xLEvdt3Qr33ANdukC/fld/TGysnX2T\nLh1s3GiTKH8RG2tbUH/zDXz+OXTq5HZE8m8nT0LNmvDLLxAfbxNwdaATf2KMWec4TuiNHqcVIxGR\nVOKLyC/Y8vcWBjw6IMlJEUDeLHn5rvF3HDt3jAZTG3Ax9uJ1H+84Dl9EfkHoraFKivzQu+/asqe3\n3rr2Y9Klsy9kt2y59mZ6N8TF2Q3+33wDn36qpMhf5cwJP/4IDzwAbdpAz55uRySSNEqMRERSgb/P\n/s17Ee/xcNGHqVe8XrLPV/aWsoTXD2f5/uW8PPdlrlddsPSPpWz9e6tadPuhtWvtxviuXeHmm6//\n2Geesd3qevSwqzRui4uz+58mTYK+fe2Kl/iv3Llh6VK7/0u9VyRQKTESEUkF3v75bc7GnGVQ7UFe\n6wjXqFQj3gl7h1EbRjFs7bBrPm545HByZsxJ49KNvXJd8Z533rEJ0Wuv3fixQUHwwQewc6cd/uqm\n+Hho29a24+7VC7p1czceEUkbAjIx0hwjEZFLIg9EMnrDaF657xXuznO3V8/dq0Yv6txVh1fnvUrE\nnv9OAT1y9gjTtk6jRZkWZE6f2avXluSJiLDlTW+/bTfJe6JOHTswtFcv2xrbDY4DL71k23G/++7V\nG0aIiPhCQCZGmmMkImLFO/F0+qETebPkpUf1Hl4/f5AJYsLTEyh+c3EaftOQPSf2XPH9rzZ8RUx8\nDO0qtPP6tSXpHAe6d7fDNTskosLRGOjdG/74w3YVS2mOA6+8YluLd+umvSoikrICMjESERFrwqYJ\nrPpzFR/V+ojsGbL75BrZM2Tnu8bfEefEUe/repyJPgNAXHwcI9aNoHrh6l5fqZLkmT0bVq+2+4Uy\nJrIPR61a8OCDNkE6d8438V2N49i9UEOGwOuvw4cfaq+KiKQsJUYiIgHq1MVTvPnjm1QqUMnnQ1WL\n5S7GlGemsOXvLbT4tgXxTjzzd81n78m9arrgZ+Li7N6iu+5K2hDUhFWjQ4fszKCU0qcPDBhgO8/1\n66ekSERSnhIjEZEA9cHiDzhy9giDHxtMkPH9n/NH7niE/g/3Z8ZvM+i9pDfDI4eTL0s+6peo7/Nr\ni+e+/ho2b7aNFNKlS9o5wsLg0Ufho4/g9Gnvxnc1I0favUTNm8PAgUqKRMQdSoxERALQ70d/Z+Dq\ngbQu15qKBSqm2HU7V+5MizIt6LGoB3O2z+HF8i8SEhySYteX64uOhvfeg7Jlbfvt5PjgAzh2zCYq\nvjRjht0H9fjjMHq07Y4nIuIG/fkREQkwjuPQeV5nMqfPzIc1P0zRaxtjGP7kcCoVqESQCaJN+TYp\nen25vtGjYfduuz8nuQlGxYpQvz707w/Hj3snvn9btAiaNIFKlWDqVEif3jfXERHxhBIjEZEAM3v7\nbObvmk/P6j3JmyVvil8/Y7qMLGi+gMi2kdye8/YUv75c3blzdpWnalWoXds75+zVy5bSffqpd853\nuQ0boG5dKFYM5syBLFm8fw0RkcQIyMRIc4xEJK26EHuB1+a/Rsk8JelYsaNrcWTPkJ2yt5R17fry\nX0OGwMGD0Lev9/bo3HMPPPssDBoER45455wAu3bBY49Bzpwwfz7kzu29c4uIJFVAJkaaYyQiadWn\nKz5l94ndfF77c9IHq+5IrKgo2yjh8cftipE39ewJ58/b83vDoUPwyCMQGwsLFthZSyIi/iAgEyMR\nkbRof9R+Plz2IQ3ubkDNojXdDkf8SP/+cOKEbbPtbXfdBS1a2Nbdf/2VvHNFRdkyv0OH4PvvoUQJ\n78QoIuINHiVGxphQY8xrxph+xphexphGxhgtfIuIpKA3fnyDeCee/o/0dzsU8SOHD8Nnn9mSt3Ll\nfHON996D+PjkJV4XLkC9erBli+1EV6mS9+ITEfGG6yZGxpiWxpj1QHcgE7ANOAJUBX40xow1xhTy\nfZgiImnb4r2LmbJlCt2qdKNwzsJuhyN+pG9fm3T06uW7axQuDG3awKhRsGdP4o+Pi4PnnoPFi2Hs\nWDsjSUTE39xo9FsWoIrjOOev9k1jTFngTuAPbwcmIiJWbHwsnX7oxO05bufNKm+6HY74kX374Isv\noFUrW/LmS++8A199ZROwMWM8P85x7JyimTNtE4fnnvNdjCIiyXHdFSPHcYZeKyn65/u/OI7zs/fD\nEhGRBMMjh/PrkV8Z8OgAMqXP5HY4kkg7dsCaNTZB8LaePW0Huvfe8/65/+3WW+Gll2DcOPj9d8+P\ne/dd+PJLePtteOUV38UnIpJcNyqleyphL5ExJo8xZpwx5ldjzBRjjPrIiIj40LmYc7z141t0nteZ\nWkVr8VSJp9wOSRIpOhoeftjupylXDkaOhDNnvHPu33+3ZWkvvQS33eadc95It26QKRO8/75nj//8\nc+jTB1580TeNIUREvOlGzRf6OI6TMO96CLABeAz4AUjEQrqIiCTGgl0LKD2sNJ+s+ISWZVsy9Zmp\nGG8Np5EUM3asLXfr3Nl+3a4dFChgV05++y155373XcicGbp3T36cnsqTx/4sU6bApk3Xf+zkyfDq\nq1C/vi330z9fEfF3N0qMgi/7vJjjOJ85jvOn4zjhQB7fhSUikjb9ffZvms9szqMTHiV9cHoWtVjE\nqLqjyJUpl9uhSSJFR9vVkvvugwEDYMMGWLEC6taFESOgZEmoWROmT4eYmMSde906mDYNunSxyUpK\nev11yJHDJmbXMn8+PP88PPigTZDS3WhHs4iIH7hRYrTon/bcmf75vD6AMaYGEOXz6ERE0gjHcRj7\ny1hKDC3BlM1TePfBd9nYfiPVCldzOzRJooTVovfft6slxsD998P48bB/v+0mt2sXPPOM7frWsycc\nOODZud95B3LntklKSsuVC954A2bNsnun/m3NGmjQAEqVso/JmDHlYxQRSQrjXGc3qDEmPfAO0Pqf\nuwoCZ4HZQDfHcVzpRmeMqQPUKVasWJsdO3a4EYKIiNfsPL6TdnPasXDPQqrcVoWRdUZSMk9Jt8OS\nZIiOhuLFIW9eWLXq2mVkcXHwww8wdCjMmwfBwfDUU3bfUPXqVz9u8WL7vX79oGtXX/4U13b6NBQt\nCuXL29WhBL//DlWrQvbssHw55M/vTnwiIpczxqxzHCf0ho+7XmL0rxPmANI5jnMsucF5S2hoqBMZ\nGel2GCIiSRITF0P/Ff3ptaQXIcEhfFzrY9pWaEuQ8Wj2tvixUaPs3J/vv4fHH/fsmF27YPhw2xL7\n+HG4+26bIDVvbkvXwHa2q1rVrkTt2GEbIbjl009tYrZ4sS2Z+/NPeOABuHjRJkXFirkXm4jI5bye\nGP1z0lyO45xIVmRepMRIRALVqj9X0WZ2GzYf2UyDuxvw+WOfc2u2W90OS7wgYbUoTx5YvTrxTQfO\nn4epU2HYMFuWliULNGtmk6Q//oA6dewepbZtfRN/YuK84w6bAH37LYSF2RLBxYttBz4REX/haWKU\n2LclNbNIRCQZTl08Rae5nXhg9AOcvHCS7xp/x7RG05QUpSLjxsHevZf2FiVWpkzQooVNqtauhUaN\n7H6lMmXg2WdtItKqlbejTlqc//sfLF0KFSvCzp3w3XdKikQkcCV2xWiD4zh+8ydPK0YiEki+/f1b\nXp77MgdOH6DTfZ3o/VBvsmXI5nZY4kXJXS26luPHITzcdnjr2dPz8jxfi46Gu+6yK1nffGObLoiI\n+BtPV4xu2EDTGPN8wqdArsu+xnGccUkPUUQkbfjr1F90+qETM3+fyb357mV6o+lUKljJ7bDEBxJW\ni4YO9e7cnty5bWvuLl28d05vCAmxneeOHoWHHnI7GhGR5PFkskCRyz7PABTGJkmeLzWJiKRB8U48\nwyOH0/3n7kTHRfNRzY/ocn8X0gendzs08YGYGDu3qGJFeOwxt6NJOffe63YEIiLeccPEyHGcngmf\nG2PqOY7Ty7chiYgENsdxWPbHMt766S1W/rmSWkVrMfyJ4dyR+w63QxMf8tVqkYiIpIzEzqLWn3oR\nkWvYd3If4zaOY+zGsew6sYubMt3EuPrjaHZvM4xeKadqMTHQu3faWy0SEUlNEpsYNfdJFCIiAeps\n9Flm/DaD8I3hLNyzEIAahWvwXrX3ePrup8kaktXlCCUlJKwWDRmi1SIRkUDlcWJkjMkExPgwFhGR\ngOA4Dkv/WEr4L+F8s/UbzkSfoWiuovSs3pPnyzxP4ZyF3Q5RUlDCalFoqP90ixMRkcTzKDEyxtQB\n+gMhQBFjTFmgl+M4dX0ZnIiIP9lzYg/jNo5j3KZx7D6xm6whWWlUshEty7akaqGqKpdLo7RaJCKS\nOng0x8gYsw54CFiUMMfIGLPJcRxXetH8k6jVKVasWJsdO3a4EYKIpBFnos8wfet0wjeGs2jvIgyG\nh4o8RMuyLXmqxFNkCcnidojiopgYO8fn5pthzRolRiIi/shrc4z+Ees4TpS/vBvqOM5sYHZoaGgb\nt2MRkdQn3olnyb4lhP8SzrSt0zgbc5ZiuYvRu0ZvmpdpTqEchdwOUfzE+PFaLRIRSS08TYw2G2Oe\nA4KNMXcCrwArfBeWiEjKuxh7kY+Xf8yYX8aw9+ResoVko0npJrQs25IHbntApXJyBe0tEhFJXTxN\njDoB7wAXgUnAfKC3r4ISEXFD1wVdGbJ2CA8XfZg+D/Whfon6ZE6f2e2wxE+NHw979sDgwVotEhFJ\nDW64x8gYEwx85DjOGykTkudCQ0OdyMhIt8MQkVRgzvY51Jlch86VOvNZ7c/cDkf8XEwMFC8ON92k\nvUUiIv7Oa3uMHMeJM8ZU8E5YIiL+5+Dpg7T6rhVl8pXho1ofuR2OBICE1aLPP1dSJCKSWnhaSrfB\nGDML+AY4m3Cn4zgzfBKViEgKiXfiafFtC85Gn2Vyg8lkSJfB7ZDEzyXsLapQAZ54wu1oRETEWzxN\njHIDx7AtuxM4gBIjEQlon638jB93/8iIJ0dwd5673Q5HAoBWi0REUieP5hj5K+0xEpHkWH9wPZVH\nVebJu55keqPp6jonNxQTAyVKQK5csHatEiMRkUDg6R6jIA9PVtAYM9MYc8QYc9gYM90YUzD5YYqI\nuONs9FmaTG9C3ix5+bLOl0qKxCMTJsDu3fD++0qKRERSG48SI2AMMAu4FSgAzP7nPhGRgNR5Xmd2\nHNvB+KfGc1Pmm9wORwKA9haJiKRunu4xyuM4zuWJULgxprMvAhIR8bVpW6cxasMoulftTo0iNdwO\nRwJEwmrR7NlaLRIRSY08XTE6aoxpZowJ/ufWDNuMQUQkoOyP2k+b2W2oeGtFelbv6XY4EiC0WiQi\nkvp5mhi1BhoBh4CDwDP/3CciEjDi4uNoNrMZsfGxTGowifTB6d0OSbzIcWDDBjh1yvvnTlgt6tFD\nq0UiIqmVR4mR4zh/OI5T13GcPI7j5HUcp77jOPt8HZyIpF7vRbxH9fDq7D6xO8Wu2XdZX5bsW8LQ\nx4dSLHexFLuu+N6WLVCzJpQvD/nzQ+vWsHKlTZaSK2G1qHx5ePLJ5J9PRET8k6dd6cYaY3Je9nUu\nY8xXvgtLRFKzhXsW8sGSD1iybwkVRlbg++3f+/yaK/ev5P1F79OkdBOa39vc59eTlHH6NHTtCmXL\nwi+/QL9+0LQpfPMNPPAA3HMPDBwIx5JR/D1xojrRiYikBZ6W0t3rOM7JhC8cxzkBlPNNSCKSmp26\neIrW37Xmztx3sqnDJgrnLMyTk5/k3YXvEhcf55NrRl2I4rkZz3Fbjtv44okv1Jo7FXAcmDQJiheH\nAQOgVSvYvt0mSSNHwoED8OWXkDUrvPYa3HorNGkCCxdCfLzn14mN1WqRiEha4WliFGSMyZXwhTEm\nN553tPM6Y0wdY8zIqKgot0IQkSTquqAr+0/tJ7x+OKXzlmZF6xW0Ltua3kt789jExzh67qjXr9lx\nbkf2R+1n0tOTyJExh9fPLylr82aoUcOuDBUoAKtW2WTo5psvPSZbNnjxRfu9jRuhfXuYP9+W2911\nF/TtCwcP3vhaEybArl1aLRIRSQs8TYw+BVYYYz4wxnwArAA+8V1Y1+c4zmzHcdrmyKEXOCKBZN7O\neXy5/ktev/91HrjtAQAypc/E6HqjGVVnFEv2LaH8iPKs/nO11645YdMEJv46kR7VenD/bfd77byS\n8k6dgi5dbNncr7/CiBE28bnvvusfd++9MGgQ/PWXTXRuuw3eftt+rF8fvv8e4q6yWKnVIhGRtMU4\nHu5MNcaUBB4CDPCz4zhbfRmYJ0JDQ53IyEi3wxARD5y8cJLSw0qTI2MO1rVdR8Z0Gf/zmPUH19Ng\nagP+OvUXA2sPlmEqKQAAIABJREFUpENoh2SVve06votyI8pR9payRLSIIDgoODk/grjEcew+nzfe\ngMOHoU0b+PBDuCkZc3l37IDRoyE83J6zQAHbsKF1ayhc2D4mPNyW6M2aBXXqeOEHERERVxhj1jmO\nE3qjx3nafOEOYJfjOEOAX4FalzdjEBG5kc7zOnPozCHC64VfNSkCKJ+/POvaruPhOx6m49yONJ/Z\nnLPRZ5N0vZi4GJrOaEqQCWLC0xOUFAWoTZugWjVo3tyu8KxebVeKkpMUAdx5J3z0EezfDzNm2FWl\n3r2haFF49FHbvEGrRSIiaYunpXTTgThjTDFgFFAEmOSzqEQkVZm1bRZjN46lW9VuVCxQ8bqPzZ0p\nN7ObzOaDGh8w6ddJVB5dme3Htif6mj0X92T1X6sZWWckhXIUSmro4pKoKOjc2SYmW7faRgqrVkHF\n6//zSbT06eGpp2DuXNi7184p+u03aNRIe4tERNIaj0rpjDHrHccpb4x5EzjvOM5gY8wGx3Fc7Uyn\nUjoR/3fs3DFKDStFvqz5WNtmLSHBIR4fu2DXAp6b/hzRcdGE1w/n6buf9ui4xXsXU2NsDVqVbcXo\neqOTGrq4wHFg/Hh48004cgTatbMrN8ldIUqMuDj48UfYuRM6dlRiJCIS6LxaSgfEGGOaAM8Dc/65\nTyPjReSGOv3QiWPnjzG2/thEJUUAj9zxCOvbrefuPHfTYGoD3ljwBrHxsdc95vj54zSb2YxiuYsx\n6LFByQldUtjGjRAWBi1a2H0+a9bAF1+kbFIEEBwMtWvDyy8rKRIRSUs8TYxaAfcDfRzH2WOMKQJM\n8F1YIpIaTN86ncmbJ/Peg+9R9paySTpHoRyFWNJyCR1CO9B/ZX9qjqvJoTOHrvpYx3FoM7sNh88c\nZnKDyWQNyZqc8CWFnDwJr7xiy+a2bYNRo2DFCgi94Xt7IiIi3uNxVzp/pFI6Ef915OwRSg0rxe05\nbmflCytJH5z8RebxG8fTbk47cmTMwdRnphJ2e9gV3x+1fhRtZreh38P96PpA12RfT3wvOhoqVIAt\nW6BDB/jgA8id2+2oREQkNfFKKZ0xZvY/w1T/84rGGFPUGNPLGNM6OYGKSOrjOA4dvu/AqYunGFt/\nrFeSIoDmZZqz+sXVZA3JSo2xNRiwcgAJb+78fvR3Xp33KrWK1qLL/V28cj3xvU8+sQNbZ8yAoUOV\nFImIiHtuVErXBggDfjfGrDXGzDXGLDTG7AZGAOscx/nK51GKSECZvHkyM36bQa/qvSiVt5RXz31P\nvnuIbBNJ3eJ1eX3B6zSa1oij547SZHoTMqfPzLj64wgynlYJi5t27LCNFRo2tINWRURE3JSYAa+F\ngfzAeWC74zjnfBeWZ1RKJ+J/Dp4+SKlhpSh+c3GWtVrms/lBjuPQf0V/uv3cjUzpMnE25iyzGs+i\nTnFN4gwEjgMPPwxr18Lvv0P+/G5HJCIiqZWnpXTpPD2h4zh7gb3JiElEUjnHcWg7py3nY88TXi/c\np0NVjTG8UeUNKhaoSLMZzWgf2l5JUQCZOBF+/hmGDVNSJCIi/sGjxMgYcxr499JSFBAJvO44zm5v\nByYigWfsxrHM2T6Hzx79jOI3F0+Ra1YvXJ39r+3HqK9ywDh2DF57DSpVsnOKRERE/IGnK0YDgAPA\nJMAAjYFbgG3AV0B1XwQnIoFjf9R+Xp33KmGFwnil0ispem0lRYHlzTdti+6RIyFI28FERMRPePqU\nVNtxnBGO45x2HOeU4zgjgccdx5kC5PJhfCISABzH4cXZLxIbH8uYemPU/ECuafFi+OoreP11uPde\nt6MRERG5xNNXL/HGmEbGmKB/bo0u+17gDkISSQMOnznMoFWDWLBrAb6aW/bl+i9ZsGsB/R7uxx25\n7/DJNSTwXbwI7dtD4cLw3ntuRyMiInIlT0vpmgKDgGH/fL0SaGaMyQS87IvARCR5fj38K5+t+oyJ\nv04kOi4agNJ5S/Na5dd47p7nyJguo1eus/fkXl5f8Do1i9SkfWh7r5xTUqdPPrEd6ObOhcyZ3Y5G\nRETkSh636/ZHatctcqV4J575O+czYNUAftr9E5nSZaJV2VZ0qNiBdQfWMWDVADYd3kTeLHl5KfQl\nOlTsQN4seZN1vZrjarLuwDo2v7SZQjkKefGnkdRk+3ZbOle/Pnz9tdvRiIhIWuJpu26PEiNjTEFg\nMFAFWzq3DHjVcZw/kxtocvhTYtS5c2d++eUXt8OQNCreiefwmcPsP7Wf8zHnCUkXQoFsBbg1262k\nC7pyYfjEhRP8eepPjp87jjGGfFnzUTB7QbKkz5Lo6/516i92Ht/JXTffRf6s6rks17ZxI5w+Dffd\nByEhbkcjIiK+VrZsWQYOHOh2GID35xiNwXaka/jP183+ue/hpIUnIt4QHRfNX6f/4sDpA8TGxZI1\nQ1ZK5ClB3sx5r9mpLVfGXOTKmItzMef489SfHD5zmEOnD5ErUy4KZi9I7ky5Pbr2+djz7D6xm9yZ\ncispkus6fNh2obvzTiVFIiLivzxdMfrFcZyyN7ovpfnTipFIStp4aCOfrfqMSb9OIjY+lrrF6/Ja\n5dd48PYHE926+ti5Y4xYN4Iha4Zw8MxBSuYpyWuVX6PpPU3JlD7TVY+Ji4+jWng1tvy9hc0dNlMg\newFv/FiSCh09CiVKwF13wbJlas8tIiIpz9MVI0+foo4aY5oZY4L/uTUDjiUvRBFJjHgnnu+3f0/N\ncTUpO6Is32z9hnYV2rHt5W182/hbqhWulqR5Pjdlvom3w95mb+e9jKs/jpDgENrMbkOhgYXoEdGD\nw2cO/+eYgasGsnz/cj6v/bmSIrmuN96AqCjNLBIREf/n6YpRIWAIcD92j9EK4BXHcf7wbXjXjKcO\nUKdYsWJtduzY4UYIIinmXMw5xm0cx8BVA9l2bBsFshWg032daFuhLbkyeX+MmOM4LN63mAErBzB7\n+2xCgkNoek9TXqv8Gvfku4ff/v6NciPKUbtYbWY+O1PDVeWaFi2CGjWge3f48EO3oxERkbTKq80X\nrnGBzo7juLqjSqV04m9i42NZ/edqYuNjk30uB4cfd/3I8HXDOX7+OBXyV6DL/V1oWLIh6YPTeyHa\nG9t+bDuDVg1izC9jOB97nlpFa3H03FH2R+1ny0tbyJc1X4rEIYHn4kXbhS42Fn79Ve25RUTEPd5u\nvnA1XQD/aDUh4ic+Wf4J7yx8x2vnMxjqlahHl8pdqFqoaoqvztx1010MfWIoHzz0ASPXjWTwmsEc\nOH2AKc9MUVIk1/XRR7ZF97x5SopERCQwJGfFaL/jOLd5OZ5E0YqR+JMT509QZFARKhWsRLcq3bxy\nziK5ilA4Z2GvnMsbouOi2Xl8JyXzlHQ7FPFj27bZ1aIGDWDSJLejERGRtC4lVowCdzKsiA8MWDmA\nqItRfFLrE8rcUsbtcHwiJDhESZFcl+NA+/Z2leizz9yORkRExHPXTYyMMae5egJkgKv38RVJg/4+\n+zcDVw+kYcmGqTYpEvHE2LG26cLIkZBP1ZYiIhJArpsYOY6TLaUCEQlknyz/hHMx5+hZvafboYi4\n5uhR6NoVqlSBF15wOxoREZHE0VQJkWQ6ePogQ9cOpek9Tbk7z91uhyPimq5d4dQpzSwSEZHApKcu\nkWTqu6wv0XHR9KjWw+1QRFyzcKEto3vzTSipbWgiIhKAlBiJJMMfUX8wYt0IWpdrzR2573A7HBFX\nXLhgGy7ccQe8471u9SIiIikqOV3pRNK83kt6A/C/B//nciQi7unbF3bsgB9/hExqyyMiIgFKK0Yi\nSbTr+C6+2vAVbcu3pVCOQm6HI+KK33+3iVGzZlCrltvRiIiIJJ1WjESSqNeSXqQPTs/bYW+7HYqI\nR/buhQ4dIF06KFbMlr4l3AoXhpCQxJ3PcaBdO8iaFT791BcRi4iIpBwlRiJJ8NvfvzFh0wS6VO5C\n/mz53Q5H5IYuXIBnnoFt26BIEdss4dy5S98PCoLbbvtvwpRwy3aV4Q1jxsCSJTBqFOTNm3I/i4iI\niC8oMRJJgvcXv0/m9Jl5s8qbboci4pFXX4V16+Dbb6FePbvac+gQ7Nr139uMGXYm0eXy5LkyaSpc\nGN54A8LCoFUrV34kERERr1JiJJJIGw9tZOqWqbwT9g55suRxOxyRGxo71s4W6tbNJkUAxkD+/PZW\ntep/j4mKgt27YefOK5OmJUtg4kSbWIWEwIgRmlkkIiKpg3Ecx+0Ykiw0NNSJjIx0OwxJY+p9XY/F\nexez59U95MqUy+1wRK5r40aoXBnuvx8WLLD7i5Lr4kW7XykoCO68M/nnExER8SVjzDrHcUJv9Dit\nGIkkwtq/1jJr2yw+qPGBkiLxe1FRdl9RrlwwebJ3kiKADBmgeHHvnEtERMRfKDESSYR3I97lpkw3\n8WqlV90OReS6HAdatrQrO4sWQb58LgckIiLi55QYiXho6b6lzN81n34P9yNbhqu06BLxI/3720YL\nAwZAlSpuRyMiIuL/tGVWxAOO4/C/iP9xS9ZbeKniS26HI3JdixbZRgvPPAOdO7sdjYiISGDQipGI\nB37e8zNL9i3h89qfkzl9ZrfDEbmmgwehcWPbFGH0aNt9TkRERG5MiZHIDTiOw7sR73Jb9ttoW6Gt\n2+GIXFNMDDRqBKdPw88/Q/bsbkckIiISOJQYidzA3B1zWfXnKkY+OZIM6TK4HY7INXXvDsuWwYQJ\nUKqU29GIiIgEFu0xErmOeCeedyPepWiuorQs29LtcESuafp0+PRT6NgRmjZ1OxoREZHAoxUjkeuY\n+dtMNhzawNj6Y0kfnN7tcESuavt2aNUKKlWyyZGIiIgknlaMRK4hLj6O9xa9R4mbS9D0Hr0FL/7p\n7Flo0ABCQmDqVDt8VURERBJPK0Yi1zBlyxS2/r2VKc9MITgo2O1wRP7DcaB9e9iyBebNg0KF3I5I\nREQkcCkxErmK2PhYeizqwb357uWZks+4HY7IVQ0fbhst9OoFjzzidjQiIiKBTYmRyFWM2ziOncd3\n8l3j7wgyqjgV/7NmjR3e+thj8M47bkcjIiIS+PSKT+RfLsZepOfinlS8tSJ17qrjdjgi/3H0KDRs\nCPnzw/jxEKS/5CIiIsmmFSORfxm9YTR/RP3ByCdHYoxxOxyRK8TFQbNmcOgQLF8ON93kdkQiIiKp\ng95nFLnM+Zjz9Fnah6qFqvLIHdq0If6nd2+YPx8+/xxCQ92ORkREXHXxIqxd63YUqYYSI5HLDI8c\nzoHTB+hdo7dWi8TvzJsHPXvC889D27ZuRyMiIq4bNgzuuw82bHA7klRBiZHIP85En6Hvsr7UKlqL\naoWruR2OyBX27YOmTaF0afjiC1DeLiIi/Pij/Th0qLtxpBJ+kxgZY4oaY0YbY6a5HYukTYNXD+bv\nc3/zQY0P3A5F5AoXL9pmC7GxMH06ZM7sdkQiIuK6mBhYuhSCg2HiRDh+3O2IAp5PEyNjzFfGmCPG\nmM3/ur+2MWabMWanMaYbgOM4ux3HecGX8YhcS9SFKPqt6McTdz5B5YKV3Q5H5P8dPw6tW9sS8vBw\nuPNOtyMSERG/sG4dnDkD3brBhQswZozbEQU8X68YhQO1L7/DGBMMDAUeA0oCTYwxJX0ch8h1fbbq\nM05cOEGvGr3cDkUEgKgoeP99KFIEJk+GHj3gqafcjkpERPxGRIT9+OqrUKWK3W8UH+9uTAHOp+26\nHcdZYowp/K+77wN2Oo6zG8AY8zVQD9jqy1gkddlxbAft5rQj8kCkV853NuYsDe5uQPn85b1yPpGk\nOn0aBg+G/v3hxAl4+mmbIN1zj9uRiYiIX4mIsE8OefLAyy9Dkya2S8/jj7sdWcByY45RAWD/ZV//\nCVQyxtwE9AHKGWO6O47T92oHG2PaAm0BChUq5OtYxc/EO/EMWTOEbj91I0O6DLQs25J0Qcn/Zxxs\ngulUqZMXIhRJmnPn7N7ZTz6xA1yffBJ69YJy5dyOTERE/E50tB1m9+KL9uunn4Z8+ewTiRKjJHMj\nMbpaLyXHcZxjQPsbHew4zkhgJEBoaKjj5djEj+05sYfWs1qzaO8iHiv2GF/W+ZIC2Qu4HZZIsly4\nACNGQN++cPgwPPqobcldqZLbkYmIiN9as8a+o1ajhv06JMTOcejdG3bvhqJF3Y0vQLnRle5P4LbL\nvi4IHHAhDgkQjuMwInIE9w6/l3UH1jGqzii+f+57JUUS0C5etOXgd9wBnTtDyZK2udC8eUqKRETk\nBiIi7NyGapeNF2nXDoKC7EwHSRI3EqO1wJ3GmCLGmBCgMTDLhTgkAOyP2k/tibVp/317KheszOaX\nNvNC+Rc0fFUCVkwMjBoFd90FHTva5goLF9pb1apuRyciIgEhIgLKloVcuS7dV6CA7dIzerRdTZJE\n83W77snASqC4MeZPY8wLjuPEAi8D84HfgKmO42zxZRwSeBzHIfyXcEp/UZrlfyxn2OPDWNBsAYVy\naF+ZBKbYWBg3DkqUgDZt4JZbYP58u0qUUAkhIiJyQxcuwIoVV3/y6NjRdu75+uuUjysV8HVXuibX\nuH8uMNeX15bAdfD0QdrOacuc7XN48PYHGVNvDEVzqVZWfC8mxpayBQdD7tz2jbiEj+mS+NcyPh6m\nTLH7hrZts80UZs+GJ56wVRAiIiKJsnKlrce+WmJUrRqUKgVDhkCrVnqiSSQ3mi8kmzGmDlCnWLFi\nbociXuQ4Dl9v/pqOcztyPvY8nz36Ga9UeoUg40bFp6Q1R49Cw4awaNHVv58tm02SLk+Y/p08/fu+\ntWvt/KEtW6B0aZg+3VY56HlKRESSLCLC7iUKC/vv94yxq0YvvQSrVsH996d8fAHMOE7gNnYLDQ11\nIiO9M8dG3HXk7BFe+v4lpv82ncoFKxNeL5ziNxd3OyxJIzZtgnr14OBB+ybbPffA8eO2GuH48Ss/\nv9p90dHXPnfx4nYOUaNG9nlMREQkWcLC7IrRmjVX//7p03a/Ud26MGFCysbmp4wx6xzHCb3R4wJy\nxUhSl+lbp9Ph+w5EXYzio5of0fWBrgQHBbsdlqQR06fD889Dzpx2v0/Fiok73nHg/Pn/JkvHj9uV\no7p1k16GJyIicoVz52D1aujS5dqPyZYNWrSAkSNhwADImzfl4gtweroW1xw/f5yX577M5M2TKZ+/\nPAvrL6R03tJuhyVpRHy83ffTq5etNJg+HfLnT/x5jIHMme2tYEHvxykiIvL/li+3G2Jv1LWnY0db\nAjFqFLz9dsrElgqosENcMWf7HEoNK8U3W7+hZ/WerHphlZIiSTGnT0ODBjYpat3almsnJSkSERFJ\nURERtgyhSpXrP65ECahZ0840io1NmdhSASVGkqLORJ+h1XetqDO5Dnmz5GVtm7W8V+090gendzs0\nSSN27bIrRLNnw6BB9s20DBncjkpERHzi3Dm7kTS1iIiA++6DrFlv/NiOHeHPP+0TnngkIBMjY0wd\nY8zIqKgot0ORRHr757cZt3Ec74S9w9o2ayl7S1m3Q5I05Oef7fPJwYN2htArr6hDnIhIqtarF4SG\n2o2fge70advu1NPhd3XqwG23wdChvo0rFQnIxMhxnNmO47TNkSOH26FIIhw+c5gv139Jq7Kt6P1Q\nb0KCQ9wOSdIIx7GrQ48+akvm1q61FQYiIpKKOQ5Mnmz35Cxf7nY0ybd0KcTFeZ4YpUsH7dvbdwV/\n+823saUSAZkYSWAauGog0XHRvFXlLbdDkTTk4kV44QXo3Nm+ebZyJRTVvGARkdRvzRr44w/7+dKl\n7sbiDREREBICDzzg+TEvvmiPGTbMd3GlIkqMJEWcvHCSoWuH0rBkQ+686U63w5E04uBBqF4dxoyB\n996zneeyZXM7KhERSRFTp9qk4N57U09iVLkyZMrk+TF589pBemPH2lI8uS4lRpIihq4Zyuno03Sv\n2t3tUCSNWLvWlpX/+itMm2Zbc2vAqohIGhEfbxOj2rXh8cchMtI2YghUJ0/Chg2el9FdrmNHmxRp\n2OsN6WWC+NzZ6LMMXD2QJ+58gjK3lHE7HEkDxo+3g8FDQmDFCtuaW0RE0pBVq2xHtkaN7BNCbKwd\njBqoliyxyd5DDyX+2EqVoHx5O9fIcbwfWyqixEh8btT6URw9d5S3wzRgTHwrNha6doXnn7ctudeu\ntRUUIiKSxkydamcx1Klj9+QYE9jldBERkDGjTXISyxh4+WXYuhUWL/Z+bKmIEiPxqei4aPqt6Ee1\n26vxwG2J2CwokkgnTsATT8Cnn9q//wsWwM03ux2ViIikuPh4+OYbeOwxyJ4dcuYM/H1GERF2qGtS\nB+81bgy5c6t19w0EZGKkOUaBY/zG8fx1+i+tFonPREfDunV2PlFEBHz5JQweDOk1M1hEJG1avhwO\nHIBnn710X1iYbUsaG+teXEl17Bhs3Ji0/UUJMmWC1q1h5kxbYihXlc7tAJLCcZzZwOzQ0NA2bsci\n1xYXH8dHyz+iQv4KPFz0YbfDkQBy8SIcPmxvhw5d+vxqtxMn7DF58156Q01ERNKwqVNt2dmTT166\nLyzM7rHZsAEqVnQvtqRIKH9LTmIE0KGDLasYOdIOvpX/CMjESALDtK3T2Hl8J9MbTccY43Y44ifi\n4mDnTvvctHPn1ZOfay0GZ88Ot9wC+fJB6dJ2SGu+fPa+J5+0w1tFRCQNi4uzrUifeAKyZr10f1iY\n/bh0aeAlRgsXQpYsyY+7aFHboW/kSPjf/2yHIrmCEiPxCcdx+HDZh5S4uQT1S9R3OxxxyfnzsHmz\nTYJ++cXeNm68smNqzpw2ucmXD8qUuZToJNx3+S1jRvd+FhERCQDLltl32xo1uvL+/PnhjjtsYtSl\nizuxJVVEBFSt6p0a8Y4dbXI0Y4bddyRXUGIkPjF3x1w2Hd7E2PpjCTIBuZVNEunYsUvJT0Ii9Pvv\n9s07sKs9ZcvaIdzlytnPS5RQsiMiIl40ZQpkzmxXjP4tLAzmzLEtqwOlkuXwYdtN7vnnvXO+Rx+1\nCeKQIUqMrkKJkXid4zj0WdqH23PcTpPSTdwOR7zMcWDv3v8mQfv3X3pMgQI2+XnqqUtJUJEigfM8\nJCIiASg2FqZPt7XVWbL89/thYRAebt+1u/vuFA8vSRYtsh+Tu78oQVAQvPQSvP66LeEoo/mSl1Ni\nJF63ZN8SVv65kqGPDyV9sFqDpSazZtmmNseO2a+DgqB4cftcU7asTYLKlIE8edyNU0RE0qAlS+DI\nkf+W0SW4fJ9RoCRGERG25KJ8ee+ds1Uru8do6FC730j+nxIj8boPl31Iviz5aFW2lduhiBfNmwcN\nG0KpUtCnj02E7rnHViyIiIi4bupUu1L02GNX/36xYnbD6tKl0LZtysaWVBER8OCDkM6LL9lz5YLn\nnoMJE+Djj+3XAgToHCPxX5EHIlmwawFd7u9CpvSZ3A5HvCQiwpbFlSwJP/8M7drZ4dtKikRExC8k\nlNHVrXvtJydj7KrRkiUpG1tSHTgA27d7r4zuch072g5J4eHeP3cAC8jESANe/VffZX3JmTEn7UPb\nux2KeMny5VCnju3yuWCB3lgSERE/FBEBR49eu4wuQVgY/PGHvfm7iAj70ReJUbly8MADMGwYxMd7\n//wBKiATI8dxZjuO0zZHjhxuhyKX2fr3Vmb8NoNO93Uie4bsbocjXhAZabt63nor/PST9g6JiIif\nmjrVzi2qXfv6j7t8n5G/i4iw70b6qkFCx452oOCCBb45fwAKyMRI/NPHyz8mc/rMvFLpFbdDES/Y\ntAkeeQRy57blcxqeKiIifikmxs7lqVfvxjMg7r3XNjMIlMSoWjXb6cgXnnnG7rkaOtQ35w9ASozE\nK/ac2MPETRNpV6EdN2e+2e1wJJl++w1q1bJl2gsXwm23uR2RiIjINfz8Mxw/Ds8+e+PHBgfbEjJ/\nT4z27YPdu31TRpcgJATatIHvv4c9e3x3nQCixEi8ot+KfgSZIF6//3W3Q5Fk2rkTata0b1AtXGjn\nD4mIiPitqVPtKtAjj3j2+LAwOzQ1YfaEP/Ll/qLLtWtnn/CHD/ftdQKEEiNJtoOnD/LVhq9oWbYl\nBbIXcDscSYZ9+2xSFB1t9xTddZfbEYmIiFxHdDTMnAn160OGDJ4dk7DPaNky38WVXBERcPPNdkaG\nLxUsaEsQR42yXerSOCVGkmyfrfqMmPgY3qryltuhSDL89ZdNiqKi7D7M0qXdjkhEROQGfvwRTp68\ncTe6y1WsaMvI/LWcznFsYlSjhu/2F13u5ZdtKeKUKb6/lp9TYiTJcvz8cb6I/ILGpRtzR+473A5H\nkujIEbun6PBhO8jVmwO2RUREfGbqVMiZEx5+2PNjMmaE++7z38Ro927Yv9/3ZXQJqle3gwqHDLFJ\nWRqmxEiSZciaIZyJPkO3Kt3cDkWS6Phx+3yyb5/df1m5stsRiYiIeODiRfj2WzuBPCQkcceGhcH6\n9XD2rG9iS46U2l+UwBh46SVYtw7WrEmZa/qpdG4HIIHrTPQZBq0eRN3idbkn3z1uhyNJEBVl96pu\n2wazZ8ODD7odkYhIKta1q30x7m9KlYLBg92OIvEWLIBTpxJXRpcgLAz69oVVq2wduT+JiIBbboHi\nxVPums8/D927Q5MmULiwd8754ovw3HPeOVcKCcgVI2NMHWPMyKioKLdDSdNGrhvJ8fPH6V61u9uh\nSBKcOWOHt27cCNOmJa4KQUREEmnbNvj0Uzh0CGJj/ef299+2hGrjRrd/Q4k3ZYodtpeUxOaBB+xK\nib+V012+v8iYlLtutmzw0Ue2GYO3/m0FYFleQK4YOY4zG5gdGhraxu1Y0qqLsRfpv6I/DxV5iMoF\nVXsVaM6fh7p17RtlU6bAk0+6HZGISCo3caLdSP/TT3DrrW5Hc8mxY3Z1YsIEKFPG7Wg8d/48fPcd\nNG4M6dMn/vgcOezP62+J0fbtcPBgypXRXe6ll+wtDQvIFSNx39iNYzl45iBvV33b7VAkkS5ehKef\nhkWLYNxeF7iYAAAgAElEQVQ4O/haRER8yHFs4lGzpn8lRQA33WTLByZNgrg4t6Px3Pz5tvQhKWV0\nCcLC7DuEMTHeiyu5Fi60H91IjESJkSRebHwsHy//mPsK3MdDRR5yOxxJhJgYOxh83jz48kto2tTt\niERE0oAVK2DPHmjWzO1Irq5ZMzhwwL5jFiimTrVJXXISiLAwOHfOv/Z9RUTYcrY71OnXDUqMJNGm\nbpnK7hO7ebvq25iUrH+VZImLg+bNbeXB4MHwwgtuRyQikkZMmACZM9vlen9Upw5kzw7jx7sdiWfO\nnYNZs6BBA0iXjF0hCYNe/aWcznFscprS+4vk/ykxkkSJd+Lpu6wvpfKUok7xOm6HIx6Kj7eJ0JQp\n8MkndpabiIikgOho+8e3fn3ImtXtaK4uY0Zo2BCmT7dJh7/74QfbZvvZZ5N3nltugWLF/Ccx2rLF\nNsN4SNU4bgnI5gvinjnb57D5yGYmPDWBIKO82p/ExsKJE3YuUcLHhNvSpbbzXM+e8MYbbkcqIpKG\nzJ1r/yj7axldgmbNYPRouxLTuLHb0Vzf1KmQN693ZkyEhdlSivh42xzDTSk9v0j+Q4mReMxxHPos\n7UORnEV4tnQy36WRGzp+HH799coE52pJT8LXp05d+1xBQfDuu/YmIiIpaMIE+yLe32ciPPgg3Hab\nLafz58To7FmYMwdatEheGV2CsDAYMwZ++83Oc3JTRAQUKQK33+5uHGmYEiPxWMTeCNb8tYbhTwwn\nXZD+6fjKzp0wYACEh9tupJdLl86ObEi43XorlC595X25cv3365w5vfP8ISIiiXDypJ2e3aGD//8R\nDgqyHXn69YMjR2wy54++/96W+yWnG93lEladli51NzGKj4fFi23JpbjGz/8vFX/y4dIPyZ81Py3K\ntnA7lFRp1Sr7fDRzph3J0KyZfdMuT55LCU7WrNqPKSISMKZNs3uM/L2MLkGzZnbI55Qp0KmT29Fc\n3dSpdm9QQuOE5CpaFPLnt4lR+/beOWdSbNpkyz9URucqJUbikdV/rubnPT/T/+H+ZEyX0e1wUo34\nePtmYr9+sHy5Xdnp3t02R8if3+3oREQkWcaPhxIloEIFtyPxTKlSUK6cjdsfE6MzZ+yK0YsvQnCw\nd85pjE2y3G7AoP1FfkGJkRdcjL3IoTOH3A7Dpz5Y8gG5MuaiXWg7t0NJFc6ft8NVBwywQ64LF4ZB\ng6B1a/9tWiQiIomwbx8sWQK9ewfWUn+zZvD667BtGxQv7nY0V5ozBy5c8F4ZXYKwMLsStW+fe/t7\nFi6EO++EAgXcub4ASoy8YuPhjVQaVcntMHyuR7UeZA3Rq/bkOHoUhg2DIUNsR84KFeDrr5M/ikFE\nRPzMpEn2Y6BN0m7SxLYvnTgRevVyO5orTZliN9dWqeLd814+z8iNxCg21ibR/tz0Io0IyJdixpg6\nQJ1ixYq5HQoARXIW4au6X7kdhk+lD05Pg7sbuB1GwNq5Ez77zDa+OX8enngCunaFatUC641EERHx\ngOPYcrSwMFsSEEjy54datWw3vZ49/edJ6tQpO7+ofXvvt9UuXRpy5LCJkRv7wTZssD+fyuhcF5CJ\nkeM4s4HZoaGhbdyOBSBPljy0KtfK7TDED61aBf37w4wZlxoqdOnifkdQERHxoQ0bbPvnESPcjiRp\nmjWD55+HFSu8vzqTVLNnw8WL3i+jA7tfqUoV9/YZaX+R39CEThEvi4+3s+LCwuD+++Hnn6FbN9i7\n187OU1IkIpLKTZgAISHQsKHbkSTNU09B5sz25/AXU6ZAwYJQubJvzh8WZpPZo0d9c/7riYiAkiUh\nX76Uv7ZcQYmRiBf9+KP921a/PuzfDwMH2o8ffqgucyIiaUJsrN1f9MQTds5CIMqa1T6RTZliV2nc\ndvIkzJ9vE01vl9ElSNhntGyZb85/LTExdqVKq0V+QYmRiBfExMDbb8Ojj9py7K+/tvuKXn1VXeZE\nRNKUn3+Gw4eheXO3I0me5s3hxAm7r8dts2bZeVDPPuu7a4SGQoYMKV9OFxkJZ88qMfITSoxEkmnf\nPqheHfr2taMV1q2zf7vVZU5EJA2aMMEOpXv8cbcjSZ5atSBvXv8op5s6FQoVgvvu8901MmSASpVS\nPjFK2F9UrVrKXleuSomRSDLMnAlly8Kvv9pVopEjbVm2iIikQWfO2G47jRrZF9qBLF0627p79my7\ncuSWEydgwQL7O/V1h7ywMFi/3v53TCkREXDvvXDzzSl3TbkmJUYiSXDhArz8Mjz9NBQrZhsQ+XKF\nX0REAsC338K5c4FfRpegeXNbwjZtmnsxfPutrVf3RTe6fwsLg7g4WLnS99cCu39r2TKV0fkRJUYi\nibRtm+02N3Sobb29fDnccYfbUYmIiOsmTLBzix54wO1IvKN8eShRwt1yuqlToUgRuwfI1+6/3zZ3\nSKlyutWr7TutSoz8hhIjkUQYNw4qVLCd5ubMgU8/tR1ZRUQkjTt40LYmbdrUd53TUpoxdqbRkiV2\n5kRKO3YMfvopZcroALJnt/XxKZUYRUTYn+vBB1PmenJDqeT/XBHfOnMGWrSwtwoV4JdfbCdWERER\nwG40jY+3iURq0rSp/ThpUspfe+ZM2/48JcroEoSF2ens0dG+v1ZEhF2VC9S27qmQEiORG9i40a7g\njx8P/9fefcdZUV5/HP8csWuCLbEriCJiAXUF20YkxBJAFEVAsPdYg6LAy0RNoiCIYkEUsGIFY0V/\nliRoiAIKioJYUIqKiCi4UqTu8/vjzHWvyLJt7p1bvu/Xi9dyy86cuwyzc+Y5z3muvRb+8x9fY05E\nROQnDz/svyyaNEk6kng1aODJwogREEJ29z1ypNeq779/9vZZWurlbZMmZXY/P/7oc5lURpdTlBiJ\nVCIEn0fUsiX88IMnRNddB/XqJR2ZiIjklGnTvJtZoY0WpXTvDh995J8xW+bP91+8nTtnp4wu5fDD\n/Wumy+nGjfNRKSVGOUWJkchaLFwIJ57onedat/ZRo1atko5KRERy0sMP+12zLl2SjiQzOnXyCbXZ\nbMLw9NPeIS6bZXQA224LjRtnPjEaM8aPmdLSzO5HakSJkcgaxo3zuZfPPw833+xNFn7zm6SjEhGR\nnFReDo88Akcd5RfVhWjLLaFdO3jsMZ/zkw1PPOEJyn77ZWd/6UpLveVseXnm9jFmjJde/upXmduH\n1JgSI5FIeTn06+fnw3r1/Jx4xRWF01xIREQyYOxY+Pzzwi2jS+neHebNg3//O/P7+vRTeO217HWj\nW1NpqZeOfPBBZrY/dy689ZbK6HJQXl7ymVl7MxtaVlaWdChSIObNg2OOgd69fdHWd9+FFi2SjkpE\nRHLeww/DZptBhw5JR5JZf/yjjxyNGJHZ/SxaBMcfD/Xrw7nnZnZflUmVt2WinG7ZMr/Q2HBDb3Ur\nOSUvE6MQwvMhhPPq16+fdCiSp8rKfLmJf/wD2reHPff089899/jovQ4tERGp0rJlMGqUT0rdbLOk\no8msjTbyEZynn/Y1LDKhvBxOPdUbPYwaBbvskpn9VKVhQ9hhh/gToxDgwgu9HfhDDxVeB8MCsH7S\nAYhk2qpVMHWqLzA9frx//eijiq6je+0FJ5wAPXrAvvsmG6uIiOSR0aP9Tluhl9GldO/udxCfeSYz\nn/n66+HZZ2HQIPj97+PffnWZ+ajR2LF+sRBXOd/tt8MDD/jaHx07xrNNiZWFbPekj1FJSUmYOHFi\n0mFIjpkzpyIBmjABJk6EpUv9tW228fbbBx/sXw86CLbYItl4RUQkTx1/vM8V+eKL4ljLobzc1xVq\n3Bhefjnebf/zn3DSSXDmmXDvvcnMLUo3eLC3pp0xw0eQ6upf//Ka/eOOgyef1ATmLDOzSSGEkqre\npxEjyWtLlvgabKkkaPx4T4zAy3f33x/OOaciGWrYMPlzrYiIFIDvvoMXX4RLLy2OpAj8Yr57d7jx\nRm8gsP328Wz3vffgtNP8F/WQIbnxizp9nlFdE6NPP/UyxL328hI6JUU5S4mR5KXycjjrLJ/zunq1\nP7fbbvC731WMBjVv7iXRIiIisRs5ElauLJ4yupRu3XyC7uOPw5//XPftffutN67Yckt46qnc+cW9\nzz5eUjJ2rCdttbVokX8+My8T3Hzz+GKU2Ckxkrx07bXw4INw/vm+tELLllprSEREsujhh2HvvaFZ\ns6Qjya4mTXz9nREj6p4YrVzpi8d+/bUnIHGNQMVhvfXgsMPq1oAh1Uzi44/hlVf8Dq7kNI3lSd4Z\nNcpvVp11lo+4t2unpEhERLJoxgx4802/6M2Fsq9sO/VUX9eiruv8XH65r1c0fLhP+s01paWe1Hzz\nTe2+/9prfZTo1luhdet4Y5OMUGIkeWXyZDjjDDj0ULjrruL8fSQiIgl7+GH/esopycaRlM6dfV7V\nI4/UfhtDh/ov8iuvzN1yxNQ8o//9r+bfm7qLe/bZ3sRB8oISI8kb8+d7A6Att/TmNblShiwiIkUk\nBE+MWrWCnXdOOppkbLstHHWUJ0bl5TX//v/9z5OFY46Bfv3ijy8uJSWw8cY1L6dLv4s7eLDu4uYR\nJUaSF1au9C6e8+b58gnbbZd0RCIiUpTefhumT/dysmJ26qnw+ec1Txo+/9zX8GnQAB59NLc7+m24\noU9irslnTN3F3Wor3cXNQ0qMJC9cdhn897++tEFJlV3oRUREMmTECL/YPfHEpCNJVocO3mEtVVZY\nHUuXetKwfDk895yXgOS60lKfT7VoUdXvTb+L+/TTuoubh5QYSc675x5vsnDVVcVbzi0iIjlg5Upv\nU33ccVC/ftLRJGvTTX3kZ9QoWLas6veH4F2TJk/2kaImTTIfYxxKS71ccNy4qt+ru7h5T4mR5LSx\nY70M+dhjfT05ERGRxLzyiq+7k6vNArKte3coK4PRo6t+7003wRNP+C/ztm0zH1tcDjnEW3dXVU6n\nu7gFwUIIScdQayUlJWHixIlJhyEZMnu2d+/caisYP97XWRMREUlMly7wr3/BV1/5/JNit3q1N6Bo\n0cInAFdm9GgfZevc2UeL8q0ZQUmJlw2+9traXx871ttx/+EP8PzzuT1vqkiZ2aQQQpXDeBoxkpy0\nZElFGfKzzyopEhGRhP3wg/9C6txZSVFKvXo+OvLii/Ddd2t/z4cf+nv2399LzPItKQIvp5swwS9K\n1jR7ts83a9Qo95tJSJWUGEnOSZUhv/cePPYY7Lln0hGJiEjRe+opn0ujMrqf697d516NHPnL1xYu\n9CYNm2zizQg23TT78cWhtNT/7SdN+vnz6c0kdBe3ICgxkpzTt6+fX/v1gz/+MeloRERE8G50jRrB\nwQcnHUluadYM9tnnl93pVq+Grl1h1ixvW73LLomEF4vDD/ev6fOMdBe3ICkxkpzy/PNwzTU+6t6z\nZ9LRiIiIAF9+CWPG+OhIPpaCZZKZ/1zefBM++6zi+auvhpdf9gVOU4lFvvrtbz3xSU+M+vb1ZhK6\ni1tQlBhJzpg2Dbp1gwMOgOHD9btHRERyxGOP+QhBt25JR5KbTjnFf2k/8og/HjECBg6Eiy6Cc89N\nNra4lJbC//7nI2G6i1uwlBhJTkiVIW+6qTe22WSTpCMSERGJjBjhJXR77JF0JLlp552hVSsvp3vr\nLU+GWrWCW29NOrL4lJZ6a/JRo3QXt4ApMZLErVrlTX5mz/a5rTvtlHREIiIikfffhylT1HShKt27\nw/TpcNRRsN12nkBssEHSUcWntNS/du+uu7gFbP2kAxC5+mp49VW/8XLooUlHI+s0YQKcd56XEeTz\nRNrKTJvm65Q88QTstVfS0RSPM86AHXeEG25IOpLMuPxyvwN0551JR1JhzBi/671yZdKR5L5ly2D9\n9f0OnlTuxBO9dG7lSu/Qts02SUcUrwYN/Dz1zTfeTEJ3cQtSXi7wambtgfa77777udOnT086HKmD\nhx6C00+HSy6B229POhqp0lFHeRZ70UW5dZEXl1RSdMopFbXyklnjx/vK8uuv7xO3Cy3h/ugjaNrU\n56d88IH/PWkhwGGHwcyZ0LFj0tHkh5ISOPPMpKPIfc88A7/5jR9fhei55/xcpWYLeae6C7zmZWKU\nUlJSEiZOnJh0GFJLEybAEUf4+fOllwprxL0gvf22r26+zTawaJG3YN1uu6Sjis8nn0CTJrD11rBg\ngT9u1CjpqApfhw7w+uu+qvOf/gS33ZZ0RPE680xPts38jvpDDyUdkf+8W7XymxsXXZR0NCIiGVfd\nxEhzjCQRX30FJ5wAO+zgaxYpKcoDffv64nUvveSlEoMGJR1RvPr3h4028hGxDTbwx5JZU6b4HdjL\nL/e6/WHDvEylUMye7ZPRzz0Xzj8fHn3UR2mSduON3n74rLOSjkREJKcoMZKsW7bMqzd++MHLkLfe\nOumIpErTpvmq5ZdcAgceCCefDHfd5e0EC8EXX/id/HPOgebN/YLxgQdgzpykIyts/frBZpvBpZdC\nr15+ciikEaObb/aRoiuvhCuugHr1YMCAZGOaOBFeeQV69NDEcRGRNSgxkqwKAS64wMvoRoyAffdN\nOiKpln79vAvPpZf64969vZxu8OBk44rLwIF+cF55pT/u2dPXqrjllmTjKmSffQaPPw4XXghbbeWL\nJ550kpd3lZUlHV3dzZvnHWVOO81bGe+4ozeZuO8+mDs3ubj69oX69f3nLiIiP6PESLLm88+9WubB\nB+G667yUTvLAzJleAnT++RVdhvbbD9q183K6JUuSja+u5s+HoUP94Nx1V3+uYUNvwHD33fDdd8nG\nV6j69/dJzD16VDzXu7cPJd91V3JxxWXQIFixwttuplx1lZehJrW2y7RpvibCJZfAr3+dTAwiIjlM\niZFkXFmZX+80buwdLq+5Bv7yl6SjkmobMADWW89LgdL16eNJw7BhycQVl9tu8xKu9AtY8NKupUvV\nLjET5szxUsWzzoLtt694fv/94dhjPXFYujSx8Ors++99NLVTp58vCNqokXc+HDLEG3xk2003+cjv\nZZdlf98iInlAiZFkzIoVcMcdsPvuXol18sne6Ovvf/frbMkDc+d66U9qnZl0hxzina1uvhmWL08i\nurorK/PSrRNP9I506Zo29WHN22/3skGJzy23eKliz56/fK1PHx/Fu/fe7McVl8GD/Zjp3fuXr/Xq\nBYsXZ7/d/axZ3oL+vPMKb30ZEZGY6PJUYheCjwztvbdPSdlvP5g0yee2F9oSJQXv1lu99Oeqq9b+\nep8+fvd/xIjsxhWXIUMqhjTXpndvv/t/993ZjauQffed/zy7doXddvvl64cf7ivMDxjgd1fyzZIl\nXkbXti00a/bL1/fdF447zkcqFy/OXlyVjfyKiMhPlBhJrMaN8+uak07yzscvvAD/+hcccEDSkUmN\nLVjgiUPnzj7stzZt2vjCh/36wapV2Y2vrpYu9ZGLY46p/AA96CD4wx+8OcOyZdmNr1Ddfrv/7Hv1\nqvw9ffp4p8B8XGR3+HD49lv/DJXp3dv/fw0dmp2Yvv7aR+BOPx122ik7+xQRyUNKjCQWn37q5fSH\nHgozZvi0k8mTfXFos6Sjk1q5806/o72uC1gzvwD87DN48snsxRaH++7zkq11XcCCvz5vHtx/f3bi\nKmSLFnlidPzxPqRcmaOP9vlG/fp5yV2+WLHCR2aOOMJPhpU5+GBo3Tp7ZahVjfyKiAigxEjq6Lvv\nfG3Gpk3hxRe929z06b4czPrrJx2d1NrixV7q076910KuS4cOsNdevmhkCNmJr65WrPCuaKmyrXU5\n4gifT9W/v19cSu3dfbeXJlZWupiSSrg/+cS7qOWLESO8tLSqZBv8PXPnepvOTFq40Lv8nXzyzxtB\niIjILygxklpZtsxvjDZq5A0WzjjDR42uvRY23zzp6KTOhg71Up/qXOCtt55f6E6Z4rWT+eDRR71U\nqzqfL3WRPmuWr7sjtbNsmZcktmkDLVpU/f4TTvC1jfIl4V692ke4DjzQyy+r0rq1/xxuuimzZaip\nkd+qklEREVFiJDVTXu5l/02aeFXG4YfD++/7dXR6113JY8uXe4nPkUd6yU91dOkCDRrADTfk/kXs\n6tW+yGXz5j6/qDratvWRs759/T+B1Nz993tJYnWSUYB69byMc/JkeOmlzMYWh1Gj/O5Qnz7Vqx9O\nJdwzZsDIkZmJafFibwTRrl3VI78iIqLESKpvzBi/wdm9uy9U/+9/w+jR654qIHnowQe9xKe6F7AA\nG2zgmfL48fD665mLLQ5PPeUlWtW9gAV/X+/e8OGH8OyzmY2vEK1c6aWIBx/sLd6rq1s3b2V5440Z\nCy0WIXiMTZr4/Knqat/eT6CZSriHDav+yK+IiGAh1+/urkNJSUmYOHFi0mEUrB9/hHffhbfe8hu2\nL7/s1yg33ACnnKK1iArSqlVevrT11jBhQs06Zyxb5qNG++0Hr7ySsRDrJATvQLd0KUyb5qMS1bV6\ntV/4brGF/6dQV5HqGzECTjsNnnvOk4GauPNOuOQS+O9/q54PlpTRo/1zPfigf86aeOQRv9v07LPe\nxjsuy5d7O/TGjf2ulohIETOzSSGEkirfp8RIwK+HP/jAr/feftv/TJlS0RBqp53g4ot9XaJNNkk2\nVsmgRx/1u/RPP12zO98pAwb4yNFbb3mr61zzf//nrRLvuw/OPLPm3z98OJx7rid+1ZlHIj4Sss8+\n3o1l8uSa31H58UdPuA84wP/9ck0I3oFu7lzvPLPBBjX7/lWrPHn5zW98xDWuhHvYMF/MVceqiIgS\nI6lcCN5d+e23KxKhd97x6w+ALbf0a9qDDvLSuYMO0vyholBe7gtSlpd7VlybIcFFi3xY8cgjc7Ob\nWGkpzJ7tc0E23LDm3798uXcc2WMP3YWvrqefho4dPenu2rV22+jXz0sZJ03KvUXRXnvNj/fBg+FP\nf6rdNu65By64wOuTW7eue0yrVvno5pZbanRTRAQlRpJm7tyKUaBUIrRwob+28cZ+nZFKgFq08Os+\n/R4tQs895623R4zw0p7auvZa+NvffAiyadP44qursWPhd7/zdXQuuaT22xk0CP78Z3jjjXWvVSN+\nF6ZFCz/hfPRR7Xv4l5XBrrv6yMeoUfHGWFdHHeUdaGbOrP1w+rJlXvbWtKmviF1Xjz3m9c5PPeXd\n/UREipwSozxSXg4ffwzjxsGbb/poThxSI0NffumP69Xzipb0kaC996555YcUoBB8Yvw333g5UF0W\nofruO7+I7dgRHnoovhjr6thjfcRh1izYdNPab2fJEv98hxwCzz8fW3gF6dVXPXEYOtRLEOvimmu8\nwcG0aT4akgvefrui5XZdF08dOBCuvNLL6Vq2rP12UiO/q1fD1KmaDCoighKjnLZokY/cvPmmJ0Pj\nx1eM4Gy5pd80rMmc8HXZcceKJGj//et2PSgF7D//gd//HoYM8ZKeurriCl8gdvp0aNiw7turq3fe\n8fVlbrwxnvVc/vEP+MtffM5Ms2Z1316hOvJI7wA4YwZstFHdtjV/vieknTt76+9c0LGjl1TOng2/\n/nXdtrV4sZehlpbWrfNhauT3oYfg1FPrFpOISIFQYpQjUqM2qdGgceN8+kaqM2vTpl6Nc8gh/mfP\nPXWDTxLQpo2Xvs2c6fWVdTVnjpcGnX023HVX3bdXV506+ST0zz+H+vXrvr2FC/0ivW1bL1uSX3rz\nTTjsMLjlFi89jMPll/tcnk8/9Z9/kqZN8yH3v/zFS0fjcP31cN11Xpq37741//70kd9PPlE5gIhI\nRIlRQpYuhYkTK5KgceP8RifAr37lFRKpRKhlSx8hEknUhAl+MTVggJfyxOX887198cyZyXbv+Ogj\nvwPRu7f3mo9Lr17+M/voI2/GID/Xrp0Ph8+aBZtvHs82v/jCJ0Ged5638U7SaafBP//po0XbbBPP\nNhcs8FGjDh28jXdNpUZ+77oLLrwwnphERAqAEqMsmjHD52OPG+eVNatW+fONG1eMBB16aLwlciKx\n6dDBGxPMnu3Ze1w++8z/E/To4QlEUs48E554wi/Qf/vb+LY7b563ke7Wzdt4S4XJk712929/8xGV\nOJ1zDjz8sP97brddvNuurpkzPRm+9FIfEYtTz56+zU8+8SSwJtq08XlFs2bFM/IrIlIgqpsYqWgr\nBsuWwb33+k3Rnj19Pvb8+d5Q4YEH/Mb5vvsqKZIcNGWKz0m49NJ4kyLwi7ouXXze0oIF8W67umbP\n9ovoc8+NNykC2HZbLxV86CEfyZAK/fr5CfHii+Pf9tVXw8qVfjcqKQMGeM3zFVfEv+0ePbz5Sf/+\nNfu+CRO83fcVVygpEhGpJSVGMdhrL+8mO2aMz+1u1y6+ygqRjOrXDzbbrG7tq9eld2/v4nbHHZnZ\nflVuvtl7z8dZIpiuZ0+f1zFwYGa2n48++QRGjvQ1fTJRK7zHHj5n7K67KrrWZNPcub5A8BlneHeb\nuG2/PZx1lt9VmzOn+t/Xt6//vONoniIiUqSUGMXArG7djUUS8dln8PjjPhdh660zs4999vFSvdtu\n83aM2TRvnpe4nXYa7LxzZvax666+5tPQoRWTCYtd//7ega5Hj8zto08fP54GD87cPipz660+YnX1\n1Znbx1VXebvt6pbpTZ3qnewyMfIrIlJElBiJFKv+/T2jz+QFLPio0cKFnjxk06BBsGJFZi9gwbe/\nbJknf8Xuiy+8tPDss73UMFP228+H5gcN8hHJbFmwwEtDu3Sp+fyfmmjY0BdovftuXxesKpke+RUR\nKRJKjESK0Zw5Xqpz1lmZ7xjXsqV3yho40BOIbFi40EcTOnXKfMe4Jk3gxBO9S1pZWWb3lesGDvTS\nwp49M7+vPn08aRg2LPP7SrnjDl9vqFevzO+rVy9vc3r77et+34wZ3jL+ggsyN/IrIlIklBiJFKNb\nbvFSnWxcwIJfxM6d6+27s2HwYC+1imMx1+ro3duToiFDsrO/XDR/vo8KduuWnTWGDjkEWrXyeWTL\nlyAc4PUAABDwSURBVGd+f4sW+ajgccfVbo2hmmraFE44wROjdZWhZmvkV0SkCCgxEik2333nJTpd\nu/oirNlw5JE+cnTTTRX97DNlyRIvsWrbFpo1y+y+Ug44AI45xhPOpUuzs89cc9ttPiKY6dLFdH36\n+OjniBGZ39fQoT4Sma1kG3xf33/v/1/X5quv4P77vSX9DjtkLy4RkQKlxEik2Nx+u1+8Z6McKMXM\nL2JnzvQ1hTJp2DBP/vr0yex+1tSnj4+a3HdfdvebC8rKvJSwY0dv05ktbdpASYnPsclkwr1smZcJ\ntm7tiyFny0EHwR/+UHkZ6i23+Oe+6qrsxSQiUsCUGIkUk0WLPDE6/njYe+/s7rtdO+9S17cvlJdn\nZh/Ll3tp1RFH+KrK2VRaCocf7qVNK1Zkd99JGzLEk6NsjqZARcL92Wfw5JOZ28+DD3opaLaTbfB9\nzpvnI0Ppkhj5FREpcEqMRIrJ3Xd7aU62L2DBF8Ts3Rs++MBXQc6EESO8tCqJC1jw/X7xBTz6aDL7\nT8LSpT5ycfTRcOCB2d9/hw4+SnXjjd74IW6rVnkJaIsWPmKUbUcc4fOp+vf3NuEpd9zhZaPZHPkV\nESlwSoxEikWqHKhNG7/IS8LJJ/vd7UxcxK5a5SVVBx7o5UdJOOYYaN7c41i9OpkYsu2++7yEMKlk\nNJVwT5kCL7wQ//afeMJLQPv08RGqbEuNis2a5euOQcXIb4cOPgorIiKxyJnEyMw2M7MHzWyYmXVL\nOh6RgnP//V6Sk9QFLHj3rKuvhrfegv/8J95tP/mkl1QldQELFRexH38MTz+dTAzZtGKFj2QcdpiX\nEialSxdo0ABuuCHehLu83Es/994b2rePb7s11batr92UKkO9557sN4IQESkCGU2MzOw+M/vGzKau\n8fwxZvaxmX1qZqk6gI7AkyGEc4HjMhmXSNFZudIvYA8+2FscJ+n0033tpBtvjG+bIfj2mjTx+VNJ\n6tgRGjfOXGlXLnn0US8dTDIZBdhgA29AMH48vP56fNt9/nkv/ezd20emkmLmMXz4oY9gDRzoa4O1\nbJlcTCIiBSjTZ/oHgGPSnzCzesBg4FigKdDVzJoCOwFfRG8rkhoUkSx5/HEvxUn6AhZgo43gyit9\nxGj8+Hi2+cILXkqV9AUsQL16Pu/j3Xfh5ZeTjSWTVq/2ksFmzeDYY5OOxltWb7ttfAl3Ktlu2BA6\nd45nm3XRqRPsvjucfTZ8/XWyI78iIgXKQobvaJpZA2B0CGGf6PEhwHUhhKOjx6lagC+BhSGE0Wb2\neAihS1XbLikpCRMnTsxM4DXx8cdw2WVJRyFSuXff9YvGyZOTTxwAFi/2RUDr1/fRlbqaOtXL9KZP\n99GDpK1Y4RexIWS/+1+2LF4Mb7zhIxgnn5x0NG7AAB85atPGE9S6WL4cXnvNG5acf34s4dXZ8OFw\n7rk+UjRuXPI3OURE8oSZTQohlFT1vvWzEcwadqRiZAg8IWoJ3A7caWZtgUpbVpnZecB5ALvssksG\nw6yBVau805dIrmrUCK6/PjeSIoDNN/dyoFSXvLraeWe/IM6FpAhgww3h1lu9dXghnxu6doUTT0w6\nigoXXOCldN9+G8/2jj/eSz9zxamn+kjrRRcpKRIRyYAkRow6AUeHEM6JHp8KtAghXFLTbefMiJGI\niIiIiOSk6o4YJXH7+Etg57THOwFfJRCHiIiIiIgIkExi9Dawh5k1NLMNgS7AcwnEISIiIiIiAmS+\nXfdjwDhgTzP70szODiGsAi4GXgY+BEaGED7IZBwiIiIiIiLrktHmCyGErpU8/yLwYib3LSIiIiIi\nUl050qKqZsysvZkNLSsrSzoUEREREREpAHmZGIUQng8hnFe/fv2kQxERERERkQKQl4mRiIiIiIhI\nnJQYiYiIiIhI0VNiJCIiIiIiRU+JkYiIiIiIFD0lRiIiIiIiUvTyMjFSu24REREREYlTXiZGatct\nIiIiIiJxysvESEREREREJE5KjEREREREpOgpMRIRERERkaKnxEhERERERIqeEiMRERERESl66ycd\nQG2YWXugPfCDmU2v4+a2Ab6te1QiP9ExJZmg40ripmNKMkHHlcQtjmNq1+q8yUIIddxPfjOziSGE\nkqTjkMKhY0oyQceVxE3HlGSCjiuJWzaPKZXSiYiIiIhI0VNiJCIiIiIiRU+JEQxNOgApODqmJBN0\nXEncdExJJui4krhl7Zgq+jlGIiIiIiIiGjESEREREZGiV7SJkZkdY2Yfm9mnZtYr6Xgkf5jZzmY2\nxsw+NLMPzOyy6PmtzOxVM5sefd0yet7M7PboWHvfzA5I9hNIrjKzemb2rpmNjh43NLMJ0TH1hJlt\nGD2/UfT40+j1BknGLbnLzLYwsyfN7KPonHWIzlVSF2b25+h331Qze8zMNta5SmrKzO4zs2/MbGra\nczU+N5nZ6dH7p5vZ6XWNqygTIzOrBwwGjgWaAl3NrGmyUUkeWQVcEULYCzgYuCg6fnoB/w4h7AH8\nO3oMfpztEf05DxiS/ZAlT1wGfJj2+Cbg1uiYWgicHT1/NrAwhLA7cGv0PpG1uQ14KYTQBGiGH186\nV0mtmNmOwKVASQhhH6Ae0AWdq6TmHgCOWeO5Gp2bzGwr4FqgJdACuDaVTNVWUSZG+A/v0xDCjBDC\nCuBxoEPCMUmeCCHMDSG8E/19EX6hsSN+DD0Yve1B4Pjo7x2Ah4IbD2xhZttnOWzJcWa2E9AWGB49\nNqA18GT0ljWPqdSx9iTw++j9Ij8xs18DvwPuBQghrAghfI/OVVI36wObmNn6wKbAXHSukhoKIfwX\nWLDG0zU9Nx0NvBpCWBBCWAi8yi+TrRop1sRoR+CLtMdfRs+J1EhUFrA/MAHYNoQwFzx5An4bvU3H\nm1THIOAqoDx6vDXwfQhhVfQ4/bj56ZiKXi+L3i+SbjdgPnB/VKI53Mw2Q+cqqaUQwhzgZuBzPCEq\nAyahc5XEo6bnptjPWcWaGK3tboXa80mNmNnmwD+By0MIP6zrrWt5Tseb/MTM2gHfhBAmpT+9lreG\narwmkrI+cAAwJISwP7CEitKUtdFxJesUlSl1ABoCOwCb4WVOa9K5SuJU2XEU+/FVrInRl8DOaY93\nAr5KKBbJQ2a2AZ4UPRJCeCp6el6q7CT6+k30vI43qcphwHFmNgsv7W2NjyBtEZWrwM+Pm5+Oqej1\n+vyyJEHkS+DLEMKE6PGTeKKkc5XUVhtgZghhfghhJfAUcCg6V0k8anpuiv2cVayJ0dvAHlEXlQ3x\niYPPJRyT5ImoPvpe4MMQwi1pLz0HpDqinA48m/b8aVFXlYOBstRQsQhACKF3CGGnEEID/Hz0nxBC\nN2AMcFL0tjWPqdSxdlL0ft2FlZ8JIXwNfGFme0ZP/R6Yhs5VUnufAweb2abR78LUMaVzlcShpuem\nl4GjzGzLaDTzqOi5WivaBV7N7I/4Hdl6wH0hhBsSDknyhJkdDowFplAxH6QPPs9oJLAL/sujUwhh\nQfTL4058QuBS4MwQwsSsBy55wcxaAVeGENqZ2W74CNJWwLtA9xDCcjPbGBiBz29bAHQJIcxIKmbJ\nXWbWHG/osSEwAzgTvymqc5XUipldD3TGO7S+C5yDz+vQuUqqzcweA1oB2wDz8O5yz1DDc5OZnYVf\ngwHcEEK4v05xFWtiJCIiIiIiklKspXQiIiIiIiI/UWIkIiIiIiJFT4mRiIiIiIgUPSVGIiIiIiJS\n9JQYiYiIiIhI0VNiJCIiIiIiRU+JkYhIjjOzYGYD0x5faWbXxbTtB8zspKrfWef9dDKzD81szFpe\nG2BmH5jZgEzHURNmtr2ZjY7+3jxa/y712nVmdmVy0a2bmZ1hZneu4/V9zeyBLIYkIpLzlBiJiOS+\n5UBHM9sm6UDSmVm9Grz9bOBPIYQj1/La+cABIYSea2x//brEF4MewLDo782BP67jvXklhDAF2MnM\ndkk6FhGRXKHESEQk960ChgJ/XvOFNUd8zGxx9LWVmb1uZiPN7BMz62dm3czsLTObYmaN0jbTxszG\nRu9rF31/vWgk520ze9/Mzk/b7hgzexSYspZ4ukbbn2pmN0XP/RU4HLh7zVEhM3sO2AyYYGado89z\nSzSydJOZtTCzN83s3ejrntH3nWFmz5jZ82Y208wuNrMe0fvGm9lW0fsamdlLZjYp+oxNouc7RTG+\nZ2b/reTnfiLwkpltCPwN6Gxmk82sc/R6UzN7zcxmmNmlaZ+pR7TtqWZ2efRcAzObmvaen0b9zOxS\nM5sW/Zwfj55b1+d+KvpM082sf9o2z4z+DV8HDkt7vrLP+jzQpZLPLiJSdJK+GyciItUzGHg//UK4\nGpoBewELgBnA8BBCCzO7DLgEuDx6XwPgCKARMMbMdgdOA8pCCAeZ2UbAG2b2SvT+FsA+IYSZ6Tsz\nsx2Am4ADgYXAK2Z2fAjhb2bWGrgyhDAx/XtCCMeZ2eIQQvNoG8cCjYE2IYTVZvZr4HchhFVm1ga4\nEU9YAPYB9gc2Bj4Frg4h7G9mt0bxD8ITygtCCNPNrCVwF9Aa+CtwdAhhjpltseYPzswaAgtDCMuj\nx38FSkIIF0ePrwOaAEcCvwI+NrMhwH7AmUBLwPCE7/Xo51GZXkDDEMLytFg+Wsfnbh597uXRfu/A\nk+fro599GTAGeDd6f2WfdWK075ocUyIiBUuJkYhIHggh/GBmDwGXAj9W89veDiHMBTCzz4BUYjMF\nv6BPGRlCKAemm9kM/IL/KGC/tNGo+sAewArgrTWToshBwGshhPnRPh8Bfgc8U814U0aFEFan7fdB\nM9sDCMAGae8bE0JYBCwyszJ8BCT1+fYzs82BQ4FRZpb6no2ir28AD5jZSOCptcSwPTC/ijhfiBKn\n5Wb2DbAtPjL2dAhhCYCZPQWUAs+tYzvvA4+Y2TNU/KzW9bn/HUIoi7Y/DdgV2Iaf/+yfwBPMdX3W\nb4AdqviMIiJFQ6V0IiL5YxA+V2eztOdWEZ3Lza/+N0x7bXna38vTHpfz8xtjYY39BHy045IQQvPo\nT8MQQiqxWlJJfFbJ8zWVvv2/4wnQPkB7fHQoparPtx7wfdpnaB5C2AsghHABcA2wMzDZzLZeI4Yf\n19jX2qTvf3W0z8p+Bj/9O0XSt90WHxE8EJhkPrequp87tV/45b+jP1n5Z92Y6ifZIiIFT4mRiEie\nCCEsAEbiyVHKLPyCGqADPx9ZqK5OZrae+byj3YCPgZeBC81sAwAza2xmm61rI8AE4Agz28a8MUNX\n4PVaxJOuPjAn+vsZNfnGEMIPwEwz6wSeOJpZs+jvjUIIE0IIfwW+xZOGdJ/gJYYpi/CSuar8Fzje\nzDaNfl4nAGOBecBvzWzrqDQxNZdrPWDnEMIY4CpgC2DzWnzuCUCraPsbAJ1SL6zjszYGpv5yUyIi\nxUmJkYhIfhmIl02lDMOTkbfweS2Vjeasy8d4AvN/+HycZcBwYBrwTtQ04B6qKL+OyvZ64/Nb3gPe\nCSE8W4t40vUH+prZG0BNuuCldAPONrP3gA/w5BFggEVNIvBk5r30b4pK4T6L5luBf6amazRf+IUQ\nwjvAA8BbeLIyPITwbghhJd7AYQIwGp9DRPSZHjazKficoFtDCN/X9HNHP/vrgHHAv4B30l6u7LMe\nCbxQ1bZFRIqFhbDWkXcREZGiZmYnAAeGEK5JOpa4RaNWrwOHhxBWJR2PiEguUPMFERGRtQghPL2W\nuUeFYhegl5IiEZEKGjESEREREZGipzlGIiIiIiJS9JQYiYiIiIhI0VNiJCIiIiIiRU+JkYiIiIiI\nFD0lRiIiIiIiUvT+H083Te+O+n4fAAAAAElFTkSuQmCC\n", + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "x = np.linspace(25,25*len(maxs),len(maxs))\n", + "plt.figure(figsize=(14,8))\n", + "plt.semilogy(x, maxs, 'g', label='Maximum')\n", + "plt.semilogy(x, means, 'b', label='Average')\n", + "plt.semilogy(x, mins, 'r', label='Minimum')\n", + "plt.semilogy(x, np.ones(len(x))*(15+scoreOffset), 'k', label='Objective')\n", + "plt.xlabel(\"Number of frames (thousands)\")\n", + "plt.ylabel(\"Log(score+6)\")\n", + "plt.legend()\n", + "_ = plt.title(\"FlappyAgent learning curves\")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.4" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/Durivaux/MemoryBuffer.py b/Durivaux/MemoryBuffer.py new file mode 100644 index 0000000..0bd8ed4 --- /dev/null +++ b/Durivaux/MemoryBuffer.py @@ -0,0 +1,61 @@ +import numpy as np +from collections import deque + +class MemoryBuffer: + """ + An experience replay buffer using numpy arrays + """ + def __init__(self, length, screen_shape, action_shape): + self.length = length + self.screen_shape = screen_shape + self.action_shape = action_shape + shape = (length,) + screen_shape + self.screens_x = np.zeros(shape, dtype=np.uint8) # starting states + self.screens_y = np.zeros(shape, dtype=np.uint8) # resulting states + shape = (length,) + action_shape + self.actions = np.zeros(shape, dtype=np.uint8) # actions + self.rewards = np.zeros((length,1), dtype=np.float64) # rewards #was uint8 + self.terminals = np.zeros((length,1), dtype=np.bool) # true if resulting state is terminal + self.terminals[-1] = True + self.index = 0 # points one position past the last inserted element + self.size = 0 # current size of the buffer + + def append(self, screenx, a, r, screeny, d): + self.screens_x[self.index] = screenx + self.actions[self.index] = a + self.rewards[self.index] = r + self.screens_y[self.index] = screeny + self.terminals[self.index] = d + self.index = (self.index+1) % self.length + self.size = np.min([self.size+1,self.length]) + + def stacked_frames_x(self, index): + im_deque = deque(maxlen=4) + pos = index % self.length + for i in range(4): + im = self.screens_x[pos] + im_deque.appendleft(im) + test_pos = (pos-1) % self.length + if self.terminals[test_pos] == False: + pos = test_pos + return np.stack(im_deque, axis=-1) + + def stacked_frames_y(self, index): + im_deque = deque(maxlen=4) + pos = index % self.length + for i in range(4): + im = self.screens_y[pos] + im_deque.appendleft(im) + test_pos = (pos-1) % self.length + if self.terminals[test_pos] == False: + pos = test_pos + return np.stack(im_deque, axis=-1) + + def minibatch(self, size): + indices = np.random.choice(self.size, size=size, replace=False) + x = np.zeros((size,)+self.screen_shape+(4,)) + y = np.zeros((size,)+self.screen_shape+(4,)) + for i in range(size): + x[i] = self.stacked_frames_x(indices[i]) + y[i] = self.stacked_frames_y(indices[i]) + return x, self.actions[indices], self.rewards[indices], y, self.terminals[indices] diff --git a/Durivaux/README.md b/Durivaux/README.md new file mode 100644 index 0000000..bfc64c6 --- /dev/null +++ b/Durivaux/README.md @@ -0,0 +1,24 @@ +# Deep Q-learning for FlappyBird agent + +Implementation of a deep Q-learning method for a pixel-based agent with no prior knowledge. + +This work is based on Emmanuel Rachelson's Machine Learning classes (ISAE-Supaéro 2017-2018), alongside [this article](https://www.nature.com/articles/nature14236). + +This particular implementation has the following features: +* the agent only chooses how to act every 2 frames, and repeats this action the next frame +* two neural networks are used: the usual one and a target-generating one, with regular (every 2500 frames) weights transfers between the first one and the second one +* training of the network is done every 5 frames for speed of training +* training on minibatches (size: 32 frames) +* replay memory (unlimited) +* initial exploration, then (decreasing) epsilon-greedy actions +* regular backup of the network: ability to choose the best one (based on learning curves) + +# Results + +Depending on the parameters, the target score of 15 can be reached in less than 200k frames. The proposed solution here took 925k frames for training, but reaches a much better average. + +![learning](./learning.png) + +Computation time, including lengthy evaluation periods: 6.5 hours (i7-4790K, GTX770, 16 GiB of RAM) + +Over 100 games: average of 116.16, with a maximum of 466. diff --git a/Durivaux/dqn-925k.h5 b/Durivaux/dqn-925k.h5 new file mode 100644 index 0000000..61dbe4b Binary files /dev/null and b/Durivaux/dqn-925k.h5 differ diff --git a/Durivaux/learning.png b/Durivaux/learning.png new file mode 100644 index 0000000..b11023d Binary files /dev/null and b/Durivaux/learning.png differ diff --git a/PyGame-Learning-Environment/ple/__init__.py b/Durivaux/ple/__init__.py similarity index 100% rename from PyGame-Learning-Environment/ple/__init__.py rename to Durivaux/ple/__init__.py diff --git a/PyGame-Learning-Environment/ple/games/__init__.py b/Durivaux/ple/games/__init__.py similarity index 100% rename from PyGame-Learning-Environment/ple/games/__init__.py rename to Durivaux/ple/games/__init__.py diff --git a/PyGame-Learning-Environment/ple/games/base/__init__.py b/Durivaux/ple/games/base/__init__.py similarity index 100% rename from PyGame-Learning-Environment/ple/games/base/__init__.py rename to Durivaux/ple/games/base/__init__.py diff --git a/PyGame-Learning-Environment/ple/games/base/doomwrapper.py b/Durivaux/ple/games/base/doomwrapper.py similarity index 100% rename from PyGame-Learning-Environment/ple/games/base/doomwrapper.py rename to Durivaux/ple/games/base/doomwrapper.py diff --git a/PyGame-Learning-Environment/ple/games/base/pygamewrapper.py b/Durivaux/ple/games/base/pygamewrapper.py similarity index 100% rename from PyGame-Learning-Environment/ple/games/base/pygamewrapper.py rename to Durivaux/ple/games/base/pygamewrapper.py diff --git a/PyGame-Learning-Environment/ple/games/catcher.py b/Durivaux/ple/games/catcher.py similarity index 100% rename from PyGame-Learning-Environment/ple/games/catcher.py rename to Durivaux/ple/games/catcher.py diff --git a/PyGame-Learning-Environment/ple/games/doom/__init__.py b/Durivaux/ple/games/doom/__init__.py similarity index 100% rename from PyGame-Learning-Environment/ple/games/doom/__init__.py rename to Durivaux/ple/games/doom/__init__.py diff --git a/PyGame-Learning-Environment/ple/games/doom/assets/README.md b/Durivaux/ple/games/doom/assets/README.md similarity index 100% rename from PyGame-Learning-Environment/ple/games/doom/assets/README.md rename to Durivaux/ple/games/doom/assets/README.md diff --git a/PyGame-Learning-Environment/ple/games/doom/assets/cfg/basic.cfg b/Durivaux/ple/games/doom/assets/cfg/basic.cfg similarity index 100% rename from PyGame-Learning-Environment/ple/games/doom/assets/cfg/basic.cfg rename to Durivaux/ple/games/doom/assets/cfg/basic.cfg diff --git a/PyGame-Learning-Environment/ple/games/doom/assets/cfg/deadly_corridor.cfg b/Durivaux/ple/games/doom/assets/cfg/deadly_corridor.cfg similarity index 100% rename from PyGame-Learning-Environment/ple/games/doom/assets/cfg/deadly_corridor.cfg rename to Durivaux/ple/games/doom/assets/cfg/deadly_corridor.cfg diff --git a/PyGame-Learning-Environment/ple/games/doom/assets/cfg/deathmatch.cfg b/Durivaux/ple/games/doom/assets/cfg/deathmatch.cfg similarity index 100% rename from PyGame-Learning-Environment/ple/games/doom/assets/cfg/deathmatch.cfg rename to Durivaux/ple/games/doom/assets/cfg/deathmatch.cfg diff --git a/PyGame-Learning-Environment/ple/games/doom/assets/cfg/defend_the_center.cfg b/Durivaux/ple/games/doom/assets/cfg/defend_the_center.cfg similarity index 100% rename from PyGame-Learning-Environment/ple/games/doom/assets/cfg/defend_the_center.cfg rename to Durivaux/ple/games/doom/assets/cfg/defend_the_center.cfg diff --git a/PyGame-Learning-Environment/ple/games/doom/assets/cfg/defend_the_line.cfg b/Durivaux/ple/games/doom/assets/cfg/defend_the_line.cfg similarity index 100% rename from PyGame-Learning-Environment/ple/games/doom/assets/cfg/defend_the_line.cfg rename to Durivaux/ple/games/doom/assets/cfg/defend_the_line.cfg diff --git a/PyGame-Learning-Environment/ple/games/doom/assets/cfg/health_gathering.cfg b/Durivaux/ple/games/doom/assets/cfg/health_gathering.cfg similarity index 100% rename from PyGame-Learning-Environment/ple/games/doom/assets/cfg/health_gathering.cfg rename to Durivaux/ple/games/doom/assets/cfg/health_gathering.cfg diff --git a/PyGame-Learning-Environment/ple/games/doom/assets/cfg/my_way_home.cfg b/Durivaux/ple/games/doom/assets/cfg/my_way_home.cfg similarity index 100% rename from PyGame-Learning-Environment/ple/games/doom/assets/cfg/my_way_home.cfg rename to Durivaux/ple/games/doom/assets/cfg/my_way_home.cfg diff --git a/PyGame-Learning-Environment/ple/games/doom/assets/cfg/predict_position.cfg b/Durivaux/ple/games/doom/assets/cfg/predict_position.cfg similarity index 100% rename from PyGame-Learning-Environment/ple/games/doom/assets/cfg/predict_position.cfg rename to Durivaux/ple/games/doom/assets/cfg/predict_position.cfg diff --git a/PyGame-Learning-Environment/ple/games/doom/assets/cfg/take_cover.cfg b/Durivaux/ple/games/doom/assets/cfg/take_cover.cfg similarity index 100% rename from PyGame-Learning-Environment/ple/games/doom/assets/cfg/take_cover.cfg rename to Durivaux/ple/games/doom/assets/cfg/take_cover.cfg diff --git a/PyGame-Learning-Environment/ple/games/doom/doom.py b/Durivaux/ple/games/doom/doom.py similarity index 100% rename from PyGame-Learning-Environment/ple/games/doom/doom.py rename to Durivaux/ple/games/doom/doom.py diff --git a/PyGame-Learning-Environment/ple/games/flappybird/__init__.py b/Durivaux/ple/games/flappybird/__init__.py similarity index 100% rename from PyGame-Learning-Environment/ple/games/flappybird/__init__.py rename to Durivaux/ple/games/flappybird/__init__.py diff --git a/PyGame-Learning-Environment/ple/games/flappybird/assets/background-blank.png b/Durivaux/ple/games/flappybird/assets/background-blank.png similarity index 100% rename from PyGame-Learning-Environment/ple/games/flappybird/assets/background-blank.png rename to Durivaux/ple/games/flappybird/assets/background-blank.png diff --git a/PyGame-Learning-Environment/ple/games/flappybird/assets/background-day.png b/Durivaux/ple/games/flappybird/assets/background-day.png similarity index 100% rename from PyGame-Learning-Environment/ple/games/flappybird/assets/background-day.png rename to Durivaux/ple/games/flappybird/assets/background-day.png diff --git a/PyGame-Learning-Environment/ple/games/flappybird/assets/background-night.png b/Durivaux/ple/games/flappybird/assets/background-night.png similarity index 100% rename from PyGame-Learning-Environment/ple/games/flappybird/assets/background-night.png rename to Durivaux/ple/games/flappybird/assets/background-night.png diff --git a/PyGame-Learning-Environment/ple/games/flappybird/assets/base.png b/Durivaux/ple/games/flappybird/assets/base.png similarity index 100% rename from PyGame-Learning-Environment/ple/games/flappybird/assets/base.png rename to Durivaux/ple/games/flappybird/assets/base.png diff --git a/PyGame-Learning-Environment/ple/games/flappybird/assets/bluebird-downflap.png b/Durivaux/ple/games/flappybird/assets/bluebird-downflap.png similarity index 100% rename from PyGame-Learning-Environment/ple/games/flappybird/assets/bluebird-downflap.png rename to Durivaux/ple/games/flappybird/assets/bluebird-downflap.png diff --git a/PyGame-Learning-Environment/ple/games/flappybird/assets/bluebird-midflap.png b/Durivaux/ple/games/flappybird/assets/bluebird-midflap.png similarity index 100% rename from PyGame-Learning-Environment/ple/games/flappybird/assets/bluebird-midflap.png rename to Durivaux/ple/games/flappybird/assets/bluebird-midflap.png diff --git a/PyGame-Learning-Environment/ple/games/flappybird/assets/bluebird-upflap.png b/Durivaux/ple/games/flappybird/assets/bluebird-upflap.png similarity index 100% rename from PyGame-Learning-Environment/ple/games/flappybird/assets/bluebird-upflap.png rename to Durivaux/ple/games/flappybird/assets/bluebird-upflap.png diff --git a/PyGame-Learning-Environment/ple/games/flappybird/assets/pipe-green.png b/Durivaux/ple/games/flappybird/assets/pipe-green.png similarity index 100% rename from PyGame-Learning-Environment/ple/games/flappybird/assets/pipe-green.png rename to Durivaux/ple/games/flappybird/assets/pipe-green.png diff --git a/PyGame-Learning-Environment/ple/games/flappybird/assets/pipe-red.png b/Durivaux/ple/games/flappybird/assets/pipe-red.png similarity index 100% rename from PyGame-Learning-Environment/ple/games/flappybird/assets/pipe-red.png rename to Durivaux/ple/games/flappybird/assets/pipe-red.png diff --git a/PyGame-Learning-Environment/ple/games/flappybird/assets/redbird-downflap.png b/Durivaux/ple/games/flappybird/assets/redbird-downflap.png similarity index 100% rename from PyGame-Learning-Environment/ple/games/flappybird/assets/redbird-downflap.png rename to Durivaux/ple/games/flappybird/assets/redbird-downflap.png diff --git a/PyGame-Learning-Environment/ple/games/flappybird/assets/redbird-midflap.png b/Durivaux/ple/games/flappybird/assets/redbird-midflap.png similarity index 100% rename from PyGame-Learning-Environment/ple/games/flappybird/assets/redbird-midflap.png rename to Durivaux/ple/games/flappybird/assets/redbird-midflap.png diff --git a/PyGame-Learning-Environment/ple/games/flappybird/assets/redbird-upflap.png b/Durivaux/ple/games/flappybird/assets/redbird-upflap.png similarity index 100% rename from PyGame-Learning-Environment/ple/games/flappybird/assets/redbird-upflap.png rename to Durivaux/ple/games/flappybird/assets/redbird-upflap.png diff --git a/PyGame-Learning-Environment/ple/games/flappybird/assets/yellowbird-downflap.png b/Durivaux/ple/games/flappybird/assets/yellowbird-downflap.png similarity index 100% rename from PyGame-Learning-Environment/ple/games/flappybird/assets/yellowbird-downflap.png rename to Durivaux/ple/games/flappybird/assets/yellowbird-downflap.png diff --git a/PyGame-Learning-Environment/ple/games/flappybird/assets/yellowbird-midflap.png b/Durivaux/ple/games/flappybird/assets/yellowbird-midflap.png similarity index 100% rename from PyGame-Learning-Environment/ple/games/flappybird/assets/yellowbird-midflap.png rename to Durivaux/ple/games/flappybird/assets/yellowbird-midflap.png diff --git a/PyGame-Learning-Environment/ple/games/flappybird/assets/yellowbird-upflap.png b/Durivaux/ple/games/flappybird/assets/yellowbird-upflap.png similarity index 100% rename from PyGame-Learning-Environment/ple/games/flappybird/assets/yellowbird-upflap.png rename to Durivaux/ple/games/flappybird/assets/yellowbird-upflap.png diff --git a/PyGame-Learning-Environment/ple/games/monsterkong/LICENSE b/Durivaux/ple/games/monsterkong/LICENSE similarity index 100% rename from PyGame-Learning-Environment/ple/games/monsterkong/LICENSE rename to Durivaux/ple/games/monsterkong/LICENSE diff --git a/PyGame-Learning-Environment/ple/games/monsterkong/__init__.py b/Durivaux/ple/games/monsterkong/__init__.py similarity index 100% rename from PyGame-Learning-Environment/ple/games/monsterkong/__init__.py rename to Durivaux/ple/games/monsterkong/__init__.py diff --git a/PyGame-Learning-Environment/ple/games/monsterkong/assets/asset_credits.txt b/Durivaux/ple/games/monsterkong/assets/asset_credits.txt similarity index 100% rename from PyGame-Learning-Environment/ple/games/monsterkong/assets/asset_credits.txt rename to Durivaux/ple/games/monsterkong/assets/asset_credits.txt diff --git a/PyGame-Learning-Environment/ple/games/monsterkong/assets/background.png b/Durivaux/ple/games/monsterkong/assets/background.png similarity index 100% rename from PyGame-Learning-Environment/ple/games/monsterkong/assets/background.png rename to Durivaux/ple/games/monsterkong/assets/background.png diff --git a/PyGame-Learning-Environment/ple/games/monsterkong/assets/coin1.png b/Durivaux/ple/games/monsterkong/assets/coin1.png similarity index 100% rename from PyGame-Learning-Environment/ple/games/monsterkong/assets/coin1.png rename to Durivaux/ple/games/monsterkong/assets/coin1.png diff --git a/PyGame-Learning-Environment/ple/games/monsterkong/assets/coin2.png b/Durivaux/ple/games/monsterkong/assets/coin2.png similarity index 100% rename from PyGame-Learning-Environment/ple/games/monsterkong/assets/coin2.png rename to Durivaux/ple/games/monsterkong/assets/coin2.png diff --git a/PyGame-Learning-Environment/ple/games/monsterkong/assets/coin3.png b/Durivaux/ple/games/monsterkong/assets/coin3.png similarity index 100% rename from PyGame-Learning-Environment/ple/games/monsterkong/assets/coin3.png rename to Durivaux/ple/games/monsterkong/assets/coin3.png diff --git a/PyGame-Learning-Environment/ple/games/monsterkong/assets/coin4.png b/Durivaux/ple/games/monsterkong/assets/coin4.png similarity index 100% rename from PyGame-Learning-Environment/ple/games/monsterkong/assets/coin4.png rename to Durivaux/ple/games/monsterkong/assets/coin4.png diff --git a/PyGame-Learning-Environment/ple/games/monsterkong/assets/coin5.png b/Durivaux/ple/games/monsterkong/assets/coin5.png similarity index 100% rename from PyGame-Learning-Environment/ple/games/monsterkong/assets/coin5.png rename to Durivaux/ple/games/monsterkong/assets/coin5.png diff --git a/PyGame-Learning-Environment/ple/games/monsterkong/assets/fireballdown.png b/Durivaux/ple/games/monsterkong/assets/fireballdown.png similarity index 100% rename from PyGame-Learning-Environment/ple/games/monsterkong/assets/fireballdown.png rename to Durivaux/ple/games/monsterkong/assets/fireballdown.png diff --git a/PyGame-Learning-Environment/ple/games/monsterkong/assets/fireballleft.png b/Durivaux/ple/games/monsterkong/assets/fireballleft.png similarity index 100% rename from PyGame-Learning-Environment/ple/games/monsterkong/assets/fireballleft.png rename to Durivaux/ple/games/monsterkong/assets/fireballleft.png diff --git a/PyGame-Learning-Environment/ple/games/monsterkong/assets/fireballright.png b/Durivaux/ple/games/monsterkong/assets/fireballright.png similarity index 100% rename from PyGame-Learning-Environment/ple/games/monsterkong/assets/fireballright.png rename to Durivaux/ple/games/monsterkong/assets/fireballright.png diff --git a/PyGame-Learning-Environment/ple/games/monsterkong/assets/ladder.png b/Durivaux/ple/games/monsterkong/assets/ladder.png similarity index 100% rename from PyGame-Learning-Environment/ple/games/monsterkong/assets/ladder.png rename to Durivaux/ple/games/monsterkong/assets/ladder.png diff --git a/PyGame-Learning-Environment/ple/games/monsterkong/assets/left.png b/Durivaux/ple/games/monsterkong/assets/left.png similarity index 100% rename from PyGame-Learning-Environment/ple/games/monsterkong/assets/left.png rename to Durivaux/ple/games/monsterkong/assets/left.png diff --git a/PyGame-Learning-Environment/ple/games/monsterkong/assets/left2.png b/Durivaux/ple/games/monsterkong/assets/left2.png similarity index 100% rename from PyGame-Learning-Environment/ple/games/monsterkong/assets/left2.png rename to Durivaux/ple/games/monsterkong/assets/left2.png diff --git a/PyGame-Learning-Environment/ple/games/monsterkong/assets/monster0.png b/Durivaux/ple/games/monsterkong/assets/monster0.png similarity index 100% rename from PyGame-Learning-Environment/ple/games/monsterkong/assets/monster0.png rename to Durivaux/ple/games/monsterkong/assets/monster0.png diff --git a/PyGame-Learning-Environment/ple/games/monsterkong/assets/monster01.png b/Durivaux/ple/games/monsterkong/assets/monster01.png similarity index 100% rename from PyGame-Learning-Environment/ple/games/monsterkong/assets/monster01.png rename to Durivaux/ple/games/monsterkong/assets/monster01.png diff --git a/PyGame-Learning-Environment/ple/games/monsterkong/assets/monster1.png b/Durivaux/ple/games/monsterkong/assets/monster1.png similarity index 100% rename from PyGame-Learning-Environment/ple/games/monsterkong/assets/monster1.png rename to Durivaux/ple/games/monsterkong/assets/monster1.png diff --git a/PyGame-Learning-Environment/ple/games/monsterkong/assets/monster11.png b/Durivaux/ple/games/monsterkong/assets/monster11.png similarity index 100% rename from PyGame-Learning-Environment/ple/games/monsterkong/assets/monster11.png rename to Durivaux/ple/games/monsterkong/assets/monster11.png diff --git a/PyGame-Learning-Environment/ple/games/monsterkong/assets/monster2.png b/Durivaux/ple/games/monsterkong/assets/monster2.png similarity index 100% rename from PyGame-Learning-Environment/ple/games/monsterkong/assets/monster2.png rename to Durivaux/ple/games/monsterkong/assets/monster2.png diff --git a/PyGame-Learning-Environment/ple/games/monsterkong/assets/monster21.png b/Durivaux/ple/games/monsterkong/assets/monster21.png similarity index 100% rename from PyGame-Learning-Environment/ple/games/monsterkong/assets/monster21.png rename to Durivaux/ple/games/monsterkong/assets/monster21.png diff --git a/PyGame-Learning-Environment/ple/games/monsterkong/assets/monster3.png b/Durivaux/ple/games/monsterkong/assets/monster3.png similarity index 100% rename from PyGame-Learning-Environment/ple/games/monsterkong/assets/monster3.png rename to Durivaux/ple/games/monsterkong/assets/monster3.png diff --git a/PyGame-Learning-Environment/ple/games/monsterkong/assets/monster31.png b/Durivaux/ple/games/monsterkong/assets/monster31.png similarity index 100% rename from PyGame-Learning-Environment/ple/games/monsterkong/assets/monster31.png rename to Durivaux/ple/games/monsterkong/assets/monster31.png diff --git a/PyGame-Learning-Environment/ple/games/monsterkong/assets/monsterstill0.png b/Durivaux/ple/games/monsterkong/assets/monsterstill0.png similarity index 100% rename from PyGame-Learning-Environment/ple/games/monsterkong/assets/monsterstill0.png rename to Durivaux/ple/games/monsterkong/assets/monsterstill0.png diff --git a/PyGame-Learning-Environment/ple/games/monsterkong/assets/monsterstill1.png b/Durivaux/ple/games/monsterkong/assets/monsterstill1.png similarity index 100% rename from PyGame-Learning-Environment/ple/games/monsterkong/assets/monsterstill1.png rename to Durivaux/ple/games/monsterkong/assets/monsterstill1.png diff --git a/PyGame-Learning-Environment/ple/games/monsterkong/assets/monsterstill10.png b/Durivaux/ple/games/monsterkong/assets/monsterstill10.png similarity index 100% rename from PyGame-Learning-Environment/ple/games/monsterkong/assets/monsterstill10.png rename to Durivaux/ple/games/monsterkong/assets/monsterstill10.png diff --git a/PyGame-Learning-Environment/ple/games/monsterkong/assets/monsterstill11.png b/Durivaux/ple/games/monsterkong/assets/monsterstill11.png similarity index 100% rename from PyGame-Learning-Environment/ple/games/monsterkong/assets/monsterstill11.png rename to Durivaux/ple/games/monsterkong/assets/monsterstill11.png diff --git a/PyGame-Learning-Environment/ple/games/monsterkong/assets/princess.png b/Durivaux/ple/games/monsterkong/assets/princess.png similarity index 100% rename from PyGame-Learning-Environment/ple/games/monsterkong/assets/princess.png rename to Durivaux/ple/games/monsterkong/assets/princess.png diff --git a/PyGame-Learning-Environment/ple/games/monsterkong/assets/right.png b/Durivaux/ple/games/monsterkong/assets/right.png similarity index 100% rename from PyGame-Learning-Environment/ple/games/monsterkong/assets/right.png rename to Durivaux/ple/games/monsterkong/assets/right.png diff --git a/PyGame-Learning-Environment/ple/games/monsterkong/assets/right2.png b/Durivaux/ple/games/monsterkong/assets/right2.png similarity index 100% rename from PyGame-Learning-Environment/ple/games/monsterkong/assets/right2.png rename to Durivaux/ple/games/monsterkong/assets/right2.png diff --git a/PyGame-Learning-Environment/ple/games/monsterkong/assets/still.png b/Durivaux/ple/games/monsterkong/assets/still.png similarity index 100% rename from PyGame-Learning-Environment/ple/games/monsterkong/assets/still.png rename to Durivaux/ple/games/monsterkong/assets/still.png diff --git a/PyGame-Learning-Environment/ple/games/monsterkong/assets/wood_block.png b/Durivaux/ple/games/monsterkong/assets/wood_block.png similarity index 100% rename from PyGame-Learning-Environment/ple/games/monsterkong/assets/wood_block.png rename to Durivaux/ple/games/monsterkong/assets/wood_block.png diff --git a/PyGame-Learning-Environment/ple/games/monsterkong/board.py b/Durivaux/ple/games/monsterkong/board.py similarity index 100% rename from PyGame-Learning-Environment/ple/games/monsterkong/board.py rename to Durivaux/ple/games/monsterkong/board.py diff --git a/PyGame-Learning-Environment/ple/games/monsterkong/coin.py b/Durivaux/ple/games/monsterkong/coin.py similarity index 100% rename from PyGame-Learning-Environment/ple/games/monsterkong/coin.py rename to Durivaux/ple/games/monsterkong/coin.py diff --git a/PyGame-Learning-Environment/ple/games/monsterkong/fireball.py b/Durivaux/ple/games/monsterkong/fireball.py similarity index 100% rename from PyGame-Learning-Environment/ple/games/monsterkong/fireball.py rename to Durivaux/ple/games/monsterkong/fireball.py diff --git a/PyGame-Learning-Environment/ple/games/monsterkong/ladder.py b/Durivaux/ple/games/monsterkong/ladder.py similarity index 100% rename from PyGame-Learning-Environment/ple/games/monsterkong/ladder.py rename to Durivaux/ple/games/monsterkong/ladder.py diff --git a/PyGame-Learning-Environment/ple/games/monsterkong/monsterPerson.py b/Durivaux/ple/games/monsterkong/monsterPerson.py similarity index 100% rename from PyGame-Learning-Environment/ple/games/monsterkong/monsterPerson.py rename to Durivaux/ple/games/monsterkong/monsterPerson.py diff --git a/PyGame-Learning-Environment/ple/games/monsterkong/onBoard.py b/Durivaux/ple/games/monsterkong/onBoard.py similarity index 100% rename from PyGame-Learning-Environment/ple/games/monsterkong/onBoard.py rename to Durivaux/ple/games/monsterkong/onBoard.py diff --git a/PyGame-Learning-Environment/ple/games/monsterkong/person.py b/Durivaux/ple/games/monsterkong/person.py similarity index 100% rename from PyGame-Learning-Environment/ple/games/monsterkong/person.py rename to Durivaux/ple/games/monsterkong/person.py diff --git a/PyGame-Learning-Environment/ple/games/monsterkong/player.py b/Durivaux/ple/games/monsterkong/player.py similarity index 100% rename from PyGame-Learning-Environment/ple/games/monsterkong/player.py rename to Durivaux/ple/games/monsterkong/player.py diff --git a/PyGame-Learning-Environment/ple/games/monsterkong/wall.py b/Durivaux/ple/games/monsterkong/wall.py similarity index 100% rename from PyGame-Learning-Environment/ple/games/monsterkong/wall.py rename to Durivaux/ple/games/monsterkong/wall.py diff --git a/PyGame-Learning-Environment/ple/games/pixelcopter.py b/Durivaux/ple/games/pixelcopter.py similarity index 100% rename from PyGame-Learning-Environment/ple/games/pixelcopter.py rename to Durivaux/ple/games/pixelcopter.py diff --git a/PyGame-Learning-Environment/ple/games/pong.py b/Durivaux/ple/games/pong.py similarity index 100% rename from PyGame-Learning-Environment/ple/games/pong.py rename to Durivaux/ple/games/pong.py diff --git a/PyGame-Learning-Environment/ple/games/primitives.py b/Durivaux/ple/games/primitives.py similarity index 100% rename from PyGame-Learning-Environment/ple/games/primitives.py rename to Durivaux/ple/games/primitives.py diff --git a/PyGame-Learning-Environment/ple/games/puckworld.py b/Durivaux/ple/games/puckworld.py similarity index 100% rename from PyGame-Learning-Environment/ple/games/puckworld.py rename to Durivaux/ple/games/puckworld.py diff --git a/PyGame-Learning-Environment/ple/games/raycast.py b/Durivaux/ple/games/raycast.py similarity index 100% rename from PyGame-Learning-Environment/ple/games/raycast.py rename to Durivaux/ple/games/raycast.py diff --git a/PyGame-Learning-Environment/ple/games/raycastmaze.py b/Durivaux/ple/games/raycastmaze.py similarity index 100% rename from PyGame-Learning-Environment/ple/games/raycastmaze.py rename to Durivaux/ple/games/raycastmaze.py diff --git a/PyGame-Learning-Environment/ple/games/snake.py b/Durivaux/ple/games/snake.py similarity index 100% rename from PyGame-Learning-Environment/ple/games/snake.py rename to Durivaux/ple/games/snake.py diff --git a/PyGame-Learning-Environment/ple/games/utils/__init__.py b/Durivaux/ple/games/utils/__init__.py similarity index 100% rename from PyGame-Learning-Environment/ple/games/utils/__init__.py rename to Durivaux/ple/games/utils/__init__.py diff --git a/PyGame-Learning-Environment/ple/games/utils/vec2d.py b/Durivaux/ple/games/utils/vec2d.py similarity index 100% rename from PyGame-Learning-Environment/ple/games/utils/vec2d.py rename to Durivaux/ple/games/utils/vec2d.py diff --git a/PyGame-Learning-Environment/ple/games/waterworld.py b/Durivaux/ple/games/waterworld.py similarity index 100% rename from PyGame-Learning-Environment/ple/games/waterworld.py rename to Durivaux/ple/games/waterworld.py diff --git a/PyGame-Learning-Environment/ple/ple.py b/Durivaux/ple/ple.py similarity index 100% rename from PyGame-Learning-Environment/ple/ple.py rename to Durivaux/ple/ple.py diff --git a/RandomBird/run.py b/Durivaux/run.py similarity index 100% rename from RandomBird/run.py rename to Durivaux/run.py diff --git a/Durivaux/run_with_display.py b/Durivaux/run_with_display.py new file mode 100644 index 0000000..93ead9f --- /dev/null +++ b/Durivaux/run_with_display.py @@ -0,0 +1,31 @@ +from ple.games.flappybird import FlappyBird +from ple import PLE +import numpy as np +from FlappyAgent import FlappyPolicy +import time +game = FlappyBird(graphics="fixed") # use "fancy" for full background, random bird color and random pipe color, use "fixed" (default) for black background and constant bird and pipe colors. +p = PLE(game, fps=30, frame_skip=1, num_steps=1, force_fps=True, display_screen=True) +# Note: if you want to see you agent act in real time, set force_fps to False. But don't use this setting for learning, just for display purposes. + +p.init() +reward = 0.0 + +nb_games = 100 +cumulated = np.zeros((nb_games)) + +for i in range(nb_games): + p.reset_game() + + while(not p.game_over()): + state = game.getGameState() + screen = p.getScreenRGB() + action=FlappyPolicy(state, screen) + + reward = p.act(action) + cumulated[i] = cumulated[i] + reward + print("{}\t{}\t{:.1f}".format(i, int(cumulated[i]), np.mean(cumulated[:i+1]))) + +average_score = np.mean(cumulated) +max_score = np.max(cumulated) +print() +print("Average: {:.2f}\t Max: {:.0f}".format(average_score, max_score)) diff --git a/Durivaux/train.py b/Durivaux/train.py new file mode 100644 index 0000000..ecee53f --- /dev/null +++ b/Durivaux/train.py @@ -0,0 +1,262 @@ +import sys +import os + +from ple.games.flappybird import FlappyBird +from ple import PLE + +import numpy as np + +from collections import deque +from time import time + +from MemoryBuffer import MemoryBuffer + +from skimage import transform, color + +from keras.models import Sequential, load_model +from keras.layers import Conv2D, Flatten, Dense +from keras.optimizers import Adam +from keras import backend as kerasBackend + +# Display screen or not +DISPLAY = True + +# Print period (in steps) of general info +PRINT_PERIOD = 5000 # %1000=0 + +### Constants +GAMMA = 0.99 +STEPS_NB = 1000000 +REPLAY_MEM_SIZE = STEPS_NB +MINIBATCH_SIZE = 32 +STEPS_BETWEEN_TRAINING = 5 +# Weight transfer (exploration -> target network) +STEPS_BETWEEN_TRANSFER = 2500 +# Epsilon +INITIAL_EXPLORATION = 20000 +INITIAL_EPSILON = 0.1 +FINAL_EPSILON = 1e-3 +EXPLORATION_STEPS = 500000 +# DQN learning rate +LEARNING_RATE = 1e-4 +# Evaluation period (in steps) +EVALUATION_PERIOD = 25000 +# Backup period (in steps) of the dqn +BACKUP_PERIOD = 25000 +# Number of steps during which the action is repeated (1st: choice, then repeat) +REPEAT = 2 # human-like is max 10 Hz (min 3) + + +def processScreen(screen): + """ + Resize and gray-ify screen + """ + return 255*transform.resize(color.rgb2gray(screen[60:,25:310,:]),(80,80)) + + +def epsilon(step): + """ + Epsilon for exploration/exploitation trade-off + """ + if step < INITIAL_EXPLORATION: + return 1 + elif step < EXPLORATION_STEPS: + return INITIAL_EPSILON + (FINAL_EPSILON - INITIAL_EPSILON)/(EXPLORATION_STEPS-INITIAL_EXPLORATION) * (step-INITIAL_EXPLORATION) + else: + return FINAL_EPSILON + + +def epsilonGreedy(dqn, x, step): + """ + Epsilon-greedy action + """ + if np.random.rand() < epsilon(step): + return np.random.randint(2) + else: + return np.argmax(dqn.predict(np.array([x]))) + + +def initX(rgb): + """ + Initialize screenX and x + """ + _screenX = processScreen(rgb) + _stackedX = deque([_screenX]*4, maxlen=4) + _x = np.stack(_stackedX, axis=-1) + return _screenX, _stackedX, _x + + +def createDQN(): + """ + Create deep Q network + """ + # Cf DeepMind article + dqn = Sequential() + dqn.add(Conv2D(filters=16, kernel_size=(8,8), strides=4, activation='relu', input_shape=(80,80,4))) + dqn.add(Conv2D(filters=32, kernel_size=(4,4), strides=2, activation='relu')) + dqn.add(Flatten()) + dqn.add(Dense(units=256, activation='relu')) + dqn.add(Dense(units=2, activation='linear')) + + dqn.compile(optimizer=Adam(lr=LEARNING_RATE), loss='mean_squared_error') + return dqn + + +def evaluate(p, nbOfGames, dqn): + """ + Evaluation + """ + actions = p.getActionSet() + scores = np.zeros((nbOfGames)) + for i in range(nbOfGames): + frames = deque([np.zeros((80,80))]*4, maxlen=4) + p.reset_game() + frame = 0 + while not p.game_over(): + screen = processScreen(p.getScreenRGB()) + frames.append(screen) + x = np.stack(frames, axis=-1) + if frame % REPEAT == 0 and REPEAT != 1 and frame > 0: + a = lastAction + else: + a = actions[np.argmax(dqn.predict(np.expand_dims(x, axis=0)))] + lastAction = a + r = p.act(a) + scores[i] += r + frame += 1 + return np.min(scores), np.mean(scores), np.max(scores) + + +### MAIN + +# Try to load DQN, or create a new one +loaded = False +if len(sys.argv) == 2 and sys.argv[1] == "load": + try: + dqnExploration = load_model('dqn.h5') + loaded = True + INITIAL_EXPLORATION = 0 + print("Loaded 'dqn.h5'") + except: + print("Usage: train.py [load]") + print("Defaulting to new network") + dqnExploration = createDQN() +else: + print("Starting from scratch") + dqnExploration = createDQN() +dqnExploration.save('dqn.h5') +dqnTarget = load_model('dqn.h5') + +# Environment +game = FlappyBird() +p = PLE(game, fps=30, frame_skip=1, num_steps=1, force_fps=True, display_screen=DISPLAY) +actions = p.getActionSet() + +# Initialization +gameNumber = 1 +p.init() +p.reset_game() +meanScore = -5 + +screenX, stackedX, x = initX(p.getScreenRGB()) +replayMemory = MemoryBuffer(REPLAY_MEM_SIZE, (80,80), (1,)) + +start = time() + +# Q-learning +if not loaded: + print("START OF INITIAL EXPLORATION") +for step in range(STEPS_NB): + # End of initial exploration + if step == INITIAL_EXPLORATION and not loaded: + print("END OF INITIAL EXPLORATION") + + # Final epsilon + if step == EXPLORATION_STEPS: + print("FINAL EPSILON REACHED") + + # Print progress info + if step % PRINT_PERIOD == 0 and step > 0: + print("Step {}k\tGame number: {}\tElapsed time: {:.0f} s\tEpsilon: {:.3f}".format(step//1000, gameNumber, time()-start, epsilon(step))) + + # Backup dqn and clear Keras session + if step % BACKUP_PERIOD == 0 and step > 0: + backupName = 'dqn-backup-'+str(step//1000)+'k.h5' + dqnExploration.save(backupName) + dqnTarget.save('dqnTarget.h5') + + # in order not to get clogged up + stdout = sys.stdout + sys.stdout = open(os.devnull, 'w') + kerasBackend.clear_session() + sys.stdout = stdout + + dqnExploration = load_model(backupName) + dqnTarget = load_model('dqnTarget.h5') + print("*** Saved dqn backup") + + # Evaluation + if step % EVALUATION_PERIOD == 0 and step > 0 and step > INITIAL_EXPLORATION: + print("Evaluation ", end="") + sys.stdout.flush() + numberOfGames = 100 + if 40 <= meanScore < 80: + numberOfGames = 50 + elif 80 <= meanScore: + numberOfGames = 25 + minScore, meanScore, maxScore = evaluate(p, numberOfGames, dqnExploration) + print("---> Min: {:.0f}\tMean: {:.1f}\tMax: {:.0f}".format(minScore, meanScore, maxScore)) + + sys.stdout.flush() + + # Act (epsilon-greedy) + if step % REPEAT == 0 and step > 0: + a = lastAction + else: + a = epsilonGreedy(dqnExploration, x, step) + lastAction = a + r = p.act(actions[a]) + d = p.game_over() + + # Reward clipping + if r != -1 and r != 1: + r = 0.1 # keep the bird moving on? + + # Next screen + screenY = processScreen(p.getScreenRGB()) + replayMemory.append(screenX, a, r, screenY, d) + + # Train (minibatch) + if step > INITIAL_EXPLORATION and step > MINIBATCH_SIZE and step % STEPS_BETWEEN_TRAINING == 0: + X, A, R, Y, D = replayMemory.minibatch(MINIBATCH_SIZE) + QY = dqnTarget.predict(Y) + QYmax = QY.max(1).reshape((MINIBATCH_SIZE, 1)) + update = R + GAMMA * (1-D) * QYmax + QX = dqnExploration.predict(X) + QX[np.arange(MINIBATCH_SIZE), A.ravel()] = update.ravel() + dqnExploration.train_on_batch(x=X, y=QX) + + # Update target dqn + if step % STEPS_BETWEEN_TRANSFER == 0: + dqnExploration.save('dqn.h5') + dqnTarget = load_model('dqn.h5') + + # Prepare next step + if p.game_over(): + gameNumber += 1 + p.reset_game() + screenX, stackedX, x = initX(p.getScreenRGB()) + else: + screenX = screenY + stackedX.append(screenX) + x = np.stack(stackedX, axis=-1) + +print("Final evaluation ", end="") +sys.stdout.flush() +numberOfGames = 100 +minScore, meanScore, maxScore = evaluate(p, numberOfGames, dqnExploration) +print("---> Min: {:.0f}\tMean: {:.1f}\tMax: {:.0f}".format(minScore, meanScore, maxScore)) +sys.stdout.flush() + +dqnExploration.save('dqn.h5') +print('Done') diff --git a/PyGame-Learning-Environment/.gitignore b/PyGame-Learning-Environment/.gitignore deleted file mode 100644 index 5260e2e..0000000 --- a/PyGame-Learning-Environment/.gitignore +++ /dev/null @@ -1,6 +0,0 @@ -/docs/_build -/docs/_templates -*.pyc -/examples/*.png -*.swp -*.egg-info diff --git a/PyGame-Learning-Environment/LICENSE b/PyGame-Learning-Environment/LICENSE deleted file mode 100644 index 8a71b54..0000000 --- a/PyGame-Learning-Environment/LICENSE +++ /dev/null @@ -1,8 +0,0 @@ -The MIT License (MIT) -Copyright (c) 2016 Norman Tasfi - -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file diff --git a/PyGame-Learning-Environment/README.md b/PyGame-Learning-Environment/README.md deleted file mode 100644 index 67d1a0a..0000000 --- a/PyGame-Learning-Environment/README.md +++ /dev/null @@ -1,121 +0,0 @@ -# PyGame-Learning-Environment - -![Games](ple_games.jpg?raw=True "Games!") - -**PyGame Learning Environment (PLE)** is a learning environment, mimicking the [Arcade Learning Environment](https://github.com/mgbellemare/Arcade-Learning-Environment) interface, allowing a quick start to Reinforcement Learning in Python. The goal of PLE is allow practitioners to focus design of models and experiments instead of environment design. - -PLE hopes to eventually build an expansive library of games. - -**Accepting PRs for games.** - -## Documentation - -Docs for the project can be [found here](http://pygame-learning-environment.readthedocs.org/). They are currently WIP. - -## Games - -Available games can be found in the [docs](http://pygame-learning-environment.readthedocs.org/en/latest/user/games.html). - -## Getting started - -A `PLE` instance requires a game exposing a set of control methods. To see the required methods look at `ple/games/base.py`. - -Here's an example of importing Pong from the games library within PLE: - -```python -from ple.games.pong import Pong - -game = Pong() -``` - -Next we configure and initialize PLE: - -```python -from ple import PLE - -p = PLE(game, fps=30, display_screen=True, force_fps=False) -p.init() -``` - -The options above instruct PLE to display the game screen, with `display_screen`, while allowing PyGame to select the appropriate delay timing between frames to ensure 30fps with `force_fps`. - -You are free to use any agent with the PLE. Below we create a fictional agent and grab the valid actions: - -```python -myAgent = MyAgent(p.getActionSet()) -``` - -We can now have our agent, with the help of PLE, interact with the game over a certain number of frames: - -```python - -nb_frames = 1000 -reward = 0.0 - -for f in range(nb_frames): - if p.game_over(): #check if the game is over - p.reset_game() - - obs = p.getScreenRGB() - action = myAgent.pickAction(reward, obs) - reward = p.act(action) - -``` - -Just like that we have our agent interacting with our game environment. - -## Installation - -PLE requires the following dependencies: -* numpy -* pygame -* pillow - -Clone the repo and install with pip. - -```bash -git clone https://github.com/ntasfi/PyGame-Learning-Environment.git -cd PyGame-Learning-Environment/ -pip install -e . -``` - -## Headless Usage - -Set the following in your code before usage: -```python -os.putenv('SDL_VIDEODRIVER', 'fbcon') -os.environ["SDL_VIDEODRIVER"] = "dummy" -``` - -Thanks to [@wooridle](https://github.com/ntasfi/PyGame-Learning-Environment/issues/26#issuecomment-289517054). - -## Updating - -`cd` into the `PyGame-Learning-Environment` directory and run the following: - -```bash -git pull -``` - -## Todos - * Documentation is currently in progress. - * Tests - * Parallel Learning (One agent, many game copies) - * Add games - * Generalize the library (eg. add Pyglet support) - - -## Citing PLE - -If PLE has helped your research please cite it in your publications. Example BibTeX entry: - -``` -@misc{tasfi2016PLE, - author = {Tasfi, Norman}, - title = {PyGame Learning Environment}, - year = {2016}, - publisher = {GitHub}, - journal = {GitHub repository}, - howpublished = {\url{https://github.com/ntasfi/PyGame-Learning-Environment}} -} -``` diff --git a/PyGame-Learning-Environment/docker/Dockerfile b/PyGame-Learning-Environment/docker/Dockerfile deleted file mode 100644 index f07bda3..0000000 --- a/PyGame-Learning-Environment/docker/Dockerfile +++ /dev/null @@ -1,62 +0,0 @@ -# VERSION: 0.1 -# DESCRIPTION: PLE -# AUTHOR: Eder Santana -# COMMENTS: -# Pygame learning environment -# SETUP: -# # Download PLE Dockerfile -# wget ... -# -# # Build atom image -# docker build -t ple . -# -# UBUNTU: -# docker run -it -v /tmp/.X11-unix:/tmp/.X11-unix \ -# -e DISPLAY=unix$DISPLAY ple /bin/bash -# -# MAC: -# in a separate window run: -# brew install socat -# socat TCP-LISTEN:6000,reuseaddr,fork UNIX-CLIENT:\"$DISPLAY\" -# finally: -# run ifcongi and look for the ip of vboxnet0, say 192.168.99.1 -# docker run -i -t -e DISPLAY=192.168.99.1:0 ple /bin/bash -# -# USAGE: -# cd ple/examples -# python keras_nonvis.py - -# FROM ubuntu:14.04 -FROM ubuntu - -MAINTAINER Eder Santana - -RUN apt-get update && apt-get install -y \ - mercurial \ - libav-tools \ - libsdl-image1.2-dev \ - libsdl-mixer1.2-dev \ - libsdl-ttf2.0-dev \ - libsmpeg-dev \ - libsdl1.2-dev \ - libportmidi-dev \ - libswscale-dev \ - libavformat-dev \ - libavcodec-dev \ - libplib-dev \ - libopenal-dev \ - libalut-dev \ - libvorbis-dev \ - libxxf86vm-dev \ - libxmu-dev \ - libgl1-mesa-dev \ - python-dev \ - python-pip \ - python-numpy \ - python-scipy \ - python-pygame \ - git - -# RUN hg clone https://bitbucket.org/pygame/pygame && cd pygame && python setup.py build && sudo python setup.py install && cd .. -RUN pip install keras git+https://github.com/ntasfi/PyGame-Learning-Environment.git -RUN git clone https://github.com/ntasfi/PyGame-Learning-Environment.git ple diff --git a/PyGame-Learning-Environment/docker/README.md b/PyGame-Learning-Environment/docker/README.md deleted file mode 100644 index f94689d..0000000 --- a/PyGame-Learning-Environment/docker/README.md +++ /dev/null @@ -1,24 +0,0 @@ -# Run PyGame-Learning-Environment on Docker - -## Setup -`wget http://raw.githubusercontent.com/ntasfi/PyGame-Learning-Environment/master/docker/Dockerfile` - -## Build PLE image -`docker build -t ple .` - -#### UBUNTU: -`docker run -it -v /tmp/.X11-unix:/tmp/.X11-unix \ - -e DISPLAY=unix$DISPLAY ple /bin/bash` - -#### MAC: -in a separate window run: - `brew install socat` - `socat TCP-LISTEN:6000,reuseaddr,fork UNIX-CLIENT:\"$DISPLAY\"` - -finally, run `ifconfig` and look for the ip of vboxnet0, say `192.168.99.1` - - `docker run -i -t -e DISPLAY=192.168.99.1:0 ple /bin/bash` - -## Usage: - `cd ple/examples` - `python keras_nonvis.py` diff --git a/PyGame-Learning-Environment/docs/Makefile b/PyGame-Learning-Environment/docs/Makefile deleted file mode 100644 index ba3856b..0000000 --- a/PyGame-Learning-Environment/docs/Makefile +++ /dev/null @@ -1,223 +0,0 @@ -# Makefile for Sphinx documentation -# - -# You can set these variables from the command line. -SPHINXOPTS = -SPHINXBUILD = sphinx-build -PAPER = -BUILDDIR = _build - -# User-friendly check for sphinx-build -ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) - $(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don\'t have Sphinx installed, grab it from http://sphinx-doc.org/) -endif - -# Internal variables. -PAPEROPT_a4 = -D latex_paper_size=a4 -PAPEROPT_letter = -D latex_paper_size=letter -ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . -# the i18n builder cannot share the environment and doctrees with the others -I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . - -.PHONY: help -help: - @echo "Please use \`make ' where is one of" - @echo " html to make standalone HTML files" - @echo " dirhtml to make HTML files named index.html in directories" - @echo " singlehtml to make a single large HTML file" - @echo " pickle to make pickle files" - @echo " json to make JSON files" - @echo " htmlhelp to make HTML files and a HTML help project" - @echo " qthelp to make HTML files and a qthelp project" - @echo " applehelp to make an Apple Help Book" - @echo " devhelp to make HTML files and a Devhelp project" - @echo " epub to make an epub" - @echo " epub3 to make an epub3" - @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" - @echo " latexpdf to make LaTeX files and run them through pdflatex" - @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" - @echo " text to make text files" - @echo " man to make manual pages" - @echo " texinfo to make Texinfo files" - @echo " info to make Texinfo files and run them through makeinfo" - @echo " gettext to make PO message catalogs" - @echo " changes to make an overview of all changed/added/deprecated items" - @echo " xml to make Docutils-native XML files" - @echo " pseudoxml to make pseudoxml-XML files for display purposes" - @echo " linkcheck to check all external links for integrity" - @echo " doctest to run all doctests embedded in the documentation (if enabled)" - @echo " coverage to run coverage check of the documentation (if enabled)" - -.PHONY: clean -clean: - rm -rf $(BUILDDIR)/* - -.PHONY: html -html: - $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html - @echo - @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." - -.PHONY: dirhtml -dirhtml: - $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml - @echo - @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." - -.PHONY: singlehtml -singlehtml: - $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml - @echo - @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." - -.PHONY: pickle -pickle: - $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle - @echo - @echo "Build finished; now you can process the pickle files." - -.PHONY: json -json: - $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json - @echo - @echo "Build finished; now you can process the JSON files." - -.PHONY: htmlhelp -htmlhelp: - $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp - @echo - @echo "Build finished; now you can run HTML Help Workshop with the" \ - ".hhp project file in $(BUILDDIR)/htmlhelp." - -.PHONY: qthelp -qthelp: - $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp - @echo - @echo "Build finished; now you can run "qcollectiongenerator" with the" \ - ".qhcp project file in $(BUILDDIR)/qthelp, like this:" - @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/PyGameLearningEnvironment.qhcp" - @echo "To view the help file:" - @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/PyGameLearningEnvironment.qhc" - -.PHONY: applehelp -applehelp: - $(SPHINXBUILD) -b applehelp $(ALLSPHINXOPTS) $(BUILDDIR)/applehelp - @echo - @echo "Build finished. The help book is in $(BUILDDIR)/applehelp." - @echo "N.B. You won't be able to view it unless you put it in" \ - "~/Library/Documentation/Help or install it in your application" \ - "bundle." - -.PHONY: devhelp -devhelp: - $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp - @echo - @echo "Build finished." - @echo "To view the help file:" - @echo "# mkdir -p $$HOME/.local/share/devhelp/PyGameLearningEnvironment" - @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/PyGameLearningEnvironment" - @echo "# devhelp" - -.PHONY: epub -epub: - $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub - @echo - @echo "Build finished. The epub file is in $(BUILDDIR)/epub." - -.PHONY: epub3 -epub3: - $(SPHINXBUILD) -b epub3 $(ALLSPHINXOPTS) $(BUILDDIR)/epub3 - @echo - @echo "Build finished. The epub3 file is in $(BUILDDIR)/epub3." - -.PHONY: latex -latex: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex - @echo - @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." - @echo "Run \`make' in that directory to run these through (pdf)latex" \ - "(use \`make latexpdf' here to do that automatically)." - -.PHONY: latexpdf -latexpdf: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex - @echo "Running LaTeX files through pdflatex..." - $(MAKE) -C $(BUILDDIR)/latex all-pdf - @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." - -.PHONY: latexpdfja -latexpdfja: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex - @echo "Running LaTeX files through platex and dvipdfmx..." - $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja - @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." - -.PHONY: text -text: - $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text - @echo - @echo "Build finished. The text files are in $(BUILDDIR)/text." - -.PHONY: man -man: - $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man - @echo - @echo "Build finished. The manual pages are in $(BUILDDIR)/man." - -.PHONY: texinfo -texinfo: - $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo - @echo - @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." - @echo "Run \`make' in that directory to run these through makeinfo" \ - "(use \`make info' here to do that automatically)." - -.PHONY: info -info: - $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo - @echo "Running Texinfo files through makeinfo..." - make -C $(BUILDDIR)/texinfo info - @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." - -.PHONY: gettext -gettext: - $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale - @echo - @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." - -.PHONY: changes -changes: - $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes - @echo - @echo "The overview file is in $(BUILDDIR)/changes." - -.PHONY: linkcheck -linkcheck: - $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck - @echo - @echo "Link check complete; look for any errors in the above output " \ - "or in $(BUILDDIR)/linkcheck/output.txt." - -.PHONY: doctest -doctest: - $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest - @echo "Testing of doctests in the sources finished, look at the " \ - "results in $(BUILDDIR)/doctest/output.txt." - -.PHONY: coverage -coverage: - $(SPHINXBUILD) -b coverage $(ALLSPHINXOPTS) $(BUILDDIR)/coverage - @echo "Testing of coverage in the sources finished, look at the " \ - "results in $(BUILDDIR)/coverage/python.txt." - -.PHONY: xml -xml: - $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml - @echo - @echo "Build finished. The XML files are in $(BUILDDIR)/xml." - -.PHONY: pseudoxml -pseudoxml: - $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml - @echo - @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." diff --git a/PyGame-Learning-Environment/docs/_static/catcher.gif b/PyGame-Learning-Environment/docs/_static/catcher.gif deleted file mode 100644 index ae89b8f..0000000 Binary files a/PyGame-Learning-Environment/docs/_static/catcher.gif and /dev/null differ diff --git a/PyGame-Learning-Environment/docs/_static/doom.gif b/PyGame-Learning-Environment/docs/_static/doom.gif deleted file mode 100644 index d3b9f61..0000000 Binary files a/PyGame-Learning-Environment/docs/_static/doom.gif and /dev/null differ diff --git a/PyGame-Learning-Environment/docs/_static/flappybird.gif b/PyGame-Learning-Environment/docs/_static/flappybird.gif deleted file mode 100644 index d145455..0000000 Binary files a/PyGame-Learning-Environment/docs/_static/flappybird.gif and /dev/null differ diff --git a/PyGame-Learning-Environment/docs/_static/monsterkong.gif b/PyGame-Learning-Environment/docs/_static/monsterkong.gif deleted file mode 100644 index 87c50b5..0000000 Binary files a/PyGame-Learning-Environment/docs/_static/monsterkong.gif and /dev/null differ diff --git a/PyGame-Learning-Environment/docs/_static/pixelcopter.gif b/PyGame-Learning-Environment/docs/_static/pixelcopter.gif deleted file mode 100644 index 6b4fb1a..0000000 Binary files a/PyGame-Learning-Environment/docs/_static/pixelcopter.gif and /dev/null differ diff --git a/PyGame-Learning-Environment/docs/_static/pong.gif b/PyGame-Learning-Environment/docs/_static/pong.gif deleted file mode 100644 index 41cefe9..0000000 Binary files a/PyGame-Learning-Environment/docs/_static/pong.gif and /dev/null differ diff --git a/PyGame-Learning-Environment/docs/_static/puckworld.gif b/PyGame-Learning-Environment/docs/_static/puckworld.gif deleted file mode 100644 index 88e696b..0000000 Binary files a/PyGame-Learning-Environment/docs/_static/puckworld.gif and /dev/null differ diff --git a/PyGame-Learning-Environment/docs/_static/raycastmaze.gif b/PyGame-Learning-Environment/docs/_static/raycastmaze.gif deleted file mode 100644 index e516a0c..0000000 Binary files a/PyGame-Learning-Environment/docs/_static/raycastmaze.gif and /dev/null differ diff --git a/PyGame-Learning-Environment/docs/_static/snake.gif b/PyGame-Learning-Environment/docs/_static/snake.gif deleted file mode 100644 index 09659e7..0000000 Binary files a/PyGame-Learning-Environment/docs/_static/snake.gif and /dev/null differ diff --git a/PyGame-Learning-Environment/docs/_static/waterworld.gif b/PyGame-Learning-Environment/docs/_static/waterworld.gif deleted file mode 100644 index 1d5374a..0000000 Binary files a/PyGame-Learning-Environment/docs/_static/waterworld.gif and /dev/null differ diff --git a/PyGame-Learning-Environment/docs/conf.py b/PyGame-Learning-Environment/docs/conf.py deleted file mode 100644 index 4712cf9..0000000 --- a/PyGame-Learning-Environment/docs/conf.py +++ /dev/null @@ -1,80 +0,0 @@ -import sys -import os -from mock import Mock -sys.modules['pygame'] = Mock() -sys.modules['pygame.constants'] = Mock() - -#so we can import ple -sys.path.append(os.path.join(os.path.dirname(__name__), "..")) - -extensions = [ - 'sphinx.ext.autodoc', - 'sphinx.ext.autosummary', - 'sphinx.ext.mathjax', - 'sphinx.ext.viewcode', - 'numpydoc' -] - -numpydoc_show_class_members = False - -# Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] - -source_suffix = '.rst' - -master_doc = 'index' - -# General information about the project. -project = u'PyGame Learning Environment' -copyright = u'2016, Norman Tasfi' -author = u'Norman Tasfi' - -import ple - -version = u'0.1.dev1' -# The full version, including alpha/beta/rc tags. -release = u'0.1.dev1' - -language = None -exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] - -pygments_style = 'sphinx' - -todo_include_todos = False - -#from lasagne! -if os.environ.get('READTHEDOCS') != 'True': - try: - import sphinx_rtd_theme - except ImportError: - pass # assume we have sphinx >= 1.3 - else: - html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] - - html_theme = 'sphinx_rtd_theme' - -html_static_path = ['_static'] - -htmlhelp_basename = 'PyGameLearningEnvironmentdoc' - -latex_elements = { -} - -# Grouping the document tree into LaTeX files. List of tuples -# (source start file, target name, title, -# author, documentclass [howto, manual, or own class]). -latex_documents = [ - (master_doc, 'PyGameLearningEnvironment.tex', u'PyGame Learning Environment Documentation', - u'Norman Tasfi', 'manual'), -] - -man_pages = [ - (master_doc, 'pygamelearningenvironment', u'PyGame Learning Environment Documentation', - [author], 1) -] - -texinfo_documents = [ - (master_doc, 'PyGameLearningEnvironment', u'PyGame Learning Environment Documentation', - author, 'PyGameLearningEnvironment', 'RL for all.', - 'Miscellaneous'), -] diff --git a/PyGame-Learning-Environment/docs/index.rst b/PyGame-Learning-Environment/docs/index.rst deleted file mode 100644 index 12af100..0000000 --- a/PyGame-Learning-Environment/docs/index.rst +++ /dev/null @@ -1,30 +0,0 @@ -PLE: A Reinforcement Learning Environment -===================================================================== - -**PyGame Learning Environment (PLE)** is a learning environment, mimicking the Arcade Learning Environment interface, allowing a quick start to Reinforcement Learning in Python. The goal of PLE is allow practitioners to focus design of models and experiments instead of environment design. - -PLE has only been tested with **Python 2.7.6**. - -User Guide ------------- - -The PLE user guide below explains the different components inside of the library. It covers how to train a reinforcement learning agent, the environments structure and function, and the required methods if one wishes to add games to library. - -.. toctree:: - :maxdepth: 2 - - /user/home - /user/tutorial - /user/games - -API Reference -------------- - -Information for specific classes and methods. - -.. toctree:: - /modules/ple - /modules/base - - -.. _GitHub: https://github.com/ntasfi/PyGame-Learning-Environment diff --git a/PyGame-Learning-Environment/docs/modules/base.rst b/PyGame-Learning-Environment/docs/modules/base.rst deleted file mode 100644 index 87dbb3d..0000000 --- a/PyGame-Learning-Environment/docs/modules/base.rst +++ /dev/null @@ -1,10 +0,0 @@ -:mod:`ple.games.base` -======================== - -.. currentmodule:: ple.games.base.pygamewrapper -.. autoclass:: PyGameWrapper - :members: - -.. currentmodule:: ple.games.base.doomwrapper -.. autoclass:: DoomWrapper - :members: diff --git a/PyGame-Learning-Environment/docs/modules/ple.rst b/PyGame-Learning-Environment/docs/modules/ple.rst deleted file mode 100644 index c6d8db8..0000000 --- a/PyGame-Learning-Environment/docs/modules/ple.rst +++ /dev/null @@ -1,7 +0,0 @@ -:mod:`ple.PLE` -======================== - -.. currentmodule:: ple - -.. autoclass:: PLE - :members: diff --git a/PyGame-Learning-Environment/docs/user/games.rst b/PyGame-Learning-Environment/docs/user/games.rst deleted file mode 100644 index 2e8a87b..0000000 --- a/PyGame-Learning-Environment/docs/user/games.rst +++ /dev/null @@ -1,14 +0,0 @@ -Available Games ----------------- - -.. toctree:: - /user/games/doom - /user/games/catcher - /user/games/monsterkong - /user/games/flappybird - /user/games/pixelcopter - /user/games/pong - /user/games/puckworld - /user/games/raycastmaze - /user/games/snake - /user/games/waterworld diff --git a/PyGame-Learning-Environment/docs/user/games/catcher.rst b/PyGame-Learning-Environment/docs/user/games/catcher.rst deleted file mode 100644 index 08096ab..0000000 --- a/PyGame-Learning-Environment/docs/user/games/catcher.rst +++ /dev/null @@ -1,25 +0,0 @@ -Catcher -======== - -.. image:: /_static/catcher.gif - -In Catcher the agent must catch falling fruit with its paddle. - - -Valid Actions -------------- -Left and right control the direction of the paddle. The paddle has a little velocity added to it to allow smooth movements. - - -Terminal states (game_over) ---------------------------- -The game is over when the agent lose the number of lives set by init_lives parmater. - - -Rewards -------- -The agent receives a positive reward, of +1, for each successful fruit catch, while it loses a point, -1, if the fruit is not caught. - -.. currentmodule:: ple.games.catcher -.. autoclass:: Catcher - :members: __init__, getGameState diff --git a/PyGame-Learning-Environment/docs/user/games/doom.rst b/PyGame-Learning-Environment/docs/user/games/doom.rst deleted file mode 100644 index 0c0dfed..0000000 --- a/PyGame-Learning-Environment/docs/user/games/doom.rst +++ /dev/null @@ -1,146 +0,0 @@ -Doom -======== - -.. image:: /_static/doom.gif - -This is a wrapper around `ViZDoom`_. It uses `doom-py`_, written by the folks over at OpenAI, to install and compile Doom easily. Currently it has only been tested on **Ubuntu**. It can work on Mac but involves a bit of work making sure the dynamic library paths have been setup correctly. - -Doom contains different scenarios which must be specified with the initilization parameter `scenario`. -The scenarios and their descriptions are written below. They are a copy of the `README.md`_ found on the ViZDoom page. - -.. _ViZDoom: https://github.com/Marqt/ViZDoom -.. _doom-py: https://github.com/openai/doom-py -.. _README.md: https://github.com/Marqt/ViZDoom/tree/master/scenarios - - -Scenarios ---------- - -**This is a copy of the** `file found`_ **in the ViZDoom repo.** - -BASIC -^^^^^^ - -The purpose of the scenario is just to check if using this framework to train some AI i 3D environment is feasible. - -Map is a rectangle with gray walls, ceiling and floor. Player is spawned along the longer wall, in the center. A red, circular monster is spawned randomly somewhere along the opposite wall. Player can only (config) go left/right and shoot. 1 hit is enough to kill the monster. Episode finishes when monster is killed or on timeout. - -REWARDS: +101 for killing the monster -5 for missing Episode ends after killing the monster or on timeout. - -Further configuration: - -living reward = -1, -3 available buttons: move left, move right, shoot (attack) -timeout = 300 - -DEADLY CORRIDOR -^^^^^^^^^^^^^^^^ - -The purpose of this scenario is to teach the agent to navigate towards his fundamental goal (the vest) and make sure he survives at the same time. - -Map is a corridor with shooting monsters on both sides (6 monsters in total). A green vest is placed at the oposite end of the corridor. Reward is proportional (negative or positive) to change of the distance between the player and the vest. If player ignores monsters on the sides and runs straight for the vest he will be killed somewhere along the way. To ensure this behavior doom_skill = 5 (config) is needed. - -REWARDS: +dX for getting closer to the vest. -dX for getting further from the vest. - -Further configuration: - -5 available buttons: turn left, turn right, move left, move right, shoot (attack) -timeout = 4200 -death penalty = 100 -doom_skill = 5 - -DEFEND THE CENTER -^^^^^^^^^^^^^^^^^^ - -The purpose of this scenario is to teach the agent that killing the monsters is GOOD and when monsters kill you is BAD. In addition, wasting amunition is not very good either. Agent is rewarded only for killing monsters so he has to figure out the rest for himself. - -Map is a large circle. Player is spawned in the exact center. 5 melee-only, monsters are spawned along the wall. Monsters are killed after a single shot. After dying each monster is respawned after some time. Episode ends when the player dies (it's inevitable becuse of limitted ammo). - -REWARDS: +1 for killing a monster - -Further configuration: - -3 available buttons: turn left, turn right, shoot (attack) -death penalty = 1 - -DEFEND THE LINE -^^^^^^^^^^^^^^^^ - -The purpose of this scenario is to teach the agent that killing the monsters is GOOD and when monsters kill you is BAD. In addition, wasting amunition is not very good either. Agent is rewarded only for killing monsters so he has to figure out the rest for himself. - -Map is a rectangle. Player is spawned along the longer wall, in the center. 3 melee-only and 3 shooting monsters are spawned along the oposite wall. Monsters are killed after a single shot, at first. After dying each monster is respawned after some time and can endure more damage. Episode ends when the player dies (it's inevitable becuse of limitted ammo). - -REWARDS: +1 for killing a monster - -Further configuration: - -3 available buttons: turn left, turn right, shoot (attack) -death penalty = 1 - -HEALTH GATHERING -^^^^^^^^^^^^^^^^^ - -The purpose of this scenario is to teach the agent how to survive without knowing what makes him survive. Agent know only that life is precious and death is bad so he must learn what prolongs his existence and that his health is connected with it. - -Map is a rectangle with green, acidic floor which hurts the player periodically. Initially there are some medkits spread uniformly over the map. A new medkit falls from the skies every now and then. Medkits heal some portions of player's health - to survive agent needs to pick them up. Episode finishes after player's death or on timeout. - -Further configuration: - -living_reward = 1 -3 available buttons: turn left, turn right, move forward -1 available game variable: HEALTH -death penalty = 100 - -MY WAY HOME -^^^^^^^^^^^^ - -The purpose of this scenario is to teach the agent how to navigate in a labirynth-like surroundings and reach his ultimate goal (and learn what it actually is). - -Map is a series of rooms with interconnection and 1 corridor with a dead end. Each room has a different color. There is a green vest in one of the rooms (the same room every time). Player is spawned in randomly choosen room facing a random direction. Episode ends when vest is reached or on timeout/ - -REWARDS: +1 for reaching the vest - -Further configuration: - -3 available buttons: turn left, turn right, move forward -living reward = -0.0001 -timeout = 2100 - -PREDICT POSITION -^^^^^^^^^^^^^^^^^ - -The purpose of the scenario is teach agent to synchronize missle weapon shot (involving a signifficant delay between shooting and hitting) with target movements. Agent should be able to shoot so that missle and monster meet each other. - -The map is a rectangle room. Player is spawned along the longer wall, in the center. A monster is spawned randomly somewhere along the opposite wall and walks between left and right corners along the wall. Player is equipped with a rocket launcher and a single rocket. Episode ends when missle hits a wall/the monster or on timeout. - -REWARDS: +1 for killing the monster - -Further configuration: - -living reward = -0.0001, -3 available buttons: move left, move right, shoot (attack) -timeout = 300 - -TAKE COVER -^^^^^^^^^^^ - -The purpose of this scenario is to teach agent to link incomming missles with his estimated lifespan. Agent should learn that being hit means health decrease and this in turn will lead to death which is undesirable. In effect agent should avoid missles. - -Map is a rectangle. Player is spawned along the longer wall, in the center. A couple of shooting monsters are spawned randomly somewhere along the opposite wall and try to kill the player with fireballs. The player can only (config) move left/right. More monsters appear with time. Episode ends when player dies. - -REWARDS: +1 for each tic of life - -Further configuration: - -living reward = 1.0, -2 available buttons: move left, move right - -.. _file found: https://github.com/Marqt/ViZDoom/blob/master/scenarios/README.md - -Valid Actions, Terminal states (game_over) & Rewards ------------------------------------------------------- -The valid actions and Terminal states depend on the scenario specified. The scenarios are listed above or can be found `.cfg` in the `ple/games/doom/scenarios/cfg` folder. - -.. currentmodule:: ple.games.doom -.. autoclass:: Doom - :members: __init__, getGameState diff --git a/PyGame-Learning-Environment/docs/user/games/flappybird.rst b/PyGame-Learning-Environment/docs/user/games/flappybird.rst deleted file mode 100644 index ad7c30e..0000000 --- a/PyGame-Learning-Environment/docs/user/games/flappybird.rst +++ /dev/null @@ -1,26 +0,0 @@ -FlappyBird -=========== - -.. image:: /_static/flappybird.gif - -Flappybird is a side-scrolling game where the agent must successfully nagivate through gaps between pipes. - -FPS Restrictions ----------------- -This game is restricted to 30fps as the physics at higher and lower framerates feels slightly off. You can remove this by setting the `allowed_fps` parameter to `None`. - -Valid Actions -------------- -Up causes the bird to accelerate upwards. - -Terminal states (game_over) ---------------------------- -If the bird makes contact with the ground, pipes or goes above the top of the screen the game is over. - -Rewards -------- -For each pipe it passes through it gains a positive reward of +1. Each time a terminal state is reached it receives a negative reward of -1. - -.. currentmodule:: ple.games.flappybird -.. autoclass:: FlappyBird - :members: __init__, getGameState diff --git a/PyGame-Learning-Environment/docs/user/games/monsterkong.rst b/PyGame-Learning-Environment/docs/user/games/monsterkong.rst deleted file mode 100644 index 73b6bc6..0000000 --- a/PyGame-Learning-Environment/docs/user/games/monsterkong.rst +++ /dev/null @@ -1,23 +0,0 @@ -Monster Kong -============= - -.. image:: /_static/monsterkong.gif - -A spinoff of the original Donkey Kong game. Objective of the game is to avoid fireballs while collecting coins and rescuing the princess. An additional monster is added each time the princess is rescued. - - -Valid Actions -------------- -Use w, a, s, d and space keys to move around player around. - - -Terminal states (game_over) ---------------------------- -The game is over when the player hits three fireballs. Touching a monster does not kill cause the agent to lose lives. - -Rewards -------- - -The player gains +5 for collecting a coin while losing a life and receiving a reward of -25 for hitting a fireball. The player gains +50 points for rescusing a princess. - -Note: Images were sourced from various authors. You can find the respective artist listed in the assets directory folder of the game. diff --git a/PyGame-Learning-Environment/docs/user/games/pixelcopter.rst b/PyGame-Learning-Environment/docs/user/games/pixelcopter.rst deleted file mode 100644 index f7947eb..0000000 --- a/PyGame-Learning-Environment/docs/user/games/pixelcopter.rst +++ /dev/null @@ -1,22 +0,0 @@ -Pixelcopter -============= - -.. image:: /_static/pixelcopter.gif - -Pixelcopter is a side-scrolling game where the agent must successfully nagivate through a cavern. This is a clone of the popular helicopter game but where the player is a humble pixel. - -Valid Actions -------------- -Up which causes the pixel to accelerate upwards. - -Terminal states (game_over) ---------------------------- -If the pixel makes contact with anything green the game is over. - -Rewards -------- -For each vertical block it passes through it gains a positive reward of +1. Each time a terminal state reached it receives a negative reward of -1. - -.. currentmodule:: ple.games.pixelcopter -.. autoclass:: Pixelcopter - :members: __init__, getGameState diff --git a/PyGame-Learning-Environment/docs/user/games/pong.rst b/PyGame-Learning-Environment/docs/user/games/pong.rst deleted file mode 100644 index e58dd5a..0000000 --- a/PyGame-Learning-Environment/docs/user/games/pong.rst +++ /dev/null @@ -1,26 +0,0 @@ -Pong -===== - -.. image:: /_static/pong.gif - -Pong simulates 2D table tennis. The agent controls an in-game paddle which is used to hit the ball back to the other side. - -The agent controls the left paddle while the CPU controls the right paddle. - -Valid Actions -------------- -Up and down control the direction of the paddle. The paddle has a little velocity added to it to allow smooth movements. - - -Terminal states (game_over) ---------------------------- -The game is over if either the agent or CPU reach the number of points set by MAX_SCORE. - - -Rewards -------- -The agent receives a positive reward, of +1, for each successful ball placed behind the opponents paddle, while it loses a point, -1, if the ball goes behind its paddle. - -.. currentmodule:: ple.games.pong -.. autoclass:: Pong - :members: __init__, getGameState diff --git a/PyGame-Learning-Environment/docs/user/games/puckworld.rst b/PyGame-Learning-Environment/docs/user/games/puckworld.rst deleted file mode 100644 index dcb21a6..0000000 --- a/PyGame-Learning-Environment/docs/user/games/puckworld.rst +++ /dev/null @@ -1,24 +0,0 @@ -PuckWorld -========== - -.. image:: /_static/puckworld.gif - -In PuckWorld the agent, a blue circle, must navigate towards the green dot while avoiding the larger red puck. - -The green dot randomly moves around the screen while the red puck slowly follows the agent. - -Valid Actions -------------- -Up, down, left and right apply thrusters to the agent. It adds velocity to the agent which decays over time. - -Terminal states (game_over) ---------------------------- -None. This is a continuous game. - -Rewards -------- -The agent is rewarded based on its distance to the green dot, where lower values are good. If the agent is within the large red radius it receives a negative reward. The negative reward is proportional to the agents distance from the pucks center. - -.. currentmodule:: ple.games.puckworld -.. autoclass:: PuckWorld - :members: __init__, getGameState diff --git a/PyGame-Learning-Environment/docs/user/games/raycastmaze.rst b/PyGame-Learning-Environment/docs/user/games/raycastmaze.rst deleted file mode 100644 index 739da36..0000000 --- a/PyGame-Learning-Environment/docs/user/games/raycastmaze.rst +++ /dev/null @@ -1,34 +0,0 @@ -RaycastMaze -============ - -.. image:: /_static/raycastmaze.gif - -In RaycastMaze the agent must navigate a 3D environment searching for the exit denoted with a bright red square. - -It is possible to increase the map size by 1 each time it successfully solves the maze. As seen below. - -Example -------- ->>> #init and setup etc. ->>> while True: ->>> if game.game_over(): ->>> game.map_size += 1 ->>> game.step(dt) #assume dt is given - -**Not valid code above.** - -Valid Actions -------------- -Forwards, backwards, turn left and turn right. - -Terminal states (game_over) ---------------------------- -When the agent is a short distance, nearly touching the red square, the game is considered over. - -Rewards -------- -Currently it receives a postive reward of +1 when it finds the red block. - -.. currentmodule:: ple.games.raycastmaze -.. autoclass:: RaycastMaze - :members: __init__, getGameState diff --git a/PyGame-Learning-Environment/docs/user/games/snake.rst b/PyGame-Learning-Environment/docs/user/games/snake.rst deleted file mode 100644 index de7dd44..0000000 --- a/PyGame-Learning-Environment/docs/user/games/snake.rst +++ /dev/null @@ -1,24 +0,0 @@ -Snake -====== - -.. image:: /_static/snake.gif - -Snake is a game where the agent must maneuver a line which grows in length each time food is touched by the head of the segment. The line follows the previous paths taken which eventually become obstacles for the agent to avoid. - -The food is randomly spawned inside of the valid window while checking it does not make contact with the snake body. - -Valid Actions -------------- -Up, down, left, and right. It cannot turn back on itself. Eg. if its moving downwards it cannot move up. - -Terminal states (game_over) ---------------------------- -If the head of the snake comes in contact with any of the walls or its own body (can occur after only 7 segments) the game is over. - -Rewards -------- -It recieves a positive reward, +1, for each red square the head comes in contact with. While getting -1 for each terminal state it reaches. - -.. currentmodule:: ple.games.snake -.. autoclass:: Snake - :members: __init__, getGameState diff --git a/PyGame-Learning-Environment/docs/user/games/waterworld.rst b/PyGame-Learning-Environment/docs/user/games/waterworld.rst deleted file mode 100644 index f642fc3..0000000 --- a/PyGame-Learning-Environment/docs/user/games/waterworld.rst +++ /dev/null @@ -1,24 +0,0 @@ -WaterWorld -=========== - -.. image:: /_static/waterworld.gif - -In WaterWorld the agent, a blue circle, must navigate around the world capturing green circles while avoiding red ones. - -After capture a circle it will respawn in a random location as either red or green. The game is over if all the green circles have been captured. - -Valid Actions -------------- -Up, down, left and right apply thrusters to the agent. It adds velocity to the agent which decays over time. - -Terminal states (game_over) ---------------------------- -The game ends when all the green circles have been captured by the agent. - -Rewards -------- -For each green circle captured the agent receives a positive reward of +1; while hitting a red circle causes a negative reward of -1. - -.. currentmodule:: ple.games.waterworld -.. autoclass:: WaterWorld - :members: __init__, getGameState diff --git a/PyGame-Learning-Environment/docs/user/home.rst b/PyGame-Learning-Environment/docs/user/home.rst deleted file mode 100644 index ec18ba1..0000000 --- a/PyGame-Learning-Environment/docs/user/home.rst +++ /dev/null @@ -1,102 +0,0 @@ -.. _home: - -===== -Home -===== - -Installation ---------------- - -PLE requires the following libraries to be installed: - -* numpy -* pillow -* pygame -* doom-py (if using doom) - -PyGame -####### - -PyGame can be installed using this `tutorial`_ (Ubuntu). For mac you can use these following instructions; - -.. code-block:: bash - - brew install sdl sdl_ttf sdl_image sdl_mixer portmidi # brew or use equivalent means - conda install -c https://conda.binstar.org/quasiben pygame # using Anaconda - -.. _tutorial: http://www.pygame.org/wiki/CompileUbuntu - -ViZDoom (optional) -################## - -If you are on mac you can install the dependicies as follows: - -.. code-block:: bash - - brew install cmake boost boost-python - -If you are on Ubuntu (only test on 14.04) type: - -.. code-block:: bash - - apt-get install -y cmake zlib1g-dev libjpeg-dev libboost-all-dev gcc libsdl2-dev wget unzip - -Finally install OpenAI's excellent wrapper `doom-py` with the following command: - -.. code-block:: bash - - sudo pip install doom-py - -Or you can clone the `repo here`_ and install with pip. - -PLE -### - -To install PLE first clone the repo: - -.. code-block:: bash - - git clone https://github.com/ntasfi/PyGame-Learning-Environment - -Then use the ``cd`` command to enter the ``PyGame-Learning-Environment`` directory and run the command: - -.. code-block:: bash - - sudo pip install -e . - -This will install PLE as an editable library with pip. - -.. _repo here: https://github.com/openai/doom-py - -Quickstart ---------------- - -PLE allows agents to train against games through a standard model supplied by :class:`ple.PLE `, which interacts and manipulates games on behalf of your agent. PLE mimics the `Arcade Learning Environment`_ (ALE) interface as closely as possible. This means projects using the ALE interface can easily be adjusted to use PLE with minimal effort. - -If you do not wish to perform such modifications you can write your own code that interacts with PLE or use libraries with PLE support such as `General Deep Q RL`_. - -Here is an example of having an agent run against :class:`FlappyBird `. - -.. code-block:: python - - from ple.games.flappybird import FlappyBird - from ple import PLE - - - game = FlappyBird() - p = PLE(game, fps=30, display_screen=True) - agent = myAgentHere(allowed_actions=p.getActionSet()) - - p.init() - reward = 0.0 - - for i in range(nb_frames): - if p.game_over(): - p.reset_game() - - observation = p.getScreenRGB() - action = agent.pickAction(reward, observation) - reward = p.act(action) - -.. _Arcade Learning Environment: https://github.com/mgbellemare/Arcade-Learning-Environment -.. _General Deep Q RL: https://github.com/VinF/General_Deep_Q_RL diff --git a/PyGame-Learning-Environment/docs/user/tutorial.rst b/PyGame-Learning-Environment/docs/user/tutorial.rst deleted file mode 100644 index 3590de7..0000000 --- a/PyGame-Learning-Environment/docs/user/tutorial.rst +++ /dev/null @@ -1,6 +0,0 @@ -Tutorials ----------- - -.. toctree:: - /user/tutorial/adding_pygame_games - /user/tutorial/non_visual_state diff --git a/PyGame-Learning-Environment/docs/user/tutorial/adding_pygame_games.rst b/PyGame-Learning-Environment/docs/user/tutorial/adding_pygame_games.rst deleted file mode 100644 index cf5b1ed..0000000 --- a/PyGame-Learning-Environment/docs/user/tutorial/adding_pygame_games.rst +++ /dev/null @@ -1,74 +0,0 @@ -Wrapping and Adding PyGame Games -================================= - -Adding or wrapping games to work with PLE is relatively easy. You must implement a few methods, explained below, to have a game be useable with PLE. We will walk through an implementation of the Catcher game inspired by Eder Santana to examine the required methods for games. As we want to focus on the important aspects related to the game interface we will ignore game specific code. - -Note: The full code is not included in each method. The full implementation, which implements scaling based on screen dimensions, is `found here`_. - -Catcher is a simple game where the agent must catch 'fruit' dropped at random from the top of the screen with the 'paddle' controlled by the agent. - -The main component of the game is enclosed in one class that inherits from :class:`base.PyGameWrapper `: - -.. code:: python - - from ple.games import base - from pygame.constants import K_a, K_d - - class Catcher(base.PyGameWrapper): - - def __init__(self, width=48, height=48): - - actions = { - "left": K_a, - "right": K_d - } - - base.Game.__init__(self, width, height, actions=actions) - - #game specific - self.lives = 0 - - -The game must inherit from :class:`base.PyGameWrapper ` as it sets attributes and methods used by PLE to control game flow, scoring and other functions. - -The crucial porition in the ``__init__`` method is to call the parent class ``__init__`` and pass the width, height and valid actions the game responds too. - -Next we cover four required methods: ``init``, ``getScore``, ``game_over``, and ``step``. These methods are all required to interact with our game. - -The code below is within our ``Catcher`` class and has the class definition repeated for clarity: - -.. code:: python - - class Catcher(base.PyGameWrapper): - - def init(self): - self.score = 0 - - #game specific - self.lives = 3 - - def getScore(self): - return self.score - - def game_over(self): - return self.lives == 0 - - def step(self, dt): - #move players - #check hits - #adjust scores - #remove lives - - -The ``init`` method sets the game to a clean state. The minimum this method must do is to reset the ``self.score`` attribute of the game. It is also strongly recommended this method perform other game specific functions such as player position and clearing the screen. This is important as the game might still be in a terminal state if the player and object positions are not reset; which would result in endless resetting of the environment. - -``getScore`` returns the current score of the agent. You are free to pull information from the game to decide on a score, such as the number of lives left etc. or you can simply return the ``self.score`` attribute. - -``game_over`` must return True if the game has hit a terminal state. This depends greatly on game. In this case the agent loses a life for each fruit it fails to catch and causes the game to end if it hits 0. - -``step`` method is responsible for the main logic of the game. It is called everytime our agent performs an action on the game environment. ``step`` performs a step in game time equal to ``dt``. ``dt`` is required to allow the game to run at different frame rates such that the movement speeds of objects are scaled by elapsed time. With that said the game can be locked to a specific frame rate, by setting ``self.allowed_fps``, and written such that ``step`` moves game objects at rates suitable for the locked frame rate. The function signature always expects ``dt`` to be passed, the game logic does not have to use it though. - -Thats it! You only need a handful of methods defined to be able to interface your game with PLE. It is suggested to look through the different games inside of the `games folder`_. - -.. _`found here`: https://github.com/ntasfi/PyGame-Learning-Environment/blob/master/ple/games/catcher.py -.. _`games folder`: https://github.com/ntasfi/PyGame-Learning-Environment/blob/master/ple/games/ diff --git a/PyGame-Learning-Environment/docs/user/tutorial/assumed_knowledge.rst b/PyGame-Learning-Environment/docs/user/tutorial/assumed_knowledge.rst deleted file mode 100644 index 8a7e120..0000000 --- a/PyGame-Learning-Environment/docs/user/tutorial/assumed_knowledge.rst +++ /dev/null @@ -1,3 +0,0 @@ -Assumed Knowledge -================= -wip diff --git a/PyGame-Learning-Environment/docs/user/tutorial/non_visual_state.rst b/PyGame-Learning-Environment/docs/user/tutorial/non_visual_state.rst deleted file mode 100644 index 19a891b..0000000 --- a/PyGame-Learning-Environment/docs/user/tutorial/non_visual_state.rst +++ /dev/null @@ -1,80 +0,0 @@ -Non-Visual State Representation -================================ - -Sometimes it is useful to have non-visual state representations of games. This could be to try reduced state space, augment visual input, or for troubleshooting purposes. The majority of current games in PLE support non-visual state representations. To use these representations instead of visual inputs one needs to inspect the state structure given in the documentation. You are free to select sub-poritions ofthe state as agent input. - -Lets setup an agent to use a non-visual state representation of :class:`Pong `. - -First start by examining the values :class:`Pong ` will return from the ``getGameState()`` method: - -.. code-block:: python - - def getGameState(self): - #other code above... - - state = { - "player_y": self.agentPlayer.pos.y, - "player_velocity": self.agentPlayer.vel.y, - "cpu_y": self.cpuPlayer.pos.y, - "ball_x": self.ball.pos.x, - "ball_y": self.ball.pos.y, - "ball_velocity_x": self.ball.pos.x, - "ball_velocity_y": self.ball.pos.y - } - - return state - -We see that ``getGameState()`` of Pong returns several values each time it is called. Using the returned dictonary we can create a numpy vector representating our state. - -This can be accomplished in the following ways: - -.. code-block:: python - - #easiest - my_state = np.array([ state.values() ]) - - #by-hand - my_state = np.array([ state["player"]["x"], state["player"]["velocity"], ... ]) - -You have control over which values you want to include in the state vector. Training an agent would look like this: - -.. code-block:: python - - from ple.games.pong import Pong - from ple import PLE - import numpy as np - - def process_state(state): - return np.array([ state.values() ]) - - game = Pong() - p = PLE(game, display_screen=True, state_preprocessor=process_state) - agent = myAgentHere(input_shape=p.getGameStateDims(), allowed_actions=p.getActionSet()) - - p.init() - nb_frames = 10000 - reward = 0.0 - for i in range(nb_frames): - if p.game_over(): - p.reset_game() - - state = p.getGameState() - action = agent.pickAction(reward, state) - reward = p.act(action) - -To make this work a state processor must be supplied to PLE's ``state_preprocessor`` initialization method. This function will be called each time we request the games state. We can also let our agent know the dimensions of the vector to expect from PLE. In the main loop just simply replace the call to ``getScreenGrayScale`` with ``getGameState``. - -Be aware different games will have different dictonary structures. The majority of games will return back a flat dictory structure but others will have lists inside of them. In particular games with variable objects to track, such as the number of segments in snake, require usage of lists within the dictonary. - - -.. code-block:: python - - state = { - "snake_head_x": self.player.head.pos.x, - "snake_head_y": self.player.head.pos.y, - "food_x": self.food.pos.x, - "food_y": self.food.pos.y, - "snake_body": [] - } - -The ``"snake_body"`` field contains a dynamic number of values. It must be taken into consideration when creating your state preprocessor. diff --git a/PyGame-Learning-Environment/docs/user/tutorial/ple_gen_dql.rst b/PyGame-Learning-Environment/docs/user/tutorial/ple_gen_dql.rst deleted file mode 100644 index fa0fb55..0000000 --- a/PyGame-Learning-Environment/docs/user/tutorial/ple_gen_dql.rst +++ /dev/null @@ -1,8 +0,0 @@ -PLE with Gen. Deep RL ----------------------- - -**wip** - -Show you how to train an RL agent using Vincent François' excellent framework: `General Deep RL`_. - -.. _General Deep RL : https://github.com/VinF/General_Deep_Q_RL diff --git a/PyGame-Learning-Environment/docs/user/tutorial/ple_keras.rst b/PyGame-Learning-Environment/docs/user/tutorial/ple_keras.rst deleted file mode 100644 index 0fb6833..0000000 --- a/PyGame-Learning-Environment/docs/user/tutorial/ple_keras.rst +++ /dev/null @@ -1,3 +0,0 @@ -PLE with Keras --------------- -wip diff --git a/PyGame-Learning-Environment/examples/example_doom.py b/PyGame-Learning-Environment/examples/example_doom.py deleted file mode 100644 index 038d0e0..0000000 --- a/PyGame-Learning-Environment/examples/example_doom.py +++ /dev/null @@ -1,37 +0,0 @@ -import numpy as np -from ple import PLE -from ple.games import Doom - -class NaiveAgent(): - """ - This is our naive agent. It picks actions at random! - """ - def __init__(self, actions): - self.actions = actions - - def pickAction(self, reward, obs): - return self.actions[np.random.randint(0, len(self.actions))] - -################################### -game = Doom(scenario="take_cover") - -env = PLE(game) -agent = NaiveAgent(env.getActionSet()) -env.init() - -reward = 0.0 -for f in range(15000): - #if the game is over - if env.game_over(): - env.reset_game() - - action = agent.pickAction(reward, env.getScreenRGB()) - reward = env.act(action) - - if f > 2000: - env.display_screen = True - env.force_fps = False - - if f > 2250: - env.display_screen = True - env.force_fps = True diff --git a/PyGame-Learning-Environment/examples/example_support.py b/PyGame-Learning-Environment/examples/example_support.py deleted file mode 100644 index c6cb71b..0000000 --- a/PyGame-Learning-Environment/examples/example_support.py +++ /dev/null @@ -1,214 +0,0 @@ -import numpy as np -from collections import deque - -# keras and model related -from keras.models import Sequential -from keras.layers.core import Dense, Flatten -from keras.layers.convolutional import Convolution2D -from keras.optimizers import SGD, Adam, RMSprop -import theano.tensor as T - - -class ExampleAgent(): - """ - Implements a DQN-ish agent. It has replay memory and epsilon decay. It is missing model freezing. The models are sensitive to the parameters and if applied to other games must be tinkered with. - - """ - - def __init__(self, env, batch_size, num_frames, - frame_skip, lr, discount, rng, optimizer="adam", frame_dim=None): - - self.env = env - self.batch_size = batch_size - self.num_frames = num_frames - self.frame_skip = frame_skip - self.lr = lr - self.discount = discount - self.rng = rng - - if optimizer == "adam": - opt = Adam(lr=self.lr) - elif optimizer == "sgd": - opt = SGD(lr=self.lr) - elif optimizer == "sgd_nesterov": - opt = SGD(lr=self.lr, nesterov=True) - elif optimizer == "rmsprop": - opt = RMSprop(lr=self.lr, rho=0.9, epsilon=0.003) - else: - raise ValueError("Unrecognized optmizer") - - self.optimizer = opt - - self.frame_dim = self.env.getScreenDims() if frame_dim is None else frame_dim - self.state_shape = (num_frames,) + self.frame_dim - self.input_shape = (batch_size,) + self.state_shape - - self.state = deque(maxlen=num_frames) - self.actions = self.env.getActionSet() - self.num_actions = len(self.actions) - self.model = None - - def q_loss(self, y_true, y_pred): - # assume clip_delta is 1.0 - # along with sum accumulator. - diff = y_true - y_pred - _quad = T.minimum(abs(diff), 1.0) - _lin = abs(diff) - _quad - loss = 0.5 * _quad ** 2 + _lin - loss = T.sum(loss) - - return loss - - def build_model(self): - - model = Sequential() - model.add(Convolution2D( - 16, 8, 8, input_shape=(self.num_frames,) + self.frame_dim, - subsample=(4, 4), activation="relu", init="he_uniform" - )) - model.add(Convolution2D( - 16, 4, 4, subsample=(2, 2), activation="relu", init="he_uniform" - )) - model.add(Convolution2D( - 32, 3, 3, subsample=(1, 1), activation="relu", init="he_uniform" - )) - model.add(Flatten()) - model.add(Dense( - 512, activation="relu", init="he_uniform" - )) - model.add(Dense( - self.num_actions, activation="linear", init="he_uniform" - )) - - model.compile(loss=self.q_loss, optimizer=self.optimizer) - - self.model = model - - def predict_single(self, state): - """ - model is expecting a batch_size worth of data. We only have one states worth of - samples so we make an empty batch and set our state as the first row. - """ - states = np.zeros(self.input_shape) - states[0, ...] = state.reshape(self.state_shape) - - return self.model.predict(states)[0] # only want the first value - - def _argmax_rand(self, arr): - # picks a random index if there is a tie - return self.rng.choice(np.where(arr == np.max(arr))[0]) - - def _best_action(self, state): - q_vals = self.predict_single(state) - - return self._argmax_rand(q_vals) # the action with the best Q-value - - def act(self, state, epsilon=1.0): - self.state.append(state) - - action = self.rng.randint(0, self.num_actions) - if len(self.state) == self.num_frames: # we havent seen enough frames - _state = np.array(self.state) - - if self.rng.rand() > epsilon: - action = self._best_action(_state) # exploit - - reward = 0.0 - for i in range(self.frame_skip): # we repeat each action a few times - # act on the environment - reward += self.env.act(self.actions[action]) - - reward = np.clip(reward, -1.0, 1.0) - - return reward, action - - def start_episode(self, N=3): - self.env.reset_game() # reset - for i in range(self.rng.randint(N)): - self.env.act(self.env.NOOP) # perform a NOOP - - def end_episode(self): - self.state.clear() - - -class ReplayMemory(): - - def __init__(self, max_size, min_size): - self.min_replay_size = min_size - self.memory = deque(maxlen=max_size) - - def __len__(self): - return len(self.memory) - - def add(self, transition): - self.memory.append(transition) - - def train_agent_batch(self, agent): - if len(self.memory) > self.min_replay_size: - states, targets = self._random_batch(agent) # get a random batch - return agent.model.train_on_batch(states, targets) # ERR? - else: - return None - - def _random_batch(self, agent): - inputs = np.zeros(agent.input_shape) - targets = np.zeros((agent.batch_size, agent.num_actions)) - - seen = [] - idx = agent.rng.randint( - 0, - high=len( - self.memory) - - agent.num_frames - - 1) - - for i in range(agent.batch_size): - while idx in seen: - idx = agent.rng.randint(0, high=len( - self.memory) - agent.num_frames - 1) - - states = np.array([self.memory[idx + j][0] - for j in range(agent.num_frames + 1)]) - art = np.array([self.memory[idx + j][1:] - for j in range(agent.num_frames)]) - - actions = art[:, 0].astype(int) - rewards = art[:, 1] - terminals = art[:, 2] - - state = states[:-1] - state_next = states[1:] - - inputs[i, ...] = state.reshape(agent.state_shape) - # we could make zeros but pointless. - targets[i] = agent.predict_single(state) - Q_prime = np.max(agent.predict_single(state_next)) - - targets[i, actions] = rewards + \ - (1 - terminals) * (agent.discount * Q_prime) - - seen.append(idx) - - return inputs, targets - - -def loop_play_forever(env, agent): - # our forever play loop - try: - # slow it down - env.display_screen = True - env.force_fps = False - - while True: - agent.start_episode() - episode_reward = 0.0 - while env.game_over() == False: - state = env.getGameState() - reward, action = agent.act(state, epsilon=0.05) - episode_reward += reward - - print "Agent score {:0.1f} reward for episode.".format(episode_reward) - agent.end_episode() - - except KeyboardInterrupt: - print "Exiting out!" diff --git a/PyGame-Learning-Environment/examples/keras_nonvis.py b/PyGame-Learning-Environment/examples/keras_nonvis.py deleted file mode 100644 index 9c05c1e..0000000 --- a/PyGame-Learning-Environment/examples/keras_nonvis.py +++ /dev/null @@ -1,166 +0,0 @@ -# thanks to @edersantana and @fchollet for suggestions & help. - -import numpy as np -from ple import PLE # our environment -from ple.games.catcher import Catcher - -from keras.models import Sequential -from keras.layers.core import Dense -from keras.optimizers import SGD - -from example_support import ExampleAgent, ReplayMemory, loop_play_forever - - -class Agent(ExampleAgent): - """ - Our agent takes 1D inputs which are flattened. - We define a full connected model below. - """ - - def __init__(self, *args, **kwargs): - ExampleAgent.__init__(self, *args, **kwargs) - - self.state_dim = self.env.getGameStateDims() - self.state_shape = np.prod((num_frames,) + self.state_dim) - self.input_shape = (batch_size, self.state_shape) - - def build_model(self): - model = Sequential() - model.add(Dense( - input_dim=self.state_shape, output_dim=256, activation="relu", init="he_uniform" - )) - model.add(Dense( - 512, activation="relu", init="he_uniform" - )) - model.add(Dense( - self.num_actions, activation="linear", init="he_uniform" - )) - - model.compile(loss=self.q_loss, optimizer=SGD(lr=self.lr)) - - self.model = model - - -def nv_state_preprocessor(state): - """ - This preprocesses our state from PLE. We rescale the values to be between - 0,1 and -1,1. - """ - # taken by inspection of source code. Better way is on its way! - max_values = np.array([128.0, 20.0, 128.0, 128.0]) - state = np.array([state.values()]) / max_values - - return state.flatten() - -if __name__ == "__main__": - # this takes about 15 epochs to converge to something that performs decently. - # feel free to play with the parameters below. - - # training parameters - num_epochs = 15 - num_steps_train = 15000 # steps per epoch of training - num_steps_test = 3000 - update_frequency = 4 # step frequency of model training/updates - - # agent settings - batch_size = 32 - num_frames = 4 # number of frames in a 'state' - frame_skip = 2 - # percentage of time we perform a random action, help exploration. - epsilon = 0.15 - epsilon_steps = 30000 # decay steps - epsilon_min = 0.1 - lr = 0.01 - discount = 0.95 # discount factor - rng = np.random.RandomState(24) - - # memory settings - max_memory_size = 100000 - min_memory_size = 1000 # number needed before model training starts - - epsilon_rate = (epsilon - epsilon_min) / epsilon_steps - - # PLE takes our game and the state_preprocessor. It will process the state - # for our agent. - game = Catcher(width=128, height=128) - env = PLE(game, fps=60, state_preprocessor=nv_state_preprocessor) - - agent = Agent(env, batch_size, num_frames, frame_skip, lr, - discount, rng, optimizer="sgd_nesterov") - agent.build_model() - - memory = ReplayMemory(max_memory_size, min_memory_size) - - env.init() - - for epoch in range(1, num_epochs + 1): - steps, num_episodes = 0, 0 - losses, rewards = [], [] - env.display_screen = False - - # training loop - while steps < num_steps_train: - episode_reward = 0.0 - agent.start_episode() - - while env.game_over() == False and steps < num_steps_train: - state = env.getGameState() - reward, action = agent.act(state, epsilon=epsilon) - memory.add([state, action, reward, env.game_over()]) - - if steps % update_frequency == 0: - loss = memory.train_agent_batch(agent) - - if loss is not None: - losses.append(loss) - epsilon = np.max(epsilon_min, epsilon - epsilon_rate) - - episode_reward += reward - steps += 1 - - if num_episodes % 5 == 0: - print "Episode {:01d}: Reward {:0.1f}".format(num_episodes, episode_reward) - - rewards.append(episode_reward) - num_episodes += 1 - agent.end_episode() - - print "\nTrain Epoch {:02d}: Epsilon {:0.4f} | Avg. Loss {:0.3f} | Avg. Reward {:0.3f}".format(epoch, epsilon, np.mean(losses), np.sum(rewards) / num_episodes) - - steps, num_episodes = 0, 0 - losses, rewards = [], [] - - # display the screen - env.display_screen = True - - # slow it down so we can watch it fail! - env.force_fps = False - - # testing loop - while steps < num_steps_test: - episode_reward = 0.0 - agent.start_episode() - - while env.game_over() == False and steps < num_steps_test: - state = env.getGameState() - reward, action = agent.act(state, epsilon=0.05) - - episode_reward += reward - steps += 1 - - # done watching after 500 steps. - if steps > 500: - env.force_fps = True - env.display_screen = False - - if num_episodes % 5 == 0: - print "Episode {:01d}: Reward {:0.1f}".format(num_episodes, episode_reward) - - rewards.append(episode_reward) - num_episodes += 1 - agent.end_episode() - - print "Test Epoch {:02d}: Best Reward {:0.3f} | Avg. Reward {:0.3f}".format(epoch, np.max(rewards), np.sum(rewards) / num_episodes) - - print "\nTraining complete. Will loop forever playing!" - loop_play_forever(env, agent) diff --git a/PyGame-Learning-Environment/examples/random_agent.py b/PyGame-Learning-Environment/examples/random_agent.py deleted file mode 100644 index 29f8195..0000000 --- a/PyGame-Learning-Environment/examples/random_agent.py +++ /dev/null @@ -1,57 +0,0 @@ -import numpy as np -from ple import PLE -from ple.games.raycastmaze import RaycastMaze - - -class NaiveAgent(): - """ - This is our naive agent. It picks actions at random! - """ - - def __init__(self, actions): - self.actions = actions - - def pickAction(self, reward, obs): - return self.actions[np.random.randint(0, len(self.actions))] - -################################### -game = RaycastMaze( - map_size=6 -) # create our game - -fps = 30 # fps we want to run at -frame_skip = 2 -num_steps = 1 -force_fps = False # slower speed -display_screen = True - -reward = 0.0 -max_noops = 20 -nb_frames = 15000 - -# make a PLE instance. -p = PLE(game, fps=fps, frame_skip=frame_skip, num_steps=num_steps, - force_fps=force_fps, display_screen=display_screen) - -# our Naive agent! -agent = NaiveAgent(p.getActionSet()) - -# init agent and game. -p.init() - -# lets do a random number of NOOP's -for i in range(np.random.randint(0, max_noops)): - reward = p.act(p.NOOP) - -# start our training loop -for f in range(nb_frames): - # if the game is over - if p.game_over(): - p.reset_game() - - obs = p.getScreenRGB() - action = agent.pickAction(reward, obs) - reward = p.act(action) - - if f % 50 == 0: - p.saveScreen("screen_capture.png") diff --git a/PyGame-Learning-Environment/examples/scaling_rewards.py b/PyGame-Learning-Environment/examples/scaling_rewards.py deleted file mode 100644 index 2897111..0000000 --- a/PyGame-Learning-Environment/examples/scaling_rewards.py +++ /dev/null @@ -1,29 +0,0 @@ -import numpy as np -from ple import PLE -from ple.games.waterworld import WaterWorld - - -# lets adjust the rewards our agent recieves -rewards = { - "tick": -0.01, # each time the game steps forward in time the agent gets -0.1 - "positive": 1.0, # each time the agent collects a green circle - "negative": -5.0, # each time the agent bumps into a red circle -} - -# make a PLE instance. -# use lower fps so we can see whats happening a little easier -game = WaterWorld(width=256, height=256, num_creeps=8) -p = PLE(game, fps=15, force_fps=False, display_screen=True, - reward_values=rewards) -# we pass in the rewards and PLE will adjust the game for us - -p.init() -actions = p.getActionSet() -for i in range(1000): - if p.game_over(): - p.reset_game() - - action = actions[np.random.randint(0, len(actions))] # random actions - reward = p.act(action) - - print "Score: {:0.3f} | Reward: {:0.3f} ".format(p.score(), reward) diff --git a/PyGame-Learning-Environment/ple_games.jpg b/PyGame-Learning-Environment/ple_games.jpg deleted file mode 100644 index bf54029..0000000 Binary files a/PyGame-Learning-Environment/ple_games.jpg and /dev/null differ diff --git a/PyGame-Learning-Environment/requirements-dev.txt b/PyGame-Learning-Environment/requirements-dev.txt deleted file mode 100644 index 6a7c32f..0000000 --- a/PyGame-Learning-Environment/requirements-dev.txt +++ /dev/null @@ -1,4 +0,0 @@ -mock==1.0.1 -numpydoc -Sphinx==1.2.3 -sphinx_rtd_theme diff --git a/PyGame-Learning-Environment/setup.py b/PyGame-Learning-Environment/setup.py deleted file mode 100644 index 5209e97..0000000 --- a/PyGame-Learning-Environment/setup.py +++ /dev/null @@ -1,33 +0,0 @@ -import os - -from setuptools import find_packages -from setuptools import setup - -here = os.path.abspath(os.path.dirname(__file__)) - -install_requires = [ - "numpy", - "Pillow" -] - -setup( - name='ple', - version='0.0.1', - description='PyGame Learning Environment', - classifiers=[ - "Intended Audience :: Developers", - "Intended Audience :: Science/Research", - "License :: OSI Approved :: MIT License", - "Programming Language :: Python :: 2.7", - "Topic :: Scientific/Engineering :: Artificial Intelligence", - ], - url='https://github.com/ntasfi/PyGame-Learning-Environment', - author='Norman Tasfi', - author_email='first letter of first name plus last at googles email service.', - keywords='', - license="MIT", - packages=find_packages(), - include_package_data=False, - zip_safe=False, - install_requires=install_requires -) diff --git a/PyGame-Learning-Environment/tests/test_ple.py b/PyGame-Learning-Environment/tests/test_ple.py deleted file mode 100644 index a814040..0000000 --- a/PyGame-Learning-Environment/tests/test_ple.py +++ /dev/null @@ -1,91 +0,0 @@ -#!/usr/bin/python - - -""" - -This tests that all the PLE games launch, except for doom; we -explicitly check that it isn't defined. - - -""" - - -import nose -import numpy as np -import unittest - -NUM_STEPS=150 - -class NaiveAgent(): - def __init__(self, actions): - self.actions = actions - def pickAction(self, reward, obs): - return self.actions[np.random.randint(0, len(self.actions))] - - -class MyTestCase(unittest.TestCase): - - def run_a_game(self,game): - from ple import PLE - p = PLE(game,display_screen=True) - agent = NaiveAgent(p.getActionSet()) - p.init() - reward = p.act(p.NOOP) - for i in range(NUM_STEPS): - obs = p.getScreenRGB() - reward = p.act(agent.pickAction(reward,obs)) - - def test_catcher(self): - from ple.games.catcher import Catcher - game = Catcher() - self.run_a_game(game) - - def test_monsterkong(self): - from ple.games.monsterkong import MonsterKong - game = MonsterKong() - self.run_a_game(game) - - def test_flappybird(self): - from ple.games.flappybird import FlappyBird - game = FlappyBird() - self.run_a_game(game) - - def test_pixelcopter(self): - from ple.games.pixelcopter import Pixelcopter - game = Pixelcopter() - self.run_a_game(game) - - def test_puckworld(self): - from ple.games.puckworld import PuckWorld - game = PuckWorld() - self.run_a_game(game) - - def test_raycastmaze(self): - from ple.games.raycastmaze import RaycastMaze - game = RaycastMaze() - self.run_a_game(game) - - def test_snake(self): - from ple.games.snake import Snake - game = Snake() - self.run_a_game(game) - - def test_waterworld(self): - from ple.games.waterworld import WaterWorld - game = WaterWorld() - self.run_a_game(game) - - def test_pong(self): - from ple.games.pong import Pong - game = Pong() - self.run_a_game(game) - - def test_doom_not_defined(self): - from nose.tools import assert_raises - def invoke_doom(): - DoomWrapper - assert_raises(NameError,invoke_doom) - - -if __name__ == "__main__": - nose.runmodule() diff --git a/PyGame-Learning-Environment/tests/test_ple_doom.py b/PyGame-Learning-Environment/tests/test_ple_doom.py deleted file mode 100644 index 56783b6..0000000 --- a/PyGame-Learning-Environment/tests/test_ple_doom.py +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/python - -import nose - - -import nose -import numpy as np -import unittest - - -class NaiveAgent(): - def __init__(self, actions): - self.actions = actions - def pickAction(self, reward, obs): - return self.actions[np.random.randint(0, len(self.actions))] - - -class MyTestCase(unittest.TestCase): - - def run_a_game(self,game): - - from ple import PLE - p = PLE(game) - agent = NaiveAgent(p.getActionSet()) - p.init() - reward = p.act(p.NOOP) - for i in range(50): - obs = p.getScreenRGB() - reward = p.act(agent.pickAction(reward,obs)) - - def test_doom(self): - from ple.games.doom import Doom - game = Doom() - self.run_a_game(game) - -if __name__ == "__main__": - nose.runmodule() - - diff --git a/RandomBird/FlappyAgent.py b/RandomBird/FlappyAgent.py deleted file mode 100644 index 9f3ec84..0000000 --- a/RandomBird/FlappyAgent.py +++ /dev/null @@ -1,9 +0,0 @@ -import numpy as np - -def FlappyPolicy(state, screen): - action=None - if(np.random.randint(0,2)<1): - action=119 - return action - -