Main.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "IuVxe65AoY_M"
      },
      "outputs": [],
      "source": [
        "import numpy as np\n",
        "import tensorflow as tf\n",
        "from tensorflow import keras\n",
        "import time\n",
        "import copy\n",
        "from matplotlib import pyplot as plt\n",
        "from IPython.display import clear_output"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "class Grid_world:\n",
        "  def __init__(self, map_num=None, shape=None, n_obstacles=None, n_goals=None, like=None, ordered=False):\n",
        "    \n",
        "    self.shape = shape\n",
        "    self.map_num = map_num\n",
        "    self.map = None\n",
        "    self.ordered = ordered\n",
        "    self.n_goals = n_goals\n",
        "    self.n_obstacles = n_obstacles\n",
        "    \n",
        "    if like != None:\n",
        "      self.shape = like.shape\n",
        "      self.map = like.map.copy()\n",
        "      self.n_goals = like.n_goals\n",
        "      self.n_obstacles = like.n_obstacles\n",
        "      self.ordered = like.ordered\n",
        "\n",
        "    elif map_num != None:\n",
        "      self.map = self.set_grid_world(map_num)\n",
        "      self.shape = self.map.shape\n",
        "    elif shape != None:\n",
        "      self.generate_grid_world(self.shape, n_obstacles, n_goals)\n",
        "    else:\n",
        "      print(\"map_num & shape can't be 'None' at the same time.\")\n",
        "\n",
        "  def set_grid_world(self, map):\n",
        "        if map == 0:\n",
        "          grid_world = np.zeros((5,5))\n",
        "\n",
        "          grid_world[0,4] = 2 # destination\n",
        "          grid_world[0,0] = 1 # starting point\n",
        "\n",
        "        elif map == 1:\n",
        "\n",
        "          grid_world = np.zeros((5,5))\n",
        "          grid_world[[2,3],3] = -1 # holes\n",
        "          grid_world[[0,1],1] = -1 # holes\n",
        "\n",
        "          grid_world[0, 4] = 2 # destination\n",
        "          grid_world[0,0] = 1 # starting point\n",
        "\n",
        "        elif map == 2:\n",
        "\n",
        "          grid_world = np.zeros((5,5))\n",
        "          grid_world[[0,2,3],3] = -1 # holes\n",
        "          grid_world[[0,1,2,4],1] = -1 # holes\n",
        "\n",
        "          grid_world[0, 4] = 2 # destination\n",
        "          grid_world[0,0] = 1 # starting point\n",
        "\n",
        "        elif map == 3:\n",
        "\n",
        "          grid_world = np.zeros((10,10))\n",
        "          grid_world[[0,6,7,9],1] = -1 # holes\n",
        "          grid_world[[0,1,2,3,8],3] = -1 # holes\n",
        "          grid_world[[0,1,2,3,4,5],5] = -1 # holes\n",
        "\n",
        "          grid_world[-1, -1] = 2 # destination\n",
        "          grid_world[0,0] = 1 # starting point\n",
        "\n",
        "        elif map == 4:\n",
        "\n",
        "          grid_world = np.zeros((10,10))\n",
        "          grid_world[[0,1,2,4,6,7,9],1] = -1 # holes\n",
        "          grid_world[[0,2,3, 6,7,8],3] = -1 # holes\n",
        "          grid_world[[1,2,3,4,5,6],5] = -1 # holes\n",
        "          grid_world[[2,3,5,6,7,9],7] = -1 # holes\n",
        "\n",
        "          grid_world[-1, -1] = 2 # destination\n",
        "          grid_world[0,0] = 1 # starting point\n",
        "        \n",
        "        elif map == 5:\n",
        "\n",
        "          grid_world = np.zeros((16,16))\n",
        "          grid_world[[0,1,2,4,6,7,9],1] = -1 # holes\n",
        "          grid_world[[0,2,3, 6,7,8],3] = -1 # holes\n",
        "          grid_world[[1,2,3,4,5,6],5] = -1 # holes\n",
        "          grid_world[[2,3,5,6,7,9],7] = -1 # holes\n",
        "          grid_world[[4,5,6,7,8,9,10,11,12,13,14,15],9] = -1 # holes\n",
        "          grid_world[[1,2,3,4,5,6,7,8, 12, 14],11] = -1 # holes\n",
        "          grid_world[[7,9,10,11,12,13,15],13] = -1 # holes\n",
        "\n",
        "          grid_world[-2, -3] = 2 # destination\n",
        "          grid_world[0,0] = 1 # starting point\n",
        "        \n",
        "        elif map == 6: # First case study of The \"Control Synthesis from LTL Specifications using RL\" Paper\n",
        "\n",
        "          grid_world = np.zeros((5,4))\n",
        "          grid_world[[0,2],2] = -1 # holes\n",
        "          grid_world[4,[1,3]] = -1 # holes\n",
        "          ## Obsticle ##\n",
        "\n",
        "          grid_world[1, [2,3]] = 2 # destination\n",
        "          grid_world[3, [0,2]] = 2 # destination\n",
        "          grid_world[0, 0] = 1 # starting point\n",
        "\n",
        "        elif map == 10:\n",
        "\n",
        "          grid_world = np.zeros((3,3))\n",
        "          grid_world[1,[0,1]] = -1 # holes\n",
        "\n",
        "          grid_world[-1, 0] = 2 # destination\n",
        "          grid_world[0,0] = 1 # starting point\n",
        "\n",
        "        elif map == 20:\n",
        "\n",
        "          grid_world = np.zeros((8,8))\n",
        "\n",
        "          grid_world[[0,-1], -1] = 2 # destinations\n",
        "          grid_world[0,0] = 1 # starting point\n",
        "\n",
        "        elif map == 21:\n",
        "\n",
        "          grid_world = np.zeros((5,5))\n",
        "\n",
        "          grid_world[[0,-1], -1] = 2 # destinations\n",
        "          grid_world[0,0] = 1 # starting point\n",
        "\n",
        "          grid_world[[0,2,3],3] = -1 # holes\n",
        "          grid_world[[0,1,2,4],1] = -1 # holes\n",
        "        \n",
        "        elif map == 22:\n",
        "\n",
        "          grid_world = np.zeros((5,5))\n",
        "\n",
        "          grid_world[[0,-1], -1] = 2 # destinations\n",
        "          grid_world[[0,-1], -3] = 2 # destinations\n",
        "          grid_world[0,0] = 1 # starting point\n",
        "\n",
        "          grid_world[[0,2,3],3] = -1 # holes\n",
        "          grid_world[[0,1,2,4],1] = -1 # holes\n",
        "\n",
        "        elif map == 24:\n",
        "\n",
        "          grid_world = np.zeros((10,10))\n",
        "\n",
        "          grid_world[[0,-1], -1] = 2 # destinations\n",
        "          grid_world[0,0] = 1 # starting point\n",
        "          \n",
        "          grid_world[[0,6,7,9],1] = -1 # holes\n",
        "          grid_world[[0,1,2,3,8],3] = -1 # holes\n",
        "          grid_world[[0,1,2,3,4,5],5] = -1 # holes\n",
        "\n",
        "        elif map == 25:\n",
        "\n",
        "          grid_world = np.zeros((16,16))\n",
        "\n",
        "          grid_world[[0,-1], -1] = 2 # destinations\n",
        "          grid_world[[2,-3], -4] = 2 # destinations\n",
        "          grid_world[0,0] = 1 # starting point\n",
        "          \n",
        "          grid_world[[0,6,7,9],1] = -1 # holes\n",
        "          grid_world[[0,1,2,3,8],3] = -1 # holes\n",
        "          grid_world[[0,1,2,3,4,5],5] = -1 # holes\n",
        "\n",
        "        elif map == 30:\n",
        "\n",
        "          grid_world = np.zeros((10,10))\n",
        "\n",
        "          grid_world[0, -1] = 2 # destinations\n",
        "          grid_world[1, -2] = 3 # destinations\n",
        "          grid_world[2, -3] = 4 # destinations\n",
        "          grid_world[0,0] = 1 # starting point\n",
        "          \n",
        "          grid_world[[0,6,7,9],1] = -1 # holes\n",
        "          grid_world[[0,1,2,3,8],3] = -1 # holes\n",
        "          grid_world[[0,1,2,3,4,5],5] = -1 # holes\n",
        "\n",
        "        return grid_world.astype(int)\n",
        "\n",
        "  def get_locations(self, target, absolute_loc=True, as_list=False):\n",
        "\n",
        "      if target==None: i, j = np.where(self.map==2)\n",
        "      elif type(target)==int: i, j = np.where(self.map==target)\n",
        "      else:\n",
        "        i, j = [], []\n",
        "        for t in target:\n",
        "          x,y = np.where(self.map==t)\n",
        "          try:\n",
        "              i.append(x[0])\n",
        "              j.append(y[0])\n",
        "          except:\n",
        "            pass # the goal was overwritten by the agent, ignore it\n",
        "\n",
        "      if absolute_loc:\n",
        "        if len(i)>1 or as_list: return [(i[k]*len(self.map) + j[k]) for k in range(len(i))]\n",
        "        else: return i[0]*len(self.map) + j[0]\n",
        "\n",
        "      else:\n",
        "        if len(i)>1 or as_list: return [(i[k], j[k], i[k]*len(self.map) + j[k]) for k in range(len(i))]\n",
        "        else: return i[0], j[0], i[0]*len(self.map) + j[0]\n",
        "\n",
        "  def get_surroundings(self, location_i, location_j):\n",
        "\n",
        "      surroundings = -np.ones(4)\n",
        "      if location_j < len(self.map)-1:\n",
        "          surroundings[0] = self.map[location_i][location_j+1]\n",
        "      if location_i > 0:\n",
        "          surroundings[1] = self.map[location_i-1][location_j]\n",
        "      if location_j > 0:\n",
        "          surroundings[2] = self.map[location_i][location_j-1]\n",
        "      if location_i < len(self.map)-1:\n",
        "          surroundings[3] = self.map[location_i+1][location_j]\n",
        "      return surroundings\n",
        "\n",
        "  def add_item(self, n_items, item):\n",
        "\n",
        "      x,y = self.shape\n",
        "      for _ in range(n_items):\n",
        "        i,j = np.random.randint(x), np.random.randint(y)\n",
        "        while(self.map[i,j]!=0):\n",
        "          i,j = np.random.randint(x), np.random.randint(y)\n",
        "        self.map[i,j] = item\n",
        "        if self.ordered: item += 1\n",
        "\n",
        "  def add_obstacles(self, n_obstacles=None):\n",
        "\n",
        "      x,y = self.shape\n",
        "\n",
        "      obstacles_in_row = x//2\n",
        "\n",
        "      k = 0\n",
        "      for j in range(1,y,2):\n",
        "        for k in range(obstacles_in_row):\n",
        "          i = np.random.randint(x)\n",
        "          if self.map[i,j]==0:\n",
        "            self.map[i,j] = -1\n",
        "        \n",
        "  def generate_grid_world(self, shape, n_obstacles, n_goals):\n",
        "\n",
        "    self.map = np.zeros(shape)\n",
        "    self.map[0,0] = 1 # starting point\n",
        "\n",
        "    self.add_item(n_goals, 2) # adding goals\n",
        "\n",
        "    self.add_obstacles(n_obstacles)\n",
        "\n",
        "  def get_goal_trajectory(self, offset=2):\n",
        "    if self.ordered: return [i+offset for i in range(self.n_goals)]\n",
        "    # else: return [offset for i in range(self.n_goals)]\n",
        "    else: return None\n",
        "  \n",
        "  def copy(self):\n",
        "    return Grid_world(like=self)"
      ],
      "metadata": {
        "id": "8SSft-fJog6y"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "grid_world = Grid_world(shape=(6,6), n_goals=2)"
      ],
      "metadata": {
        "id": "EsZ8-BQA233n"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from keras.layers import Input, Dense, Concatenate, Flatten\n",
        "from keras.models import Model\n",
        "\n",
        "def build_model(grid_world_shape):\n",
        "  inputs = Input(shape=grid_world_shape)\n",
        "\n",
        "  inputs = Flatten()(inputs)\n",
        "  x = Dense(32, activation='relu')(inputs)\n",
        "  x = Dense(16, activation='relu')(x)\n",
        "  move_predictions = Dense(5, activation='softmax')(x)\n",
        "  rew_predictions = Dense(1, activation='tanh')(x)\n",
        "\n",
        "  model = Model(inputs=inputs, outputs=(move_predictions, rew_predictions))\n",
        "  # model = Model(inputs=inputs, outputs=move_predictions)\n",
        "  model.compile(optimizer='adam',\n",
        "                loss=['categorical_crossentropy', 'mse'],\n",
        "                metrics=['accuracy'])\n",
        "  return model\n",
        "\n",
        "model = build_model(grid_world.shape)\n",
        "model.summary()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MyMZk6xeoira",
        "outputId": "2733faf5-1887-427e-c1ef-55d3d30c773e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"model\"\n",
            "__________________________________________________________________________________________________\n",
            " Layer (type)                   Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            " input_2 (InputLayer)           [(None, 36)]         0           []                               \n",
            "                                                                                                  \n",
            " dense (Dense)                  (None, 32)           1184        ['input_2[0][0]']                \n",
            "                                                                                                  \n",
            " dense_1 (Dense)                (None, 16)           528         ['dense[1][0]']                  \n",
            "                                                                                                  \n",
            " dense_2 (Dense)                (None, 5)            85          ['dense_1[1][0]']                \n",
            "                                                                                                  \n",
            " dense_3 (Dense)                (None, 1)            17          ['dense_1[1][0]']                \n",
            "                                                                                                  \n",
            "==================================================================================================\n",
            "Total params: 1,814\n",
            "Trainable params: 1,814\n",
            "Non-trainable params: 0\n",
            "__________________________________________________________________________________________________\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def make_move(grid_world, loc_i, loc_j, move):\n",
        "    \n",
        "    new_loc_i, new_loc_j = loc_i, loc_j\n",
        "    \n",
        "    if move==4:\n",
        "        pass\n",
        "    elif move==0 and loc_j< len(grid_world.map[0])-1:\n",
        "        new_loc_j += 1\n",
        "    elif move==1 and loc_i > 0:\n",
        "        new_loc_i -= 1\n",
        "    elif move==2 and loc_j > 0:\n",
        "        new_loc_j -= 1\n",
        "    elif move==3 and loc_i < len(grid_world.map)-1:\n",
        "        new_loc_i += 1\n",
        "\n",
        "    new_grid_world = Grid_world(like=grid_world)\n",
        "    # apply the change in the real world\n",
        "    new_grid_world.map[loc_i][loc_j] = 0\n",
        "    new_grid_world.map[new_loc_i][new_loc_j] = 1\n",
        "\n",
        "    return new_grid_world, new_loc_i, new_loc_j"
      ],
      "metadata": {
        "id": "0d_RAn0FoiuL"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def check_LTL(formula, trajectory, predicates):\n",
        "  \"\"\"\n",
        "  a recursive function to evaluate an LTL formula over a tragectory. (for now only supports &, |, ~, ^ and #).\n",
        "  Note on formula structure: Formula should be a nested list of operators/operands.\n",
        "  each member is either a list of (evaluated) booleans, or an unevaluated tuple of shape: ('operator','operand', [other operands]) like ('^', 'sth' ) for eventually sth.\n",
        "  or ('&', 'sth1', 'sth2') for sth1 & sth2. also for predicates, the operator is 'None'.\n",
        "  Example: '^#a' (eventually always a) -> the formula should look like this: ('^', ('#', (None, 'a') ) )\n",
        "\n",
        "  inputs:\n",
        "  formula: is the LTL formula that we want to evaluate. it can include: '^' for eventually and '#' for always.\n",
        "  trajectory: is the history of the visited states by the agent. we evaluate the formula over this (example: [1,4,7,3,7,...])\n",
        "  predicates: a dict of predicates used in the formula, with their corresponding True conditions. (example: {'a':[1,3,4,...]})\n",
        "\n",
        "  outputs:\n",
        "  a sequence of evaluations of the input formula over every time step of the trajectory. (example: [False, True, False,...])\n",
        "  \"\"\"\n",
        "\n",
        "  # check if current formula needs evaluation\n",
        "  if type(formula)==list: # already evaluated\n",
        "    return list\n",
        "  \n",
        "  # check if current formula is a predicate or not\n",
        "  if formula[0]==None: # a predicate\n",
        "    # need to evaluate predicate over the trajectory, and return it\n",
        "    evaluation = []\n",
        "    for i in trajectory:\n",
        "      eval = i in predicates[formula[1]]\n",
        "      evaluation.append(eval)\n",
        "    return evaluation\n",
        "  \n",
        "  # the formula is an unevaluated operator\n",
        "  # now evaluate the operator over the operand(s)\n",
        "  evaluation = []\n",
        "  if formula[0]=='^':\n",
        "    # Evaluate the operand.\n",
        "    evaluated_operand = check_LTL(formula[1], trajectory, predicates)\n",
        "    evaluation = [True in evaluated_operand[i:] for i in range(0,len(evaluated_operand))]\n",
        "\n",
        "  elif formula[0]=='#':\n",
        "    # Evaluate the operand.\n",
        "    evaluated_operand = check_LTL(formula[1], trajectory, predicates)\n",
        "    evaluation = [all(evaluated_operand[i:]) for i in range(0,len(evaluated_operand))]\n",
        "\n",
        "  elif formula[0]=='&':\n",
        "    # Evaluate the operands.\n",
        "    evaluated_operand1 = check_LTL(formula[1], trajectory, predicates)\n",
        "    evaluated_operand2 = check_LTL(formula[2], trajectory, predicates)\n",
        "    evaluation = [all([evaluated_operand1[i],evaluated_operand2[i]]) for i in range(0, len(evaluated_operand1))]\n",
        "\n",
        "  elif formula[0]=='|':\n",
        "    # Evaluate the operands.\n",
        "    evaluated_operand1 = check_LTL(formula[1], trajectory, predicates)\n",
        "    evaluated_operand2 = check_LTL(formula[2], trajectory, predicates)\n",
        "    evaluation = [any([evaluated_operand1[i],evaluated_operand2[i]]) for i in range(0, len(evaluated_operand1))]\n",
        "\n",
        "  elif formula[0]=='~':\n",
        "    # Evaluate the operand.\n",
        "    evaluated_operand = check_LTL(formula[1], trajectory, predicates)\n",
        "    evaluation = [not i for i in evaluated_operand]\n",
        "\n",
        "  else:\n",
        "    print(\"Unknown operator\")\n",
        "  \n",
        "  # end of evaluation\n",
        "  \n",
        "  return evaluation\n",
        "\n",
        "predicates={'a':[1], 'b':[2], 'c':[3]}\n",
        "\n",
        "formula1 = ('#', ('~', (None, 'c')))\n",
        "formula21 = ('^', (None, 'a'))\n",
        "formula22 = ('^', ('#', (None, 'a')))\n",
        "formula31 = ('^', (None, 'b'))\n",
        "formula32 = ('^', ('#', (None, 'b')))\n",
        "formula4 = ('|', formula22, formula32)\n",
        "formula = ('&', formula1, formula4)\n",
        "\n",
        "trajectory = [0,0,0,2,0,0,1,1]\n",
        "\n",
        "print(check_LTL(formula, trajectory, predicates))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "AhRE7ndDdS2Z",
        "outputId": "2854d3c3-dde2-4c86-96e2-a1e1cf474e00"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[True, True, True, True, True, True, True, True]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def hash_state(state):\n",
        "  hash = ''\n",
        "  for i in state.flatten():\n",
        "    hash += str(i)\n",
        "  return hash\n",
        "\n",
        "# def check_obstacles(trajectory, obstacle_locations):\n",
        "\n",
        "#   # check obstacles\n",
        "#   if any([i in obstacle_locations for i in trajectory]): return True\n",
        "\n",
        "#   return False\n",
        "\n",
        "# def check_specifications(trajectory, goal_locations, obstacle_locations, goal_trajectory=None, end_at_goal=True):\n",
        "\n",
        "#   # check end at goal specification\n",
        "#   if end_at_goal:\n",
        "#         if trajectory[-1] in goal_locations: return 1\n",
        "#         else: return 0\n",
        "\n",
        "#   # check goals visited\n",
        "#   if all([i in trajectory for i in goal_locations]):\n",
        "\n",
        "#     if goal_trajectory != None: # order of visiting goal states\n",
        "#         for i in range(len(goal_locations)-1):\n",
        "#             if trajectory.index(goal_locations[i]) > trajectory.index(goal_locations[i+1]):\n",
        "#                 return 0\n",
        "#         return 1\n",
        "#     else:\n",
        "#         return 1"
      ],
      "metadata": {
        "id": "D8TT-Xm4bzoR"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# a_t = argmax(Q(s_t,a) +  U(s-t,a))\n",
        "\n",
        "# NODE:\n",
        "# N(s,a): visit count \n",
        "# W(s,a): total action-value\n",
        "# Q(s,a): mean action-value\n",
        "# P(s,a): prior probability\n",
        "# U(s,a) = C * P(s,a) * (np.sqrt(np.sum(N(s,:))))/(1+N(s,a))\n",
        "\n",
        "# leaf nodes:\n",
        "# N(s_l, a) = 0\n",
        "# W(s_l, a) = 0\n",
        "# Q(s_l, a) = 0\n",
        "# P(s_l, a) = P_a\n",
        "\n",
        "# backward pass\n",
        "# N(s,a) += 1\n",
        "# W(s,a) += v\n",
        "# Q(s,a) = W(s,a)/N(s,a)\n",
        "\n",
        "# Pi(a|s_0) = N(s_0, a)**(1/tow) / np.sum(N(s_0,:)**(1/tow))\n",
        "\n",
        "mc_calls = 0\n",
        "\n",
        "def MCTS_rec(model, root, tree, LTL_formula, trajectory, predicates, N={}, W={}, Q={}, P={}, C=1, depth=100):\n",
        "\n",
        "  # tree structure: a dict from 'state_hash' -> (grid_world, children)\n",
        "\n",
        "  global mc_calls # for time analysis purposes only\n",
        "  mc_calls += 1 # counting number of calls to this function\n",
        "\n",
        " \n",
        "  grid_world = tree[root][0] # get the raw grid_world state\n",
        "\n",
        "  location_i, location_j, location = grid_world.get_locations(target=1, absolute_loc=False) # get current location of the agent\n",
        "  trajectory.append(location)\n",
        "\n",
        "  # check if LTL specs are violated\n",
        "  ####\n",
        "\n",
        "  if depth < 0: # search depth limit reached\n",
        "      outcome = check_LTL(LTL_formula, trajectory, predicates)\n",
        "      if all(outcome): return 1\n",
        "      else: return -1\n",
        "\n",
        "  elif root not in N: # unexplored leaf node\n",
        "      model_output = model(grid_world.map.reshape(1, -1))\n",
        "      value = model_output[1].numpy()[0][0]\n",
        "      P[root] = model_output[0].numpy()[0]\n",
        "      N[root] = np.zeros(5)\n",
        "      W[root] = np.zeros(5)\n",
        "      Q[root] = np.zeros(5)\n",
        "      return value\n",
        "\n",
        "  ### selecting the next node to expand ###\n",
        "  U = C * P[root] * (np.sqrt(np.sum(N[root])))/(1+N[root])\n",
        "  next_move = (U + Q[root]).argmax()\n",
        "  # print(U, Q[root])\n",
        "  # next_move = np.random.randint(4)\n",
        "  #########################################\n",
        "  \n",
        "  ### creating the next subtree ###\n",
        "  next_grid_world, new_location_i, new_location_j = make_move(grid_world, location_i, location_j, next_move)\n",
        "  sub_root = hash_state(next_grid_world.map)\n",
        "  sub_tree = {sub_root :(next_grid_world, {})}\n",
        "  #################################\n",
        "  ### expanding the next move and back tracking ###\n",
        "  value = MCTS_rec(model, sub_root, sub_tree, LTL_formula, trajectory, predicates,  N, W, Q, P, C=C, depth=depth-1)\n",
        "  N[root][next_move] += 1\n",
        "  W[root][next_move] += value\n",
        "  # if root == '1-10-120-10000-10-10000-100-1000' and next_move==3:\n",
        "    # print(\"value:\", value)\n",
        "    # print('U:',U,'Q:', Q[root])\n",
        "  Q[root][next_move] = W[root][next_move]/N[root][next_move]\n",
        "  #################################################\n",
        "  return value\n",
        "\n",
        "\n",
        "\n",
        "def MCTS(model, root, tree, LTL_formula, predicates, N, W, Q, P, n_samples=100, tow=1, C=1, depth=100):\n",
        "\n",
        "  grid_world = tree[root][0] # get the grid_world\n",
        "\n",
        "  for sample in range(n_samples):\n",
        "    MCTS_rec(model, root, tree.copy(), LTL_formula, [], predicates,  N, W, Q, P, C=C, depth=depth)\n",
        "\n",
        "  Pi = (N[root]**(1/tow)) / np.sum(N[root]**(1/tow))\n",
        "  return Pi\n",
        "\n",
        "grid_world = Grid_world(map_num=2)\n",
        "goal_trajectory = grid_world.get_goal_trajectory()\n",
        "goal_locations = grid_world.get_locations(goal_trajectory, as_list=True)\n",
        "obstacle_locations = grid_world.get_locations(-1, as_list=True)\n",
        "LTL_formula = ('&', ('^', (None, 'g')), ('#', ('~', (None, 'o'))) )\n",
        "predicates = {\n",
        "    'g':goal_locations,\n",
        "    'o':obstacle_locations\n",
        "}\n",
        "model = build_model(grid_world.shape)\n",
        "\n",
        "t1 = time.time()\n",
        "N, W, Q, P = {}, {}, {}, {}\n",
        "# P[hash_state(grid_world)] = model(grid_world.reshape(1, -1))[0].numpy()[0]\n",
        "Pi = MCTS(model, hash_state(grid_world.map), {hash_state(grid_world.map):(grid_world, {})}, LTL_formula, predicates, N, W, Q, P, n_samples=500, tow=0.1, C=5)\n",
        "t2 = time.time()\n",
        "print('MCTS runtime',t2 - t1, 'sec', \"(\",(t2-t1)/mc_calls,\"/\",mc_calls,\")\" )\n",
        "print(\"Policy:\", Pi)\n",
        "print(grid_world.map)\n",
        "\n",
        "# for i in N:\n",
        "#   print(i, N[i], Q[i])"
      ],
      "metadata": {
        "id": "7F9gjXjezN11",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "61ab5c21-a556-4caf-f7c7-4957bf75b473"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "MCTS runtime 2.414477586746216 sec ( 7.25373306118553e-05 / 33286 )\n",
            "Policy: [9.95839634e-01 4.47344556e-06 4.47344556e-06 4.14694596e-03\n",
            " 4.47344556e-06]\n",
            "[[ 1 -1  0 -1  2]\n",
            " [ 0 -1  0  0  0]\n",
            " [ 0 -1  0 -1  0]\n",
            " [ 0  0  0 -1  0]\n",
            " [ 0 -1  0  0  0]]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "grid_world.map"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "iOa8KWe5td-L",
        "outputId": "b6abff29-2e4e-4d6b-a468-599bb831a9b1"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[ 1, -1,  0, -1,  2],\n",
              "       [ 0, -1,  0,  0,  0],\n",
              "       [ 0, -1,  0, -1,  0],\n",
              "       [ 0,  0,  0, -1,  0],\n",
              "       [ 0, -1,  0,  0,  0]])"
            ]
          },
          "metadata": {},
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "for i in N:\n",
        "  print(i, N[i])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "owE1BnRZbUMN",
        "outputId": "1748a636-e29b-4297-e894-494d7a6da1d5"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "1-10-120-10000-10-10000-100-1000 [346. 101. 101. 200. 101.]\n",
            "010-120-10000-10-10000-100-1000 [ 80. 100.  51. 310. 100.]\n",
            "0-10-121-10000-10-10000-100-1000 [143.  50. 100.  52. 100.]\n",
            "0-10-12010000-10-10000-100-1000 [55. 42. 51. 91. 99.]\n",
            "001-120-10000-10-10000-100-1000 [24. 99. 50. 52. 99.]\n",
            "000-12010000-10-10000-100-1000 [154. 104.  51. 197.  99.]\n",
            "100-120-10000-10-10000-100-1000 [50.  0.  0.  0.  0.]\n",
            "000120-10000-10-10000-100-1000 [16. 98. 50. 51. 98.]\n",
            "000-12001000-10-10000-100-1000 [ 53. 142.  49.  51.  98.]\n",
            "010-12000000-10-10000-100-1000 [ 88.  98.  50. 100.  98.]\n",
            "000-1200000010-10000-100-1000 [203.  82.  51.  51.  98.]\n",
            "000-120-11000-10-10000-100-1000 [ 1. 49.  0.  0.  0.]\n",
            "000-12100000-10-10000-100-1000 [49.  0.  0.  0.  0.]\n",
            "000-12000100-10-10000-100-1000 [ 1.  1. 49.  1.  0.]\n",
            "001-12000000-10-10000-100-1000 [128.  97.  50.  49.  97.]\n",
            "000-1200000001-10000-100-1000 [147.  51.  49.  50.  97.]\n",
            "00012000000-10-10000-100-1000 [121.  96.  49.  50.  96.]\n",
            "000-1201000000-10000-100-1000 [73. 52. 50. 49. 97.]\n",
            "000-12000000-11-10000-100-1000 [ 1. 49.  0.  0.  0.]\n",
            "000-1200000100-10000-100-1000 [49.  1.  0.  0.  0.]\n",
            "000-1200000000-10010-100-1000 [ 1. 49.  0.  0.  0.]\n",
            "000-120000000010000-100-1000 [ 59.  56.  49. 122.  96.]\n",
            "000-1200100000-10000-100-1000 [51. 67. 48. 48. 96.]\n",
            "000-1200000000-10001-100-1000 [ 1. 48.  0.  0.  0.]\n",
            "000-1200010000-10000-100-1000 [ 1.  1. 48.  0.  0.]\n",
            "010-1200000000-10000-100-1000 [49. 96. 48. 48. 94.]\n",
            "000-120000000001000-100-1000 [95. 52. 48. 50. 95.]\n",
            "000-1210000000-10000-100-1000 [48.  1.  0.  0.  0.]\n",
            "000-1200000000-10000100-1000 [0. 0. 0. 0. 0.]\n",
            "001-1200000000-10000-100-1000 [15. 95. 50. 48. 95.]\n",
            "000-120001000000000-100-1000 [ 2.  3. 49. 48. 93.]\n",
            "000-120000000100000-100-1000 [48.  0.  0.  0.  0.]\n",
            "000-120000000000000100-1000 [105.  79. 161.  57.  95.]\n",
            "000-120000100000000-100-1000 [94.  2.  1. 47. 94.]\n",
            "000-120000000000000010-1000 [94. 49. 94. 49. 94.]\n",
            "000120000000000000-100-1000 [ 1. 94.  0.  0.  0.]\n",
            "000-120010000000000-100-1000 [47.  0.  0.  0.  0.]\n",
            "000-120000000000001000-1000 [ 94. 104.  48.  49.  94.]\n",
            "000-120000000000000000-1010 [51. 47. 49. 94. 92.]\n",
            "000-120000000010000000-1000 [142.  72.  94.  47.  94.]\n",
            "000-120000000000000000-1001 [93. 47. 49. 93. 93.]\n",
            "000-120000000001000000-1000 [93. 92. 94. 47. 91.]\n",
            "000-120000000100000000-1000 [144.  49.  48.  47.  93.]\n",
            "000-120001000000000000-1000 [46. 21. 50. 46. 89.]\n",
            "000-120000000000000000-1100 [47. 47.  1. 93. 93.]\n",
            "000-120000100000000000-1000 [90. 42. 46. 45. 90.]\n",
            "000-120000000000010000-1000 [47.  0.  0.  0.  0.]\n",
            "000-120000001000000000-1000 [46.  0.  0.  0.  0.]\n",
            "000-1200000000000000001000 [0. 0. 0. 0. 0.]\n",
            "000-120010000000000000-1000 [47.  1.  1. 46. 92.]\n",
            "000120000000000000000-1000 [13. 92. 47. 48. 92.]\n",
            "000-120100000000000000-1000 [0. 0. 0. 0. 0.]\n",
            "000-110000000000000000-1000 [89. 89. 33. 49. 89.]\n",
            "000-100000100000000000-1000 [88. 44.  1.  1.  0.]\n",
            "000100000000000000000-1000 [96. 88. 97. 96. 88.]\n",
            "000010000000000000000-1000 [87. 87. 88. 60. 87.]\n",
            "000-100000000001000000-1000 [0. 0. 0. 0. 0.]\n",
            "000-100001000000000000-1000 [0. 0. 0. 0. 0.]\n",
            "001000000000000000000-1000 [88. 87. 45. 47. 87.]\n",
            "000000001000000000000-1000 [46. 89. 48. 55. 87.]\n",
            "000000000100000000000-1000 [86. 43. 56. 46. 86.]\n",
            "010000000000000000000-1000 [43.  0.  0.  0.  0.]\n",
            "000000010000000000000-1000 [45. 43. 44. 45. 86.]\n",
            "000000000000010000000-1000 [47. 43. 43. 48. 88.]\n",
            "000000000000100000000-1000 [44. 45. 66. 27. 87.]\n",
            "000000100000000000000-1000 [43.  0.  1. 44. 87.]\n",
            "000000000000001000000-1000 [85. 45. 44.  1. 87.]\n",
            "0001200000000-10000-100-1000 [ 8. 94. 48. 49. 94.]\n",
            "001-120000000000000000-1000 [0. 0. 0. 0. 0.]\n",
            "001020000000000000000-1000 [46.  0.  0.  0.  0.]\n",
            "000020001000000000000-1000 [ 1. 46.  0.  0.  0.]\n",
            "000020000100000000000-1000 [0. 0. 0. 0. 0.]\n",
            "000000000000000000100-1000 [ 1. 44. 44.  1. 87.]\n",
            "000-120000000000000-110-1000 [94. 47.  0.  0.  0.]\n",
            "000000000000000000010-1000 [86.  0.  0.  0.  0.]\n",
            "000000000000000001000-1000 [44. 22. 22.  2. 86.]\n",
            "000-110000000000000-100-1000 [93.  0.  0.  0.  0.]\n",
            "000-12000000-1010000-100-1000 [1. 0. 0. 0. 0.]\n",
            "000-12000010-10-10000-100-1000 [0. 0. 0. 0. 0.]\n",
            "00001000000-10-10000-100-1000 [ 95.  95. 154. 104.  95.]\n",
            "000-12000000-1001000-100-1000 [0. 0. 0. 0. 0.]\n",
            "100-12000000-10-10000-100-1000 [49.  0.  0.  0.  0.]\n",
            "00002000100-10-10000-100-1000 [ 1. 48.  0.  0.  0.]\n",
            "00102000000-10-10000-100-1000 [48.  0.  0.  0.  0.]\n",
            "00002000010-10-10000-100-1000 [0. 0. 0. 0. 0.]\n",
            "00000000010-10-10000-100-1000 [94. 94. 48. 49. 94.]\n",
            "00010000000-10-10000-100-1000 [ 47.  94.  48. 147.  94.]\n",
            "00000000100-10-10000-100-1000 [47. 47. 48. 97. 93.]\n",
            "00000000000-10-11000-100-1000 [93. 47.  0.  0.  0.]\n",
            "000-1200001000-10000-100-1000 [0. 0. 0. 0. 0.]\n",
            "0000100000000-10000-100-1000 [93. 93. 48.  2. 93.]\n",
            "100-1200000000-10000-100-1000 [48.  0.  0.  0.  0.]\n",
            "0010200000000-10000-100-1000 [47.  0.  0.  0.  0.]\n",
            "0000200010000-10000-100-1000 [ 1. 47.  0.  0.  0.]\n",
            "0000200001000-10000-100-1000 [0. 0. 0. 0. 0.]\n",
            "00100000000-10-10000-100-1000 [47.  0.  0.  0.  0.]\n",
            "00000000000-1010000-100-1000 [50. 55. 47. 78. 92.]\n",
            "00000001000-10-10000-100-1000 [46.  0.  0.  0.  0.]\n",
            "00000000000-1001000-100-1000 [91.  1. 46.  1.  0.]\n",
            "00000000100-1000000-100-1000 [ 2.  3. 47. 46. 91.]\n",
            "00000000000-1100000-100-1000 [46.  0.  0.  0.  0.]\n",
            "00000000000-1000000100-1000 [54. 74. 80. 48. 91.]\n",
            "00000000010-1000000-100-1000 [90.  1.  0.  0.  0.]\n",
            "00000000000-1000000010-1000 [90. 48. 45. 46. 90.]\n",
            "00010000000-1000000-100-1000 [ 1. 90.  0.  0.  0.]\n",
            "00000000000-1000001000-1000 [45. 71. 46. 47. 90.]\n",
            "00000000000-1010000000-1000 [48. 64. 46. 45. 90.]\n",
            "00000000000-1000000000-1010 [ 1. 45. 44.  0.  0.]\n",
            "00000000000-1000000-110-1000 [0. 0. 0. 0. 0.]\n",
            "00000000000-1001000000-1000 [89. 46. 45. 46. 89.]\n",
            "00000000000-1000000000-1001 [87. 45.  0.  0.  0.]\n",
            "00000000100-1000000000-1000 [46. 57. 46. 45. 89.]\n",
            "00000000000-1100000000-1000 [45. 45. 24. 45. 89.]\n",
            "00000000010-1000000000-1000 [88. 43. 44. 44.  0.]\n",
            "00010000000-1000000000-1000 [49. 88. 46. 44. 88.]\n",
            "00000001000-1000000000-1000 [44.  0.  0. 44.  0.]\n",
            "00001000000-1000000000-1000 [87. 87. 44. 44. 87.]\n",
            "000-120-10100-10-10000-100-1000 [0. 0. 0. 0. 0.]\n",
            "000010-10000-10-10000-100-1000 [ 97.  97.  50. 106.  97.]\n",
            "001020-10000-10-10000-100-1000 [49.  0.  0.  0.  0.]\n",
            "000020-10100-10-10000-100-1000 [ 1. 49.  0.  0.  0.]\n",
            "000020-10010-10-10000-100-1000 [0. 0. 0. 0. 0.]\n",
            "00100000000-1000000000-1000 [44. 87.  0.  0.  0.]\n",
            "000000-10010-10-10000-100-1000 [96. 96. 49. 50. 96.]\n",
            "000100-10000-10-10000-100-1000 [48.  0.  0.  0.  0.]\n",
            "0-10-120-10001-10-10000-100-1000 [ 1. 50.  0.  0.  0.]\n",
            "0-10-12001000-10-10000-100-1000 [ 1.  2. 49.  1.  0.]\n",
            "0-10-120-1000010-10000-100-1000 [0. 0. 0. 0. 0.]\n",
            "0-10-1200000010-10000-100-1000 [126.  54.  50.  51.  98.]\n",
            "0-10-12100000-10-10000-100-1000 [49.  0.  0.  0.  0.]\n",
            "0-10-12000100-10-10000-100-1000 [0. 0. 0. 0. 0.]\n",
            "0-10-1200000001-10000-100-1000 [70. 51. 49. 50. 97.]\n",
            "0-11-12000000-10-10000-100-1000 [1. 0. 0. 0. 0.]\n",
            "0-10-1201000000-10000-100-1000 [ 1.  1.  1. 49. 97.]\n",
            "0-10-12000000-11-10000-100-1000 [0. 0. 0. 0. 0.]\n",
            "0-1012000000-10-10000-100-1000 [0. 0. 0. 0. 0.]\n",
            "0-10-1200000100-10000-100-1000 [49.  0.  0.  0.  0.]\n",
            "0-10-1200000000-10010-100-1000 [ 1. 49.  0.  0.  0.]\n",
            "00000000000-1000000000-1100 [45. 45.  0.  0.  0.]\n",
            "0-10-120000000010000-100-1000 [51. 51. 49. 58. 96.]\n",
            "00000000000-1000010000-1000 [45.  0.  0.  0.  0.]\n",
            "0-10-1200100000-10000-100-1000 [ 1.  1.  0. 48.  0.]\n",
            "0-10-1200000000-10001-100-1000 [ 1. 48.  0.  0.  0.]\n",
            "000000000001000000000-1000 [74. 46. 45. 28. 86.]\n",
            "0-10-1200010000-10000-100-1000 [0. 0. 0. 0. 0.]\n",
            "0-10-1210000000-10000-100-1000 [0. 0. 0. 0. 0.]\n",
            "0-10-1200000000-10000100-1000 [0. 0. 0. 0. 0.]\n",
            "0000000001000-10000-100-1000 [92.  0.  0.  0.  0.]\n",
            "0-10-120000000001000-100-1000 [95.  1. 48.  0.  0.]\n",
            "0-11-1200000000-10000-100-1000 [0. 0. 0. 0. 0.]\n",
            "0-10-120000000100000-100-1000 [48.  0.  0.  0.  0.]\n",
            "0-10-120001000000000-100-1000 [ 1.  1.  0. 48.  0.]\n",
            "0-10-120000000000000100-1000 [104.  52.  79.  56.  95.]\n",
            "0-10-120000100000000-100-1000 [94.  0.  0.  0.  0.]\n",
            "0-10-120000000000000010-1000 [94. 49. 94. 48. 94.]\n",
            "0-10120000000000000-100-1000 [0. 0. 0. 0. 0.]\n",
            "0-10-120000000000001000-1000 [47. 48. 69. 50. 94.]\n",
            "0-10-120000000000000000-1010 [ 4. 47. 48. 92. 94.]\n",
            "0-10-120000000010000000-1000 [ 2.  1.  1. 47. 92.]\n",
            "000000000010000000000-1000 [44.  0.  0.  0.  0.]\n",
            "0-10-120000000000000000-1001 [93. 47.  1. 93. 93.]\n",
            "0-10-120000000001000000-1000 [93.  1.  1. 47.  0.]\n",
            "0-10-120000000000000000-1100 [47. 47.  1. 93. 93.]\n",
            "0-10-120000000100000000-1000 [ 1.  0. 46. 47.  0.]\n",
            "000001000000000000000-1000 [0. 0. 0. 0. 0.]\n",
            "000000000000000010000-1000 [22. 22. 45.  2. 87.]\n",
            "000000000000000000000-1010 [1. 0. 0. 0. 0.]\n",
            "0-10-120001000000000000-1000 [0. 0. 0. 0. 0.]\n",
            "0-10-120000100000000000-1000 [0. 0. 0. 0. 0.]\n",
            "000000000000000000000-1100 [1. 0. 0. 0. 0.]\n",
            "0-10-120000000000010000-1000 [47. 51. 48. 13. 93.]\n",
            "0-10-1200000000000000001000 [49. 50. 47. 92. 92.]\n",
            "0001000000000-10000-100-1000 [46.  0.  0.  0.  0.]\n",
            "000000-10100-10-10000-100-1000 [48.  0.  0.  0.  0.]\n",
            "000000-10000-10-11000-100-1000 [95. 48.  0.  0.  0.]\n",
            "000010000000000000-100-1000 [0. 0. 0. 0. 0.]\n",
            "000000000000000000000-1001 [0. 0. 0. 0. 0.]\n",
            "0-10-120000000000100000-1000 [46.  0.  0.  0.  0.]\n",
            "000000000000000100000-1000 [43.  0.  0.  0.  0.]\n",
            "0-10-1200000000000000000100 [ 1.  0. 46.  0. 91.]\n",
            "0-10-120000001000000000-1000 [46.  0.  1. 46. 92.]\n",
            "0-10-1200000000000000010000 [46.  0.  0.  0.  0.]\n",
            "0-10-120000010000000000-1000 [0. 0. 0. 0. 0.]\n",
            "0-10-1200000000000100000000 [ 1.  0.  1. 46. 91.]\n",
            "00000001000-1000000-100-1000 [45.  0.  0.  0.  0.]\n",
            "00001000000-1000000-100-1000 [89.  0.  0.  0.  0.]\n",
            "0-10-1200000000000000000010 [0. 0. 0. 0. 0.]\n",
            "0-10-1200000000000010000000 [0. 0. 0. 0. 0.]\n",
            "0-10-1200000000001000000000 [0. 0. 0. 0. 0.]\n",
            "0000000000000000000001000 [1. 0. 0. 0. 0.]\n",
            "0000000000000000000000100 [0. 0. 0. 0. 0.]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def run_episode(grid_world, model, LTL_formula, predicates, n_steps=150, C=3, tow=1, n_samples=300, N={}, W={}, Q={}, P={}, verbose=0):\n",
        "    state_history = []\n",
        "    action_history = []\n",
        "    better_policy = []\n",
        "    trajectory = []\n",
        "    reward = 0\n",
        "\n",
        "    for step in range(n_steps):\n",
        "        \n",
        "        location_i, location_j, location = grid_world.get_locations(1, False)\n",
        "        trajectory.append(location)\n",
        "\n",
        "        # check if LTL specs are violated\n",
        "        ###\n",
        "\n",
        "        state_history.append(grid_world.map.copy())\n",
        "            \n",
        "        # MCTS - policy improvment\n",
        "        Pi = MCTS(model, hash_state(grid_world.map), {hash_state(grid_world.map):(grid_world, {})}, LTL_formula, predicates, N, W, Q, P, n_samples=n_samples, tow=tow, C=C, depth=n_steps+1)\n",
        "        better_policy.append(Pi.copy())\n",
        "          \n",
        "        # move based on enhanced policy\n",
        "        # action = Pi.argmax() # greedy\n",
        "        action = np.random.choice(5, p=Pi)\n",
        "        action_history.append(action)\n",
        "        \n",
        "        # making the move in the grid world\n",
        "        grid_world, _, _ = make_move(grid_world, location_i, location_j, action)\n",
        "\n",
        "        if verbose==1:\n",
        "          print(action, end=\", \")\n",
        "        elif verbose==2:\n",
        "          clear_output(wait=True)\n",
        "          print(grid_world.map)\n",
        "\n",
        "    outcome = check_LTL(LTL_formula, trajectory, predicates)\n",
        "    if all(outcome): reward = 1\n",
        "\n",
        "    return state_history, action_history, reward, better_policy\n",
        "\n",
        "grid_world = Grid_world(map_num=10)\n",
        "goal_trajectory = grid_world.get_goal_trajectory()\n",
        "goal_locations = grid_world.get_locations(goal_trajectory, as_list=True)\n",
        "obstacle_locations = grid_world.get_locations(-1, as_list=True)\n",
        "LTL_formula = ('&', ('^', (None, 'g')), ('#', ('~', (None, 'o'))) )\n",
        "predicates = {\n",
        "    'g':goal_locations,\n",
        "    'o':obstacle_locations\n",
        "}\n",
        "print(grid_world.map)\n",
        "model = build_model(grid_world.shape)\n",
        "state_history, action_history, reward, better_policy = run_episode(grid_world, model, LTL_formula, predicates, n_steps=8)"
      ],
      "metadata": {
        "id": "RjGcTM0ApSbQ",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "217564eb-fc10-4ed1-9086-16e4eb4a7d66"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[[ 1  0  0]\n",
            " [-1 -1  0]\n",
            " [ 2  0  0]]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "action_history, better_policy, reward"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SE5w54AV7SWm",
        "outputId": "e7a49eb0-98bc-4ab7-fedb-47f07162f5cd"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "([3, 3, 0, 2, 0, 2, 0, 2],\n",
              " [array([0.23483366, 0.09784736, 0.15655577, 0.37377691, 0.1369863 ]),\n",
              "  array([0.19035533, 0.18274112, 0.20812183, 0.28426396, 0.13451777]),\n",
              "  array([0.78039216, 0.0754902 , 0.05098039, 0.04215686, 0.05098039]),\n",
              "  array([0.01364213, 0.00761421, 0.95717005, 0.00951777, 0.01205584]),\n",
              "  array([0.91093439, 0.0306163 , 0.02067594, 0.01709742, 0.02067594]),\n",
              "  array([0.00918183, 0.00541072, 0.96933924, 0.00819807, 0.00787014]),\n",
              "  array([0.94347003, 0.01943218, 0.01312303, 0.01085174, 0.01312303]),\n",
              "  array([0.00783924, 0.00452689, 0.97460528, 0.00662471, 0.00640389])],\n",
              " 0)"
            ]
          },
          "metadata": {},
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# The agent manages to solve multi-goal problems, but as the number of the goals increase the solution becomes less and less optimal.\n",
        " \n",
        "\n",
        "# TO INCREASE IMPROVMENT:\n",
        "# * introduce ranked rewards?\n",
        "\n",
        "# LTL: complex spesifications\n",
        "# * introduce LTL parser?\n",
        "\n",
        "# more general reward mechanism\n",
        "# tested with different scenarios\n",
        "# refactored some parts of the code (map maker)"
      ],
      "metadata": {
        "id": "1k_P10kX5Yvw"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def decayed_reward(size, init_rew, l=0.95):\n",
        "  \n",
        "  rew_seq = []\n",
        "\n",
        "  for i in range(size):\n",
        "    rew_seq.append(init_rew)\n",
        "    init_rew *= l\n",
        "  \n",
        "  return np.array(rew_seq)"
      ],
      "metadata": {
        "id": "URzY8wg9gUKa"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def run_simulation(grid_world_shape, n_goals, ordered_goals=False, n_episodes=10, C=3, tow=1, n_steps=100, n_samples=300, N={}, W={}, Q={}, P={}, verbose=True):\n",
        "\n",
        "    grid_world_0 = Grid_world(shape=grid_world_shape, n_goals=n_goals, ordered=ordered_goals)\n",
        "    if verbose: print(grid_world_0.map)\n",
        "    model = build_model(grid_world_0.shape)\n",
        "    goal_trajectory = grid_world_0.get_goal_trajectory()\n",
        "    goal_locations = grid_world_0.get_locations(goal_trajectory, as_list=True)\n",
        "    obstacle_locations = grid_world_0.get_locations(-1, as_list=True)\n",
        "    LTL_formula = ('&', ('^', (None, 'g')), ('#', ('~', (None, 'o'))) )\n",
        "    predicates = {\n",
        "        'g':goal_locations,\n",
        "        'o':obstacle_locations\n",
        "    }\n",
        "\n",
        "    for e in range(n_episodes):\n",
        "        \n",
        "        grid_world = grid_world_0.copy()\n",
        "        \n",
        "        state_history, action_history, reward, better_policy = run_episode(grid_world, model, LTL_formula, predicates, n_steps=n_steps, n_samples=n_samples, tow=tow, C=C, N=N, W=W, Q=Q, P=P, verbose=2)\n",
        "\n",
        "        X = np.array(state_history).reshape(-1, grid_world.shape[0]*grid_world.shape[1])\n",
        "        y1 = np.array(better_policy)\n",
        "        # y2 = np.repeat(reward, len(better_policy))\n",
        "        y2 = decayed_reward(size=len(better_policy), init_rew=reward, l=1)[::-1]\n",
        "        model.fit(X, [y1, y2], epochs=50, verbose=0)\n",
        "\n",
        "        # model.fit(np.array(state_history), np.array(better_policy), epochs=30, verbose=0) # without value output\n",
        "\n",
        "        if verbose:\n",
        "          # print(N[hash_state(grid_world)])\n",
        "          # print(state_history)\n",
        "          root = hash_state(state_history[-1])\n",
        "          # print(\"N:\",N[root])\n",
        "          # print(\"Q:\",Q[root])\n",
        "          # print(\"U:\", C * P[root] * (np.sqrt(np.sum(N[root])))/(1+N[root]))\n",
        "          print(\"episode reward:\",reward, \",model value estimate of last step\", model(state_history[-1].reshape(1, -1))[1].numpy()[0][0])\n",
        "          print(\"Action history:\",\"(\",len(action_history),\")\", action_history)\n",
        "        if reward == 1: return True\n",
        "    return False\n",
        "\n",
        "\n",
        "N, W, Q, P = {}, {}, {}, {}\n",
        "run_simulation((5,5), n_goals= 1, ordered_goals=False, n_episodes=1, C=1, tow=0.1, n_steps=10, n_samples=500, N=N, W=W, Q=Q, P=P)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "X7xYmG0vpSdL",
        "outputId": "2c5d9cc5-3d0d-48c7-edfb-8331f8140a21"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[[ 0.  0.  0.  0.  0.]\n",
            " [ 0.  0.  0. -1.  0.]\n",
            " [ 0.  0.  0.  1.  0.]\n",
            " [ 0. -1.  0.  0.  0.]\n",
            " [ 0.  0.  0.  0.  0.]]\n",
            "episode reward: 0 ,model value estimate of last step 0.00010449486\n",
            "Action history: ( 10 ) [3, 3, 0, 0, 3, 0, 0, 1, 2, 4]\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "False"
            ]
          },
          "metadata": {},
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Test cases for end at goal position"
      ],
      "metadata": {
        "id": "1mbX1VgUqvkQ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "shapes = [(i,i) for i in range(5,25,2)]\n",
        "results_1 = []\n",
        "\n",
        "for gw_shape in shapes:\n",
        "  print(gw_shape, end=\": \")\n",
        "  for _ in range(5):\n",
        "    N, W, Q, P = {}, {}, {}, {}\n",
        "    outcome = run_simulation(gw_shape, 2, n_episodes=3, C=1, tow=0.1, n_steps=gw_shape[0]*2+5, n_samples=500, N=N, W=W, Q=Q, P=P, verbose=False)\n",
        "    results_1.append(outcome)\n",
        "    print(outcome,end=\", \")\n",
        "  print()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 468
        },
        "id": "TpX4MpwSq1Ye",
        "outputId": "e8103401-80aa-46e4-e874-7753197b7a25"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[[ 0.  0.  0.  0.  0.]\n",
            " [ 0. -1.  0. -1.  0.]\n",
            " [ 0.  0.  0. -1.  0.]\n",
            " [ 1.  2.  0.  0.  0.]\n",
            " [ 0. -1.  0.  0.  0.]]\n"
          ]
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-34-f12e05e4dc7e>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      6\u001b[0m   \u001b[0;32mfor\u001b[0m \u001b[0m_\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m     \u001b[0mN\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mW\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mQ\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mP\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 8\u001b[0;31m     \u001b[0moutcome\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mrun_simulation\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgw_shape\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mn_episodes\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m3\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mC\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtow\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0.1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mn_steps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mgw_shape\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mn_samples\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m500\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mN\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mN\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mW\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mW\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mQ\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mQ\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mP\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mP\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      9\u001b[0m     \u001b[0mresults_1\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutcome\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     10\u001b[0m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutcome\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mend\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\", \"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-32-d4e24ad2b0f0>\u001b[0m in \u001b[0;36mrun_simulation\u001b[0;34m(grid_world_shape, n_goals, ordered_goals, n_episodes, C, tow, n_steps, n_samples, N, W, Q, P, verbose)\u001b[0m\n\u001b[1;32m     17\u001b[0m         \u001b[0mgrid_world\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgrid_world_0\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcopy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     18\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 19\u001b[0;31m         \u001b[0mstate_history\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maction_history\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreward\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbetter_policy\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mrun_episode\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgrid_world\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mLTL_formula\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpredicates\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mn_steps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mn_steps\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mn_samples\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mn_samples\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtow\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtow\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mC\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mC\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mN\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mN\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mW\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mW\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mQ\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mQ\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mP\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mP\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     20\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     21\u001b[0m         \u001b[0mX\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0marray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstate_history\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreshape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgrid_world\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mgrid_world\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-24-522e7a8a666a>\u001b[0m in \u001b[0;36mrun_episode\u001b[0;34m(grid_world, model, LTL_formula, predicates, n_steps, C, tow, n_samples, N, W, Q, P, verbose)\u001b[0m\n\u001b[1;32m     17\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     18\u001b[0m         \u001b[0;31m# MCTS - policy improvment\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 19\u001b[0;31m         \u001b[0mPi\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mMCTS\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhash_state\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgrid_world\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmap\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0mhash_state\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgrid_world\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmap\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgrid_world\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mLTL_formula\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpredicates\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mN\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mW\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mQ\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mP\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mn_samples\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mn_samples\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtow\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtow\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mC\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mC\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdepth\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mn_steps\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     20\u001b[0m         \u001b[0mbetter_policy\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mPi\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcopy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     21\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-20-c7b65a52744c>\u001b[0m in \u001b[0;36mMCTS\u001b[0;34m(model, root, tree, LTL_formula, predicates, N, W, Q, P, n_samples, tow, C, depth)\u001b[0m\n\u001b[1;32m     84\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     85\u001b[0m   \u001b[0;32mfor\u001b[0m \u001b[0msample\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mn_samples\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 86\u001b[0;31m     \u001b[0mMCTS_rec\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mroot\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtree\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcopy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mLTL_formula\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpredicates\u001b[0m\u001b[0;34m,\u001b[0m  \u001b[0mN\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mW\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mQ\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mP\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mC\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mC\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdepth\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdepth\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     87\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     88\u001b[0m   \u001b[0mPi\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mN\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mroot\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m/\u001b[0m\u001b[0mtow\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msum\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mN\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mroot\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m/\u001b[0m\u001b[0mtow\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-20-c7b65a52744c>\u001b[0m in \u001b[0;36mMCTS_rec\u001b[0;34m(model, root, tree, LTL_formula, trajectory, predicates, N, W, Q, P, C, depth)\u001b[0m\n\u001b[1;32m     66\u001b[0m   \u001b[0;31m#################################\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     67\u001b[0m   \u001b[0;31m### expanding the next move and back tracking ###\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 68\u001b[0;31m   \u001b[0mvalue\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mMCTS_rec\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msub_root\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msub_tree\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mLTL_formula\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrajectory\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpredicates\u001b[0m\u001b[0;34m,\u001b[0m  \u001b[0mN\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mW\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mQ\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mP\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mC\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mC\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdepth\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdepth\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     69\u001b[0m   \u001b[0mN\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mroot\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mnext_move\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     70\u001b[0m   \u001b[0mW\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mroot\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mnext_move\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-20-c7b65a52744c>\u001b[0m in \u001b[0;36mMCTS_rec\u001b[0;34m(model, root, tree, LTL_formula, trajectory, predicates, N, W, Q, P, C, depth)\u001b[0m\n\u001b[1;32m     66\u001b[0m   \u001b[0;31m#################################\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     67\u001b[0m   \u001b[0;31m### expanding the next move and back tracking ###\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 68\u001b[0;31m   \u001b[0mvalue\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mMCTS_rec\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msub_root\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msub_tree\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mLTL_formula\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrajectory\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpredicates\u001b[0m\u001b[0;34m,\u001b[0m  \u001b[0mN\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mW\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mQ\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mP\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mC\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mC\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdepth\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdepth\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     69\u001b[0m   \u001b[0mN\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mroot\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mnext_move\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     70\u001b[0m   \u001b[0mW\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mroot\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mnext_move\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-20-c7b65a52744c>\u001b[0m in \u001b[0;36mMCTS_rec\u001b[0;34m(model, root, tree, LTL_formula, trajectory, predicates, N, W, Q, P, C, depth)\u001b[0m\n\u001b[1;32m     66\u001b[0m   \u001b[0;31m#################################\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     67\u001b[0m   \u001b[0;31m### expanding the next move and back tracking ###\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 68\u001b[0;31m   \u001b[0mvalue\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mMCTS_rec\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msub_root\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msub_tree\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mLTL_formula\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrajectory\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpredicates\u001b[0m\u001b[0;34m,\u001b[0m  \u001b[0mN\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mW\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mQ\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mP\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mC\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mC\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdepth\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdepth\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     69\u001b[0m   \u001b[0mN\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mroot\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mnext_move\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     70\u001b[0m   \u001b[0mW\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mroot\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mnext_move\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-20-c7b65a52744c>\u001b[0m in \u001b[0;36mMCTS_rec\u001b[0;34m(model, root, tree, LTL_formula, trajectory, predicates, N, W, Q, P, C, depth)\u001b[0m\n\u001b[1;32m     66\u001b[0m   \u001b[0;31m#################################\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     67\u001b[0m   \u001b[0;31m### expanding the next move and back tracking ###\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 68\u001b[0;31m   \u001b[0mvalue\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mMCTS_rec\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msub_root\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msub_tree\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mLTL_formula\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrajectory\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpredicates\u001b[0m\u001b[0;34m,\u001b[0m  \u001b[0mN\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mW\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mQ\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mP\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mC\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mC\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdepth\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdepth\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     69\u001b[0m   \u001b[0mN\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mroot\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mnext_move\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     70\u001b[0m   \u001b[0mW\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mroot\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mnext_move\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-20-c7b65a52744c>\u001b[0m in \u001b[0;36mMCTS_rec\u001b[0;34m(model, root, tree, LTL_formula, trajectory, predicates, N, W, Q, P, C, depth)\u001b[0m\n\u001b[1;32m     66\u001b[0m   \u001b[0;31m#################################\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     67\u001b[0m   \u001b[0;31m### expanding the next move and back tracking ###\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 68\u001b[0;31m   \u001b[0mvalue\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mMCTS_rec\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msub_root\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msub_tree\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mLTL_formula\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrajectory\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpredicates\u001b[0m\u001b[0;34m,\u001b[0m  \u001b[0mN\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mW\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mQ\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mP\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mC\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mC\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdepth\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdepth\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     69\u001b[0m   \u001b[0mN\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mroot\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mnext_move\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     70\u001b[0m   \u001b[0mW\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mroot\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mnext_move\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-20-c7b65a52744c>\u001b[0m in \u001b[0;36mMCTS_rec\u001b[0;34m(model, root, tree, LTL_formula, trajectory, predicates, N, W, Q, P, C, depth)\u001b[0m\n\u001b[1;32m     66\u001b[0m   \u001b[0;31m#################################\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     67\u001b[0m   \u001b[0;31m### expanding the next move and back tracking ###\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 68\u001b[0;31m   \u001b[0mvalue\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mMCTS_rec\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msub_root\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msub_tree\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mLTL_formula\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrajectory\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpredicates\u001b[0m\u001b[0;34m,\u001b[0m  \u001b[0mN\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mW\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mQ\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mP\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mC\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mC\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdepth\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdepth\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     69\u001b[0m   \u001b[0mN\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mroot\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mnext_move\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     70\u001b[0m   \u001b[0mW\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mroot\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mnext_move\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-20-c7b65a52744c>\u001b[0m in \u001b[0;36mMCTS_rec\u001b[0;34m(model, root, tree, LTL_formula, trajectory, predicates, N, W, Q, P, C, depth)\u001b[0m\n\u001b[1;32m     66\u001b[0m   \u001b[0;31m#################################\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     67\u001b[0m   \u001b[0;31m### expanding the next move and back tracking ###\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 68\u001b[0;31m   \u001b[0mvalue\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mMCTS_rec\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msub_root\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msub_tree\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mLTL_formula\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrajectory\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpredicates\u001b[0m\u001b[0;34m,\u001b[0m  \u001b[0mN\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mW\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mQ\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mP\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mC\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mC\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdepth\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdepth\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     69\u001b[0m   \u001b[0mN\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mroot\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mnext_move\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     70\u001b[0m   \u001b[0mW\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mroot\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mnext_move\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-20-c7b65a52744c>\u001b[0m in \u001b[0;36mMCTS_rec\u001b[0;34m(model, root, tree, LTL_formula, trajectory, predicates, N, W, Q, P, C, depth)\u001b[0m\n\u001b[1;32m     66\u001b[0m   \u001b[0;31m#################################\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     67\u001b[0m   \u001b[0;31m### expanding the next move and back tracking ###\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 68\u001b[0;31m   \u001b[0mvalue\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mMCTS_rec\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msub_root\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msub_tree\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mLTL_formula\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrajectory\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpredicates\u001b[0m\u001b[0;34m,\u001b[0m  \u001b[0mN\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mW\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mQ\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mP\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mC\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mC\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdepth\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdepth\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     69\u001b[0m   \u001b[0mN\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mroot\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mnext_move\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     70\u001b[0m   \u001b[0mW\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mroot\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mnext_move\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-20-c7b65a52744c>\u001b[0m in \u001b[0;36mMCTS_rec\u001b[0;34m(model, root, tree, LTL_formula, trajectory, predicates, N, W, Q, P, C, depth)\u001b[0m\n\u001b[1;32m     66\u001b[0m   \u001b[0;31m#################################\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     67\u001b[0m   \u001b[0;31m### expanding the next move and back tracking ###\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 68\u001b[0;31m   \u001b[0mvalue\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mMCTS_rec\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msub_root\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msub_tree\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mLTL_formula\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrajectory\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpredicates\u001b[0m\u001b[0;34m,\u001b[0m  \u001b[0mN\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mW\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mQ\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mP\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mC\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mC\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdepth\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdepth\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     69\u001b[0m   \u001b[0mN\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mroot\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mnext_move\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     70\u001b[0m   \u001b[0mW\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mroot\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mnext_move\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-20-c7b65a52744c>\u001b[0m in \u001b[0;36mMCTS_rec\u001b[0;34m(model, root, tree, LTL_formula, trajectory, predicates, N, W, Q, P, C, depth)\u001b[0m\n\u001b[1;32m     66\u001b[0m   \u001b[0;31m#################################\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     67\u001b[0m   \u001b[0;31m### expanding the next move and back tracking ###\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 68\u001b[0;31m   \u001b[0mvalue\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mMCTS_rec\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msub_root\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msub_tree\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mLTL_formula\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrajectory\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpredicates\u001b[0m\u001b[0;34m,\u001b[0m  \u001b[0mN\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mW\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mQ\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mP\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mC\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mC\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdepth\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdepth\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     69\u001b[0m   \u001b[0mN\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mroot\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mnext_move\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     70\u001b[0m   \u001b[0mW\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mroot\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mnext_move\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-20-c7b65a52744c>\u001b[0m in \u001b[0;36mMCTS_rec\u001b[0;34m(model, root, tree, LTL_formula, trajectory, predicates, N, W, Q, P, C, depth)\u001b[0m\n\u001b[1;32m     66\u001b[0m   \u001b[0;31m#################################\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     67\u001b[0m   \u001b[0;31m### expanding the next move and back tracking ###\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 68\u001b[0;31m   \u001b[0mvalue\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mMCTS_rec\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msub_root\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msub_tree\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mLTL_formula\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrajectory\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpredicates\u001b[0m\u001b[0;34m,\u001b[0m  \u001b[0mN\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mW\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mQ\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mP\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mC\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mC\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdepth\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdepth\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     69\u001b[0m   \u001b[0mN\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mroot\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mnext_move\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     70\u001b[0m   \u001b[0mW\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mroot\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mnext_move\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-20-c7b65a52744c>\u001b[0m in \u001b[0;36mMCTS_rec\u001b[0;34m(model, root, tree, LTL_formula, trajectory, predicates, N, W, Q, P, C, depth)\u001b[0m\n\u001b[1;32m     66\u001b[0m   \u001b[0;31m#################################\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     67\u001b[0m   \u001b[0;31m### expanding the next move and back tracking ###\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 68\u001b[0;31m   \u001b[0mvalue\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mMCTS_rec\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msub_root\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msub_tree\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mLTL_formula\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrajectory\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpredicates\u001b[0m\u001b[0;34m,\u001b[0m  \u001b[0mN\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mW\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mQ\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mP\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mC\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mC\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdepth\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdepth\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     69\u001b[0m   \u001b[0mN\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mroot\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mnext_move\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     70\u001b[0m   \u001b[0mW\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mroot\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mnext_move\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-20-c7b65a52744c>\u001b[0m in \u001b[0;36mMCTS_rec\u001b[0;34m(model, root, tree, LTL_formula, trajectory, predicates, N, W, Q, P, C, depth)\u001b[0m\n\u001b[1;32m     66\u001b[0m   \u001b[0;31m#################################\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     67\u001b[0m   \u001b[0;31m### expanding the next move and back tracking ###\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 68\u001b[0;31m   \u001b[0mvalue\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mMCTS_rec\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msub_root\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msub_tree\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mLTL_formula\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrajectory\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpredicates\u001b[0m\u001b[0;34m,\u001b[0m  \u001b[0mN\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mW\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mQ\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mP\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mC\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mC\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdepth\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdepth\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     69\u001b[0m   \u001b[0mN\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mroot\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mnext_move\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     70\u001b[0m   \u001b[0mW\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mroot\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mnext_move\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-20-c7b65a52744c>\u001b[0m in \u001b[0;36mMCTS_rec\u001b[0;34m(model, root, tree, LTL_formula, trajectory, predicates, N, W, Q, P, C, depth)\u001b[0m\n\u001b[1;32m     66\u001b[0m   \u001b[0;31m#################################\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     67\u001b[0m   \u001b[0;31m### expanding the next move and back tracking ###\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 68\u001b[0;31m   \u001b[0mvalue\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mMCTS_rec\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msub_root\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msub_tree\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mLTL_formula\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrajectory\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpredicates\u001b[0m\u001b[0;34m,\u001b[0m  \u001b[0mN\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mW\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mQ\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mP\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mC\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mC\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdepth\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdepth\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     69\u001b[0m   \u001b[0mN\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mroot\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mnext_move\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     70\u001b[0m   \u001b[0mW\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mroot\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mnext_move\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-20-c7b65a52744c>\u001b[0m in \u001b[0;36mMCTS_rec\u001b[0;34m(model, root, tree, LTL_formula, trajectory, predicates, N, W, Q, P, C, depth)\u001b[0m\n\u001b[1;32m     66\u001b[0m   \u001b[0;31m#################################\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     67\u001b[0m   \u001b[0;31m### expanding the next move and back tracking ###\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 68\u001b[0;31m   \u001b[0mvalue\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mMCTS_rec\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msub_root\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msub_tree\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mLTL_formula\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrajectory\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpredicates\u001b[0m\u001b[0;34m,\u001b[0m  \u001b[0mN\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mW\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mQ\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mP\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mC\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mC\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdepth\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdepth\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     69\u001b[0m   \u001b[0mN\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mroot\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mnext_move\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     70\u001b[0m   \u001b[0mW\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mroot\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mnext_move\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-20-c7b65a52744c>\u001b[0m in \u001b[0;36mMCTS_rec\u001b[0;34m(model, root, tree, LTL_formula, trajectory, predicates, N, W, Q, P, C, depth)\u001b[0m\n\u001b[1;32m     66\u001b[0m   \u001b[0;31m#################################\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     67\u001b[0m   \u001b[0;31m### expanding the next move and back tracking ###\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 68\u001b[0;31m   \u001b[0mvalue\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mMCTS_rec\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msub_root\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msub_tree\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mLTL_formula\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrajectory\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpredicates\u001b[0m\u001b[0;34m,\u001b[0m  \u001b[0mN\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mW\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mQ\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mP\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mC\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mC\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdepth\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdepth\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     69\u001b[0m   \u001b[0mN\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mroot\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mnext_move\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     70\u001b[0m   \u001b[0mW\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mroot\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mnext_move\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-20-c7b65a52744c>\u001b[0m in \u001b[0;36mMCTS_rec\u001b[0;34m(model, root, tree, LTL_formula, trajectory, predicates, N, W, Q, P, C, depth)\u001b[0m\n\u001b[1;32m     66\u001b[0m   \u001b[0;31m#################################\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     67\u001b[0m   \u001b[0;31m### expanding the next move and back tracking ###\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 68\u001b[0;31m   \u001b[0mvalue\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mMCTS_rec\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msub_root\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msub_tree\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mLTL_formula\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrajectory\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpredicates\u001b[0m\u001b[0;34m,\u001b[0m  \u001b[0mN\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mW\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mQ\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mP\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mC\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mC\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdepth\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdepth\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     69\u001b[0m   \u001b[0mN\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mroot\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mnext_move\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     70\u001b[0m   \u001b[0mW\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mroot\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mnext_move\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-20-c7b65a52744c>\u001b[0m in \u001b[0;36mMCTS_rec\u001b[0;34m(model, root, tree, LTL_formula, trajectory, predicates, N, W, Q, P, C, depth)\u001b[0m\n\u001b[1;32m     40\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     41\u001b[0m   \u001b[0;32mif\u001b[0m \u001b[0mdepth\u001b[0m \u001b[0;34m<\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;31m# search depth limit reached\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 42\u001b[0;31m       \u001b[0moutcome\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcheck_LTL\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mLTL_formula\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrajectory\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpredicates\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     43\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mall\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutcome\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     44\u001b[0m       \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-7-70191d6b39b3>\u001b[0m in \u001b[0;36mcheck_LTL\u001b[0;34m(formula, trajectory, predicates)\u001b[0m\n\u001b[1;32m     45\u001b[0m     \u001b[0;31m# Evaluate the operands.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     46\u001b[0m     \u001b[0mevaluated_operand1\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcheck_LTL\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mformula\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrajectory\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpredicates\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 47\u001b[0;31m     \u001b[0mevaluated_operand2\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcheck_LTL\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mformula\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrajectory\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpredicates\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     48\u001b[0m     \u001b[0mevaluation\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mall\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mevaluated_operand1\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mevaluated_operand2\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mevaluated_operand1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     49\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-7-70191d6b39b3>\u001b[0m in \u001b[0;36mcheck_LTL\u001b[0;34m(formula, trajectory, predicates)\u001b[0m\n\u001b[1;32m     40\u001b[0m     \u001b[0;31m# Evaluate the operand.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     41\u001b[0m     \u001b[0mevaluated_operand\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcheck_LTL\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mformula\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrajectory\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpredicates\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 42\u001b[0;31m     \u001b[0mevaluation\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mall\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mevaluated_operand\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mevaluated_operand\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     43\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     44\u001b[0m   \u001b[0;32melif\u001b[0m \u001b[0mformula\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m==\u001b[0m\u001b[0;34m'&'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-7-70191d6b39b3>\u001b[0m in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m     40\u001b[0m     \u001b[0;31m# Evaluate the operand.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     41\u001b[0m     \u001b[0mevaluated_operand\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcheck_LTL\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mformula\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrajectory\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpredicates\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 42\u001b[0;31m     \u001b[0mevaluation\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mall\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mevaluated_operand\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mevaluated_operand\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     43\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     44\u001b[0m   \u001b[0;32melif\u001b[0m \u001b[0mformula\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m==\u001b[0m\u001b[0;34m'&'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "same as above, but also with goal trajectory"
      ],
      "metadata": {
        "id": "WErcV2Reus-R"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "np.reshape(results_1[:55],(-1,5)), results_1[55:]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "fbgCbfO5DyAK",
        "outputId": "ecd25720-0867-4b59-bc4b-fdc851b4abc9"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(array([[ True,  True,  True,  True,  True],\n",
              "        [ True,  True,  True,  True,  True],\n",
              "        [ True,  True,  True,  True,  True],\n",
              "        [ True,  True,  True,  True,  True],\n",
              "        [ True,  True,  True,  True,  True],\n",
              "        [ True,  True,  True,  True,  True],\n",
              "        [ True,  True,  True,  True,  True],\n",
              "        [ True,  True,  True,  True,  True],\n",
              "        [ True,  True,  True,  True, False],\n",
              "        [ True,  True,  True, False,  True],\n",
              "        [ True, False,  True, False,  True]]), [True, True])"
            ]
          },
          "metadata": {},
          "execution_count": 72
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "shapes = [(i,i) for i in range(5,40,2)]\n",
        "results_2 = []\n",
        "# 2 Goals\n",
        "for gw_shape in shapes:\n",
        "  print(gw_shape, end=\": \")\n",
        "  for _ in range(5):\n",
        "    N, W, Q, P = {}, {}, {}, {}\n",
        "    outcome = run_simulation(gw_shape, 2, ordered_goals=True, n_episodes=3, C=1, tow=0.1, n_steps=gw_shape[0]*2+5, n_samples=500, N=N, W=W, Q=Q, P=P, verbose=False)\n",
        "    results_2.append(outcome)\n",
        "    print(outcome,end=\", \")\n",
        "  print()\n",
        "\n",
        "results_3 = []\n",
        "# 3 Goals\n",
        "for gw_shape in shapes:\n",
        "  print(gw_shape, end=\": \")\n",
        "  for _ in range(5):\n",
        "    N, W, Q, P = {}, {}, {}, {}\n",
        "    outcome = run_simulation(gw_shape, 3, ordered_goals=True, n_episodes=3, C=1, tow=0.1, n_steps=gw_shape[0]*2+5, n_samples=500, N=N, W=W, Q=Q, P=P, verbose=False)\n",
        "    results_3.append(outcome)\n",
        "    print(outcome,end=\", \")\n",
        "  print()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Ug1BpQ3ErPJe",
        "outputId": "73ae131e-0de4-4995-d111-a348df25a834"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[True, True]"
            ]
          },
          "metadata": {},
          "execution_count": 62
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "test cases for multiple goal reach (trajectory)"
      ],
      "metadata": {
        "id": "4G4dvraKqoJF"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "shapes = [(i,i) for i in range(5,38, 2)]\n",
        "results_1 = []\n",
        "print(\"1 goal\")\n",
        "for gw_shape in shapes:\n",
        "  print(gw_shape, end=\": \")\n",
        "  for _ in range(5):\n",
        "    N, W, Q, P = {}, {}, {}, {}\n",
        "    outcome = run_simulation(gw_shape, 1, n_episodes=3, C=1, tow=0.1, n_steps=gw_shape[0]*2+5, n_samples=500, N=N, W=W, Q=Q, P=P, verbose=False)\n",
        "    results_1.append(outcome)\n",
        "    print(outcome,end=\", \")\n",
        "  print()\n",
        "\n",
        "print(\"2 goals\")\n",
        "results_2 = []\n",
        "for gw_shape in shapes:\n",
        "  print(gw_shape, end=\": \")\n",
        "  for _ in range(5):\n",
        "    N, W, Q, P = {}, {}, {}, {}\n",
        "    outcome = run_simulation(gw_shape, 2, n_episodes=3, C=1, tow=0.1, n_steps=gw_shape[0]*2+5, n_samples=500, N=N, W=W, Q=Q, P=P, verbose=False)\n",
        "    results_2.append(outcome)\n",
        "    print(outcome,end=\", \")\n",
        "  print()\n",
        "\n",
        "print(\"3 goals\")\n",
        "results_3 = []\n",
        "for gw_shape in shapes:\n",
        "  print(gw_shape, end=\": \")\n",
        "  for _ in range(5):\n",
        "    N, W, Q, P = {}, {}, {}, {}\n",
        "    outcome = run_simulation(gw_shape, 3, n_episodes=3, C=1, tow=0.1, n_steps=gw_shape[0]*2+5, n_samples=500, N=N, W=W, Q=Q, P=P, verbose=False)\n",
        "    results_3.append(outcome)\n",
        "    print(outcome,end=\", \")\n",
        "  print()\n",
        "\n",
        "print(\"3 goals ordered\")\n",
        "results_3_1 = []\n",
        "for gw_shape in shapes:\n",
        "  print(gw_shape, end=\": \")\n",
        "  for _ in range(5):\n",
        "    N, W, Q, P = {}, {}, {}, {}\n",
        "    outcome = run_simulation(gw_shape, n_goals= 3, ordered_goals=True, n_episodes=3, C=1, tow=0.1, n_steps=gw_shape[0]*2+5, n_samples=500, N=N, W=W, Q=Q, P=P, verbose=False)\n",
        "    results_3_1.append(outcome)\n",
        "    print(outcome,end=\", \")\n",
        "  print()"
      ],
      "metadata": {
        "id": "pxGio4Lf4F5x"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "plt.bar(range(5,27,2), np.reshape(results_1[:55], (-1, 5)).mean(1))\n",
        "plt.title(\"success rate based on grid world size - stay at end goal (1 goal)\")\n",
        "plt.xlabel(\"dim of grid_wrold\")\n",
        "plt.ylabel(\"success rate\")\n",
        "plt.figure()\n",
        "\n",
        "plt.bar(range(5,38,2), np.reshape(results_2, (-1, 5)).mean(1))\n",
        "plt.title(\"success rate based on levels of complexity (2 goals)\")\n",
        "plt.xlabel(\"dim of grid_wrold\")\n",
        "plt.ylabel(\"success rate\")"
      ],
      "metadata": {
        "id": "xlIbjGwDYD4Y",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 331
        },
        "outputId": "69aa7188-d003-4000-d1df-86d7053d5190"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 0 Axes>"
            ]
          },
          "metadata": {},
          "execution_count": 77
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZwAAAEXCAYAAACZNvIiAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dabgcVbn28f9NwkwAYyIHSEIYAq8BPAgRZFBAUSYFnJCIYhAEXwZBkEHhKKAcwQn1yKggg0wRwRMlCg7MMgUEJAlogEASIiRIwjyF53xYa4ei0713Jeyq3uncv+va1+6qWlX19OpV9VStqq5WRGBmZla1pdodgJmZLRmccMzMrBZOOGZmVgsnHDMzq4UTjpmZ1cIJx8zMauGE00dIGiPp5jbHsJ2k6e2MoRVJZ0n6r26mh6T1ao7pfEnf7s2YJO0t6dq3Hp0tjHa1/RJtaLCkByQtX2dcPZE0VdIO+fWhkk4tM58TTi/pCwmjk0XElyLiW+2Oo2oRcXFEfLjdcXQp7lhqXu/wnLD7173uPuZY4PyIeBFA0p6S/irpBUnXtze0+X4G7C3pHT0VdMIpwY2+vST1a3cMjfpiTNZZJC0LfB74ZWH0v4EfAae0JagmIuIl4PfAPj2V7dWEI+kYSTMkPSvpQUkfzOPfdNrYePoqaaikKyXNkvSUpJ8Wpn1R0uS8zEmSNs3j15D06zzPI5K+XJhnc0kTJD0j6QlJP8zjl5P0y7yOOZLulLRai/cyNb+f+4DnJfWXdKykhwqxfCyXfSdwFrClpOckzcnjl5X0fUmP5TjO6uHUWJJ+KmluPo3+YGHCvoV6eFjSgYVpgyT9Lr+nf0u6SdJSJepp+fzZPC1pEvCeHj7frXKdzc3/typMu17StyTdkmO8VtKgbpZ1tKSZkh6XtH+x+ynHdKak8ZKeB7Zv0oaOKsz/hW7Ws72kvxeG/yjpzsLwTZL2yK/fmd/HHEkTJe1WKLdATE3WVSqmXHZM/hyfzZ/L3oXxNxfq6LnC36uSzs/TVpF0bl7fDEnf1iImwVbtR9JFwDDgt3n9R+fyv5L0r9wObpS0YR7/ntzO+xWW/XFJ97ZY766S/qa0nU6TdEJh8o35/5y87i2bzL+U3tgmn5I0VtLAPK3rDOnzStvfbEnHFeZd2Lb/YaV92lxJZ0i6QdL+hTiOl/SopCclXShplcK8TeurhC2AORExf18ZEX+KiLHA42UW0MN2tkqOdVaO/Xi9sd9YV9Jfcr3OlnSxpFW7WdX1wK49BhQRvfIHbABMA9bIw8OBdfPr84FvF8puB0zPr/sB9wKnASsCywHb5GmfAmaQGoOA9YC1SInyLuAbwDLAOsDDwI55vluBz+XXKwHvza8PBH4LrJDXuxmwcov3MxW4BxgKLF+IZ428/k8DzwOr52ljgJsblnEaMA4YCAzI6/5Oi/WNAV4DvgIsnZc/FxiYp+8KrJvrYVvgBWDTPO07pIS3dP57Xy7XUz2dAtyU4xsK3N/1uTSJbyDwNPA5oD8wOg+/PU+/HngIWB9YPg+f0mJZOwH/AjbMn8UvgQDWK7SXucDW+T0sR6EN5fmfADYitZlLivM3rGt54CVgUK6bJ0htakCe9iLw9jxtCvD1XFcfAJ4FNqggphWBZwrLXh3YsFU7yuOHknYyO+fhq4Cz87LeAdwBHLiI227T9lPYDnZoKP+FXH/Lko627ylMm9QVYyHOI1usdztg41yf78r1t0dh/xFA/27iPgy4DRiSYzkbuLRh/p/lz/k/gZeBdy5C2x+UP6+Pk9r+YcCrwP6F+phC2r5WAq4ELipZX/PbUJP1Hgxc3WLa/sD1PXyuPW1nFwL/m2MbDvwD2C9PWw/4UI55MOkA4EcN+8cdCsObAv/usa0tSgNt8ebWA54EdgCWbpj2pkrlzQlnS2BWs4YFXAMc1mT8FsBjDeO+Bvwiv74ROBEY1GRD+SvwrhLvZyrwhR7K3APsnl+PobCjIO3wnycn3cJ7faTFssaQdigqjLuDnDiblP9NV90AJ+WGs15DmZ7q6WFgp8K0A2i90X0OuKNh3K3AmPz6euD4wrSDgD+0WNZ5FBJvbjuNCefCVm0oz39KYdr6tNi55+k3kXYW7wWuBcaSNsbtgftymfeRNs6lCvNdCpzQ2zGRksQc4BPkg5mGdtB44LI86cDhmDy8GmnnuXyhzGjgup7adYv6adp+CtvBDt3Mu2p+n6vk4WOAi/PrgaQDo9VLxvEj4LT8ejg9J5zJwAcLw6uTEkH/wvxDGranvRah7e8D3FoYFunguivh/Bk4qDB9g644StTX/DbUpOxxwGUtppVJOC23M9IB9yvAyML0A1stE9gD+FurdgGMAOb19Bn3WpdaREwBDgdOAJ6UdJmkNUrMOhR4NCJeazHtoSbj1wLWyF0Ac5S6sL5O2hAB9iNt8A8odf18JI+/iJTELsunmN+VtHQ3sU0rDkjaR9I9hXVuRDr6aWYw6ajirkL5P+TxrcyI/Ollj5LOqJC0s6TbcpfHHGCXwrq/RzrCujZ30xybx/dUT2s0vMdHu4ltjSbTHwXWLAz/q/D6BdLRXqtlFdc7rUmZZuNazd9d3AA3kA5y3p9fX086S9w2D89fZkS83rDc4vvrlZgi4nnSGeyXgJmSrpb0/7pZ9rnAgxHRdSfQWqQzkZmFz/Vs0pnOAhq65YY1KdKq/TRbVj9Jp+RurGdIOx54oy3+EviopBWBPYGbImJmi2VtIem63KUzl1QfLbthm1gLuKpQB5OBebzRvqF1m1zYtj+/bN5GpzdML87/KCnprVaivrrzNOnsY1F1t511nfE3xr0mgKTV8j58Ro77lz3EPIDUA9CtXr2GExGXRMQ2pIYQQNcG8jxp59vlPwqvpwHD1PzC/DRSN1Kz8Y9ExKqFvwERsUuO458RMZq0AZ4KXCFpxYh4NSJOjIiRwFbAR+j+Qtf8nb+ktUin54eQupFWJZ2Gq7FsNpvUXbNhIcZVIqLVThhgTUkqDA8DHle6ePhr4PvAannd47vWHRHPRsSREbEOsBtwhNL1n27rCZhJSurF9bXyOOlzLRpG6p5aWDNJ3SBdhjYp01ifjfOXjRsWTDg3sGDCeRwY2tWHXVhu8f31WkwRcU1EfIh0VP4AqW0tIO/81ycdRHWZRjrDGVT4XFeOiKbXBiJipcLfY02mt2o/sOB7/gywO6knYxXSmQS80RZnkM58P046K76om2q4hNTlPDQiViF167XanpqZRuq+K7bv5XIMPVmYz+tN7TVvo8X227htDCN1jz9BD/XVg/tIn/2i6m47m006C2uMu6vu/pv0GWwcESsDn6X7mN9JujTSrV5LOJI2kPSBvHN8ibSz7TpavAfYRdJASf9BOhPqcgepYk6RtKLShf2t87SfA1+VtJmS9fKO/w7gWaWL+svno4iNJL0nx/JZSYPz0eqcvKzXlS4gb6x0UfMZUoUXj2i7syLpA5iV17Ev6QynyxPAEEnLAOR1/ww4Tfl2QUlrStqxm3W8A/iypKUlfYr0IY4nXVNYNq/7NUk7A/NvnZX0kVw3Ih1lzMvvq9t6InUtfU3S2yQNAQ7tJrbxwPqSPqN0A8WngZHA77qvtqbGAvsqXaRfAWj5/Zpu5h8jaWSe/5s9lP8rqZtjc1K34ETShrYFb1ycvp10BHx0rv/tgI8Cl/V2TPnocfd8FvAy8BxN2mH+nL8MfCzybbEA+YzhWuAHklZWumi9rqRtS8bauJ5W7QdSu16nUHxAjvkp0kHkfzdZ5IXA0aTrM1d2s+oBpH7/lyRtTto5d5mVY1in6ZzJWcDJeZ/Q9Z2V3bspX7Qwbf9qYGNJe+QD44N580HzpcBXJK0taSVSnVyee23K1FcrdwCrSpp/lp234eVIZ1BL5f1lq16alttZRMzL00+WNCDX4RG8cUfcAFK7nJvXf1QPsW5LulOtW715hrMs6ULcbNJp7DtI1wsgHeXcSzqdvBa4vGum/MY/SupXfIx0qvrpPO1XwMmkI6FnSdctBuZ5PgJsAjyS1/lz0hEEpP75iZKeA35M6rd9kdRIriAlm8mko9vujsDmi4hJwA9IR29PkDamWwpF/gJMBP4laXYedwypq+K2fFr6J9KOr5XbSX2hs/P7/mREPBURz5J2PGNJp9mfIR0ZdhmRl/1cju+MiLiuRD2dSDqNfoT0ubSsi4h4Ki/rSNLGczTwkYiY3Wqebpb1e+AnwHXk+smTXl6I+X9EqvMp+X935Z8H7gYmRsQrefStpK7cJ3OZV0jtcGdSPZ0B7BMRD1QQ01Kkjftx0m2u2wL/v0m5T5O6YCfrjS6xs/K0fUgHIpNIbeIK0tnSomjafvK07wDH526rr5KSyaOkI+FJvPHZFV1F7u6KiBe6We9BwEmSniXd2DK2a0Ke72Tglrzu9zaZ/8ek7eDavIzbSAcRZSxM259NumHou6S2PxKYwBvt9bw8/415eS/xRgIrU1+t1vsK6RrPZwujP0c6mD+TdN3xRVqcHZfYzg4l9T49DNxM2s+el6edSLoRYC4p4bY8cMgJcBfggp7eU9edKGZto3Rb+f3Asi2u5dliRtJDpLvm/tTuWHpb7nadDuxdSMxVrWsw6aaXdxfPchdxWZVsZ5IOJXWLHt1jWSccawel7zCNJ3UzXAC8HhF7tDcq6w2SPkG6drp+w00Yi63cFX476YziKFK32jpvNQlUra9tZ37SgLXLgaTb6B8iXTNo1qVkixmlx62cCRzcKckm25LUVmeTul736OvJJutT25nPcMzMrBY+wzEzs1osdg+lHDRoUAwfPrzdYZiZLVbuuuuu2RHR3RfPK7fYJZzhw4czYcKEdodhZrZYkdTTEzkq5y41MzOrhROOmZnVwgnHzMxq4YRjZma1cMIxM7NaOOGYmVktKks4ks5T+n3v+1tMl6SfSJoi6T5Jm1YVi5mZtV+VZzjnk34moJWdSY9FH0H6edczK4zFzMzarLKEExE3kn7ro5XdSb8RHxFxG+mHhhb19zzMzKyPa+eTBtbkzb+xPT2PW+D3zyUdQDoLYtiwnn5NuLXhx169UOWnnrLrIq9rUdfbjnX21nqXlHUu7Hr9mZoli8VNAxFxTkSMiohRgwe39VFAZma2iNqZcGYAQwvDQ/I4MzPrQO1MOOOAffLdau8F5kbEAt1pZmbWGSq7hiPpUmA7YJCk6cA3gaUBIuIs0s+e7gJMAV4A9q0qFjMza7/KEk5EjO5hepB+F9zMzJYAi8VNA2ZmtvhzwjEzs1o44ZiZWS2ccMzMrBZOOGZmVgsnHDMzq4UTjpmZ1cIJx8zMauGEY2ZmtXDCMTOzWjjhmJlZLZxwzMysFk44ZmZWCyccMzOrhROOmZnVwgnHzMxq4YRjZma1cMIxM7NaOOGYmVktnHDMzKwWTjhmZlYLJxwzM6uFE46ZmdXCCcfMzGrhhGNmZrVwwjEzs1o44ZiZWS2ccMzMrBZOOGZmVgsnHDMzq4UTjpmZ1cIJx8zMalFpwpG0k6QHJU2RdGyT6cMkXSfpb5Luk7RLlfGYmVn7VJZwJPUDTgd2BkYCoyWNbCh2PDA2It4N7AWcUVU8ZmbWXlWe4WwOTImIhyPiFeAyYPeGMgGsnF+vAjxeYTxmZtZG/Stc9prAtMLwdGCLhjInANdKOhRYEdih2YIkHQAcADBs2LBeD9TM+o7hx169UOWnnrJr7evtrXUuadp908Bo4PyIGALsAlwkaYGYIuKciBgVEaMGDx5ce5BmZvbWVZlwZgBDC8ND8rii/YCxABFxK7AcMKjCmMzMrE2qTDh3AiMkrS1pGdJNAeMayjwGfBBA0jtJCWdWhTGZmVmbVJZwIuI14BDgGmAy6W60iZJOkrRbLnYk8EVJ9wKXAmMiIqqKyczM2qfKmwaIiPHA+IZx3yi8ngRsXWUMZmbWN7T7pgEzM1tCOOGYmVktnHDMzKwWTjhmZlYLJxwzM6uFE46ZmdXCCcfMzGrhhGNmZrVwwjEzs1o44ZiZWS2ccMzMrBZOOGZmVgsnHDMzq4UTjpmZ1cIJx8zMauGEY2ZmtXDCMTOzWjjhmJlZLZxwzMysFk44ZmZWCyccMzOrRemEI2mFKgMxM7PO1mPCkbSVpEnAA3n4PyWdUXlkZmbWUcqc4ZwG7Ag8BRAR9wLvrzIoMzPrPKW61CJiWsOoeRXEYmZmHax/iTLTJG0FhKSlgcOAydWGZWZmnabMGc6XgIOBNYEZwCbAQVUGZWZmnafMGc4GEbF3cYSkrYFbqgnJzMw6UZkznP8pOc7MzKyllmc4krYEtgIGSzqiMGlloF/VgZmZWWfprkttGWClXGZAYfwzwCerDMrMzDpPy4QTETcAN0g6PyIerTEmMzPrQGWu4bwg6XuSxkv6S9dfmYVL2knSg5KmSDq2RZk9JU2SNFHSJQsVvZmZLTbKJJyLSY+1WRs4EZgK3NnTTJL6AacDOwMjgdGSRjaUGQF8Ddg6IjYEDl+Y4M3MbPFRJuG8PSLOBV6NiBsi4gvAB0rMtzkwJSIejohXgMuA3RvKfBE4PSKeBoiIJxcidjMzW4yUSTiv5v8zJe0q6d3AwBLzrQkUH4kzPY8rWh9YX9Itkm6TtFOJ5ZqZ2WKozBc/vy1pFeBI0vdvVga+0ovrHwFsBwwBbpS0cUTMKRaSdABwAMCwYcN6adVmZu01/NirF6r81FN2rSiSenR7hpOvw4yIiLkRcX9EbB8Rm0XEuBLLngEMLQwPyeOKpgPjIuLViHgE+AcpAb1JRJwTEaMiYtTgwYNLrNrMzPqabhNORMwDRi/isu8ERkhaW9IywF5AY6L6DensBkmDSF1sDy/i+szMrA8r06V2i6SfApcDz3eNjIi7u5spIl6TdAhwDenJBOdFxERJJwET8lnSNcCH8w+8zQOOioinFvG9mJlZH1Ym4WyS/59UGBeUuFMtIsYD4xvGfaPwOoAj8p+ZmXWwHhNORGxfRyBmZtbZSv3ip5mZ2VvlhGNmZrVwwjEzs1r0mHAkfUrSgPz6eElXStq0+tDMzKyTlDnD+a+IeFbSNsAOwLnAmdWGZWZmnaZMwpmX/+8KnBMRV5N+nM3MzKy0MglnhqSzgU8D4yUtW3I+MzOz+cokjj1JTwTYMT9UcyBwVKVRmZlZxynzpIHVgasj4mVJ2wHvAi6sNCozM+s4Zc5wfg3Mk7QecA7pCdD+KWgzM1soZRLO6xHxGvBx4H8i4ijSWY+ZmVlppX7xU9JoYB/gd3nc0tWFZGZmnahMwtkX2BI4OSIekbQ2cFG1YZmZWacp87ToSZKOAYbl4UeAU6sOzMzMOkuZR9t8FLgH+EMe3kRSmZ+YNjMzm69Ml9oJwObAHICIuAdYp8KYzMysA5W6aSAi5jaMe72KYMzMrHOV+eLnREmfAfpJGgF8GfhrtWGZmVmnKXOGcyiwIfAy6Qufc4HDqwzKzMw6T5m71F4Ajst/ZmZmi6TMXWp/lLRqYfhtkq6pNiwzM+s0ZbrUBuWnRAMQEU8D76guJDMz60SlnqUmaVjXgKS1gKguJDMz60Rl7lI7DrhZ0g2AgPcBB1QalZmZdZwyNw38QdKmwHvzqMMjYna1YZmZWacpc9PAx0hf/vxdRPwOeE3SHtWHZmZmnaTMNZxvFp80kG8g+GZ1IZmZWScqk3CalSlz7cfMzGy+MglngqQfSlo3//0QuKvqwMzMrLOUfbTNK8Dl+e9l4OAqgzIzs85T5i6154Fja4jFzMw6WI8JR9J1NPmiZ0R8oJKIzMysI5W5+P/VwuvlgE8Ar5VZuKSdgB8D/YCfR8QpLcp9ArgCeE9ETCizbDMzW7yU6VJrvEHgFkl39DSfpH7A6cCHgOnAnZLGRcSkhnIDgMOA20tHbWZmi50yX/wcWPgbJGlHYJUSy94cmBIRD0fEK8BlwO5Nyn0LOBV4aWECNzOzxUuZLrW7SNdwROpKewTYr8R8awLTCsPTgS2KBfIjc4ZGxNWSjmq1IEkHkJ/fNmzYsFbFzMysDyvTpbZ2FSuWtBTwQ2BMiRjOAc4BGDVqlJ9UbWa2GCrTpfapfJ0FScdLujKfmfRkBjC0MDwkj+syANgIuF7SVNLDQcdJGlU2eDMzW3yU+eLnf0XEs5K2AXYAzgXOLDHfncAISWtLWgbYCxjXNTEi5kbEoIgYHhHDgduA3XyXmplZZyqTcObl/7sC50TE1cAyPc0UEa8BhwDXAJOBsRExUdJJknZb1IDNzGzxVOamgRmSzibd3nyqpGUpl6iIiPHA+IZx32hRdrsyyzQzs8VTmcSxJ+ksZcf80wQDgZZ3lJmZmTVT5i61F4ArC8MzgZlVBmVmZp2nVNeYmZnZW+WEY2ZmtXDCMTOzWjjhmJlZLZxwzMysFk44ZmZWCyccMzOrhROOmZnVwgnHzMxq4YRjZma1cMIxM7NaOOGYmVktnHDMzKwWTjhmZlYLJxwzM6uFE46ZmdXCCcfMzGrhhGNmZrVwwjEzs1o44ZiZWS2ccMzMrBZOOGZmVgsnHDMzq4UTjpmZ1cIJx8zMauGEY2ZmtXDCMTOzWjjhmJlZLZxwzMysFk44ZmZWi0oTjqSdJD0oaYqkY5tMP0LSJEn3SfqzpLWqjMfMzNqnsoQjqR9wOrAzMBIYLWlkQ7G/AaMi4l3AFcB3q4rHzMzaq8oznM2BKRHxcES8AlwG7F4sEBHXRcQLefA2YEiF8ZiZWRtVmXDWBKYVhqfnca3sB/y+2QRJB0iaIGnCrFmzejFEMzOrS5+4aUDSZ4FRwPeaTY+IcyJiVESMGjx4cL3BmZlZr+hf4bJnAEMLw0PyuDeRtANwHLBtRLxcYTxmZtZGVZ7h3AmMkLS2pGWAvYBxxQKS3g2cDewWEU9WGIuZmbVZZQknIl4DDgGuASYDYyNioqSTJO2Wi30PWAn4laR7JI1rsTgzM1vMVdmlRkSMB8Y3jPtG4fUOVa7fzMz6jj5x04CZmXU+JxwzM6uFE46ZmdXCCcfMzGrhhGNmZrVwwjEzs1o44ZiZWS2ccMzMrBZOOGZmVgsnHDMzq4UTjpmZ1cIJx8zMauGEY2ZmtXDCMTOzWjjhmJlZLZxwzMysFk44ZmZWCyccMzOrhROOmZnVwgnHzMxq4YRjZma1cMIxM7NaOOGYmVktnHDMzKwWTjhmZlYLJxwzM6uFE46ZmdXCCcfMzGrhhGNmZrVwwjEzs1o44ZiZWS2ccMzMrBZOOGZmVotKE46knSQ9KGmKpGObTF9W0uV5+u2ShlcZj5mZtU9lCUdSP+B0YGdgJDBa0siGYvsBT0fEesBpwKlVxWNmZu1V5RnO5sCUiHg4Il4BLgN2byizO3BBfn0F8EFJqjAmMzNrE0VENQuWPgnsFBH75+HPAVtExCGFMvfnMtPz8EO5zOyGZR0AHJAHNwAe7OVwBwGzeyy1ZHMd9cx1VI7rqWdV1NFaETG4l5e5UPq3c+VlRcQ5wDlVLV/ShIgYVdXyO4HrqGeuo3JcTz3r1DqqskttBjC0MDwkj2taRlJ/YBXgqQpjMjOzNqky4dwJjJC0tqRlgL2AcQ1lxgGfz68/CfwlqurjMzOztqqsSy0iXpN0CHAN0A84LyImSjoJmBAR44BzgYskTQH+TUpK7VBZd10HcR31zHVUjuupZx1ZR5XdNGBmZlbkJw2YmVktnHDMzKwWS3zCkTRV0t8l3SNpQrvj6QsknSfpyfw9qa5xAyX9UdI/8/+3tTPGdmtRRydImpHb0j2SdmlnjO0maaik6yRNkjRR0mF5vNtS1k0ddWRbWuKv4UiaCoxq/LLpkkzS+4HngAsjYqM87rvAvyPilPxcvLdFxDHtjLOdWtTRCcBzEfH9dsbWV0haHVg9Iu6WNAC4C9gDGIPbEtBtHe1JB7alJf4MxxYUETeS7hosKj6G6ALSRrHEalFHVhARMyPi7vz6WWAysCZuS/N1U0cdyQkHArhW0l35ETrW3GoRMTO//hewWjuD6cMOkXRf7nJbYruKGuUnwb8buB23paYa6gg6sC054cA2EbEp6anWB+euEutG/nLukt0X29yZwLrAJsBM4AftDadvkLQS8Gvg8Ih4pjjNbSlpUkcd2ZaW+IQTETPy/yeBq0hPubYFPZH7m7v6nZ9sczx9TkQ8ERHzIuJ14Ge4LSFpadKO9OKIuDKPdlsqaFZHndqWluiEI2nFfKEOSSsCHwbu736uJVbxMUSfB/63jbH0SV070exjLOFtKf/UyLnA5Ij4YWGS21LWqo46tS0t0XepSVqHdFYD6TE/l0TEyW0MqU+QdCmwHekR6U8A3wR+A4wFhgGPAntGxBJ70bxFHW1H6gIJYCpwYOFaxRJH0jbATcDfgdfz6K+TrlG4LdFtHY2mA9vSEp1wzMysPkt0l5qZmdXHCcfMzGrhhGNmZrVwwjEzs1o44ZiZWS2ccMzMrBZOOLZYyY9t/2p+fZKkHSpYx5clTZZ08VtYxs8ljWwyfoykn761CEutf349NYwfXvxJBbM69W93AGaLKiK+UdGiDwJ2iIjpizKzpH4RsX8vx9RsPf0j4rWq12PWW3yGY32epOMk/UPSzcAGhfHnS/pkfj1V0ne6fkhP0qaSrpH0kKQvtVjuEZLuz3+H53FnAesAv5f0lYbyK0gam38s6ypJt0salac9J+kHku4FtpR0fWHavjn+O4Ctu3mf/SQ9omRVSfO6HiYr6UZJI/KZy0WSbgEuymcsf8lPFf6zpGFNlruZpHtzbAcvRNWb9Sqf4VifJmkzYC/SYz76A3eTfqSqmcciYhNJpwHnk3buy5GeQ3VWk+XuC2wBCLhd0g0R8SVJOwHbN/lRvoOApyNipKSNgHsK01YEbo+II/Pyu9azOnAisBkwF7gO+Fuz4CNinqQHgZHA2vm9vk/S7cDQiPhnXu5I0lPOX5T0W+CCiLhA0heAn7Dg78v8AjgkIm6U9L0WdWdWOZ/hWF/3PuCqiHghP7Z9XDdlu6b9nbTzfzYiZgEvS1q1oew2ebnPR8RzwJV5Xd3ZBrgMICLuB+4rTJtHeuJvoy2A6yNiVkS8AlzewzpuAt6f/76T1/ke4M5CmXER8WJ+vSVwSX59US4/X37fq+YfjOsqY9YWTjjWSV7O/18vvO4arkfyi4sAAAFASURBVPps/qWImNcLy7mRlPg2B8YDq5IeCnpToczzvbAes9o54VhfdyOwh6Tl809JfLSXlntTXu4K+acpPsabd+rN3EL6rXnyHWgbl1jP7cC2kt6ef/fkUz2UvwPYCng9Il4iddsdSKqHZv5K6nIE2JuG9xARc4A5+anEXWXM2sLXcKxPi4i7JV0O3Ev6oa47e5hlYZZ7PmkHD/DziGh6baXgDOACSZOAB4CJpOsy3a1npqQTgFuBObz5uk+z8i9LmgbclkfdRHpU/d9bzHIo8AtJRwGzSNelGu0LnCcpgGu7W79ZlfzzBGYlSeoHLB0RL0laF/gTsEG+NmNmPfAZjll5KwDX5a4xAQc52ZiV5zMcszaQdBwLXs/5lX9x1jqZE46ZmdXCd6mZmVktnHDMzKwWTjhmZlYLJxwzM6vF/wH1ILqzpaPMyQAAAABJRU5ErkJggg==\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 0 Axes>"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "plt.bar(range(5,38,2), np.reshape(results_3, (-1, 5)).mean(1))\n",
        "plt.title(\"success rate based on levels of complexity (3 goals)\")\n",
        "plt.xlabel(\"dim of grid_wrold\")\n",
        "plt.ylabel(\"success rate\");"
      ],
      "metadata": {
        "id": "FNujD7wNnP7d"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "plt.bar(range(5,38,2), np.reshape(results_3_2, (-1, 5)).mean(1))\n",
        "plt.title(\"success rate based on levels of complexity (3 ordered goals)\")\n",
        "plt.xlabel(\"dim of grid_wrold\")\n",
        "plt.ylabel(\"success rate\");"
      ],
      "metadata": {
        "id": "OtKilKVxmQGU"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# np.reshape(results, (-1, 10)).mean(1)\n",
        "# plt.plot(results)\n",
        "fin_res = np.ones(30)\n",
        "fin_res[5:25] = res.mean(1)\n",
        "fin_res[25:] = np.reshape(results, (-1, 10)).mean(1)\n",
        "fin_res"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_xyTx_GCj82H",
        "outputId": "113247c9-36d2-4f69-d05b-3cc679f7c469"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([1. , 1. , 1. , 1. , 1. , 1. , 1. , 0.9, 1. , 1. , 0.9, 1. , 1. ,\n",
              "       1. , 1. , 0.8, 0.9, 0.7, 0.6, 0.9, 0.7, 1. , 0.5, 0.7, 0.7, 1. ,\n",
              "       1. , 1. , 1. , 1. ])"
            ]
          },
          "metadata": {},
          "execution_count": 55
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# res = np.reshape(results, (20, 10))\n",
        "res.mean(1)\n",
        "plt.bar(range(5,25), res.mean(1))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 282
        },
        "id": "ovghDaJciyhc",
        "outputId": "43d77cd4-36cd-425d-ed9f-99a615eb77bb"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<BarContainer object of 20 artists>"
            ]
          },
          "metadata": {},
          "execution_count": 50
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXkAAAD4CAYAAAAJmJb0AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAPP0lEQVR4nO3dfYxld13H8feHXaoJVCjuiHUf2KoLcX2kmZQqiE2odbuaXR8I6UalQGVDwhoIqFmDqaT+YyFigqngKg0PwZaCghtdsiDWkBjbdAtt6baUTtdidy3tArVoiJbVr3/cs+YyzJ05M3vvzJ0f71dyM+fhd+d8eu/ZT889d+65qSokSW162loHkCRNjiUvSQ2z5CWpYZa8JDXMkpekhm1cqw1v2rSptm/fvlabl6R16c477/xyVc30Hb9mJb99+3aOHTu2VpuXpHUpyReXM97TNZLUMEtekhpmyUtSwyx5SWqYJS9JDbPkJalhS5Z8khuTPJ7k3hHrk+SdSeaS3JPk4vHHlCStRJ8j+fcCuxZZfyWwo7vtB9517rEkSeOwZMlX1aeBry4yZC/w/hq4DXh2kgvHFVCStHLj+MTrZuCRofmT3bJH5w9Msp/B0T7btm1b8Qa3H/y7Zd/n4T/8+RXff/i+52K95j7Xba/mfce5ba2uc32utbBVfeO1qg5V1WxVzc7M9L70giRphcZR8qeArUPzW7plkqQ1No6SPwy8svsrm0uBJ6vqW07VSJJW35Ln5JPcBFwGbEpyEvh94OkAVfVu4AiwG5gDvg68elJhJUnLs2TJV9W+JdYX8PqxJZIkjY2feJWkhlnyktQwS16SGmbJS1LDLHlJapglL0kNs+QlqWGWvCQ1zJKXpIZZ8pLUMEtekhpmyUtSwyx5SWqYJS9JDRvHd7x+W1mv3xm6XnOvV+v1+0rXa26N5pG8JDXMkpekhlnyktQwS16SGmbJS1LDLHlJapglL0kNs+QlqWGWvCQ1zJKXpIZZ8pLUMEtekhpmyUtSwyx5SWqYJS9JDbPkJalhlrwkNaxXySfZleSBJHNJDi6wfluSW5N8Nsk9SXaPP6okabmWLPkkG4AbgCuBncC+JDvnDfs94JaqeiFwFfCn4w4qSVq+PkfylwBzVXWiqp4Cbgb2zhtTwHd1088C/m18ESVJK9Xni7w3A48MzZ8EXjRvzFuBTyT5TeAZwOUL/aIk+4H9ANu2bVtuVmlV+eXnasG43njdB7y3qrYAu4EPJPmW311Vh6pqtqpmZ2ZmxrRpSdIofUr+FLB1aH5Lt2zYNcAtAFX1z8B3ApvGEVCStHJ9Sv4OYEeSi5Kcx+CN1cPzxvwr8DKAJD/EoORPjzOoJGn5liz5qjoDHACOAvcz+Cua40muS7KnG/Zm4LVJ7gZuAl5VVTWp0JKkfvq88UpVHQGOzFt27dD0fcCLxxtNknSu/MSrJDXMkpekhlnyktQwS16SGmbJS1LDLHlJapglL0kNs+QlqWGWvCQ1zJKXpIZZ8pLUMEtekhpmyUtSwyx5SWqYJS9JDet1PXlJq2u9fon4t2Pu5d53/v0nzSN5SWqYJS9JDbPkJalhlrwkNcySl6SGWfKS1DBLXpIaZslLUsMseUlqmCUvSQ2z5CWpYZa8JDXMkpekhlnyktQwS16SGmbJS1LDepV8kl1JHkgyl+TgiDGvSHJfkuNJ/nK8MSVJK7HkN0Ml2QDcAPwscBK4I8nhqrpvaMwO4HeBF1fVE0m+Z1KBJUn99TmSvwSYq6oTVfUUcDOwd96Y1wI3VNUTAFX1+HhjSpJWok/JbwYeGZo/2S0b9nzg+Un+KcltSXaNK6AkaeXG9UXeG4EdwGXAFuDTSX60qv59eFCS/cB+gG3bto1p05KGrdcv09Zk9DmSPwVsHZrf0i0bdhI4XFXfqKp/Ab7AoPS/SVUdqqrZqpqdmZlZaWZJUk99Sv4OYEeSi5KcB1wFHJ435mMMjuJJsonB6ZsTY8wpSVqBJUu+qs4AB4CjwP3ALVV1PMl1SfZ0w44CX0lyH3Ar8NtV9ZVJhZYk9dPrnHxVHQGOzFt27dB0AW/qbpKkKeEnXiWpYZa8JDXMkpekhlnyktQwS16SGmbJS1LDLHlJapglL0kNs+QlqWGWvCQ1zJKXpIZZ8pLUMEtekhpmyUtSwyx5SWrYuL7jVZLWlN9tuzCP5CWpYZa8JDXMkpekhlnyktQwS16SGmbJS1LDLHlJapglL0kNs+QlqWGWvCQ1zJKXpIZZ8pLUMEtekhpmyUtSwyx5SWqYJS9JDbPkJalhvUo+ya4kDySZS3JwkXG/kqSSzI4voiRppZYs+SQbgBuAK4GdwL4kOxcYdz7wBuD2cYeUJK1MnyP5S4C5qjpRVU8BNwN7Fxj3B8D1wH+NMZ8k6Rz0KfnNwCND8ye7Zf8vycXA1qpa9Jt0k+xPcizJsdOnTy87rCRpec75jdckTwPeAbx5qbFVdaiqZqtqdmZm5lw3LUlaQp+SPwVsHZrf0i0763zgR4B/TPIwcClw2DdfJWnt9Sn5O4AdSS5Kch5wFXD47MqqerKqNlXV9qraDtwG7KmqYxNJLEnqbcmSr6ozwAHgKHA/cEtVHU9yXZI9kw4oSVq5jX0GVdUR4Mi8ZdeOGHvZuceSJI2Dn3iVpIZZ8pLUMEtekhpmyUtSwyx5SWqYJS9JDbPkJalhlrwkNcySl6SGWfKS1DBLXpIaZslLUsMseUlqmCUvSQ2z5CWpYZa8JDXMkpekhlnyktQwS16SGmbJS1LDLHlJapglL0kNs+QlqWGWvCQ1zJKXpIZZ8pLUMEtekhpmyUtSwyx5SWqYJS9JDbPkJalhlrwkNaxXySfZleSBJHNJDi6w/k1J7ktyT5JPJXne+KNKkpZryZJPsgG4AbgS2AnsS7Jz3rDPArNV9WPAR4C3jTuoJGn5+hzJXwLMVdWJqnoKuBnYOzygqm6tqq93s7cBW8YbU5K0En1KfjPwyND8yW7ZKNcAH19oRZL9SY4lOXb69On+KSVJKzLWN16T/BowC7x9ofVVdaiqZqtqdmZmZpybliQtYGOPMaeArUPzW7pl3yTJ5cBbgJ+pqv8eTzxJ0rnocyR/B7AjyUVJzgOuAg4PD0jyQuDPgD1V9fj4Y0qSVmLJkq+qM8AB4ChwP3BLVR1Pcl2SPd2wtwPPBD6c5K4kh0f8OknSKupzuoaqOgIcmbfs2qHpy8ecS5I0Bn7iVZIaZslLUsMseUlqmCUvSQ2z5CWpYZa8JDXMkpekhlnyktQwS16SGmbJS1LDLHlJapglL0kNs+QlqWGWvCQ1zJKXpIZZ8pLUMEtekhpmyUtSwyx5SWqYJS9JDbPkJalhlrwkNcySl6SGWfKS1DBLXpIaZslLUsMseUlqmCUvSQ2z5CWpYZa8JDXMkpekhlnyktQwS16SGtar5JPsSvJAkrkkBxdY/x1JPtStvz3J9nEHlSQt35Iln2QDcANwJbAT2Jdk57xh1wBPVNUPAn8MXD/uoJKk5etzJH8JMFdVJ6rqKeBmYO+8MXuB93XTHwFeliTjiylJWolU1eIDkpcDu6rqN7r5XwdeVFUHhsbc24052c0/1I358rzftR/Y382+AHhgXP8hnU3Al5cctTamNdu05oLpzTatuWB6s5lr+UZle15VzfT9JRvHl2dpVXUIODSp35/kWFXNTur3n4tpzTatuWB6s01rLpjebOZavnFl63O65hSwdWh+S7dswTFJNgLPAr5yruEkSeemT8nfAexIclGS84CrgMPzxhwGru6mXw78Qy11HkiSNHFLnq6pqjNJDgBHgQ3AjVV1PMl1wLGqOgy8B/hAkjngqwz+R7AWJnYqaAymNdu05oLpzTatuWB6s5lr+caSbck3XiVJ65efeJWkhlnyktSwdVnySR5O8rkkdyU5tsD6JHlnd5mFe5JcvEq5XtBlOnv7WpI3zhtzWZInh8ZcO6EsNyZ5vPsMw9llz0nyySQPdj8vGHHfq7sxDya5eqExE8j29iSf756vjyZ59oj7LvrcTyDXW5OcGnq+do+476KX/phQtg8N5Xo4yV0j7jvJx2xrkluT3JfkeJI3dMvXdF9bJNc07Gejsk1mX6uqdXcDHgY2LbJ+N/BxIMClwO1rkHED8CUGH1wYXn4Z8LersP2XAhcD9w4textwsJs+CFy/wP2eA5zofl7QTV+wCtmuADZ209cvlK3Pcz+BXG8FfqvHc/0Q8P3AecDdwM5JZ5u3/o+Aa9fgMbsQuLibPh/4AoPLn6zpvrZIrmnYz0Zlm8i+ti6P5HvYC7y/Bm4Dnp3kwlXO8DLgoar64ipvF4Cq+jSDv3QaNnz5ifcBv7jAXX8O+GRVfbWqngA+CeyadLaq+kRVnelmb2PweYxVNeIx66PPpT8mli1JgFcAN41zm31U1aNV9Zlu+j+A+4HNrPG+NirXlOxnox6zPpa9r63Xki/gE0nuzOBSCfNtBh4Zmj9J/wdxXK5i9D+6n0xyd5KPJ/nhVcz03Kp6tJv+EvDcBcZMw2P3GgavxBay1HM/CQe6l/c3jjjtsNaP2U8Dj1XVgyPWr8pjlsHVZ18I3M4U7Wvzcg1b8/1sgWxj39fWa8m/pKouZnBlzNcneelaBxqWwYfG9gAfXmD1Zxicwvlx4E+Aj61mtrNq8Npv6v5+NslbgDPAB0cMWe3n/l3ADwA/ATzK4LTItNnH4kfxE3/MkjwT+CvgjVX1teF1a7mvjco1DfvZAtkmsq+ty5KvqlPdz8eBjzJ4CTOsz6UYJulK4DNV9dj8FVX1tar6z276CPD0JJtWKddjZ09bdT8fX2DMmj12SV4F/ALwq10xfIsez/1YVdVjVfU/VfW/wJ+P2N5aPmYbgV8GPjRqzKQfsyRPZ1BWH6yqv+4Wr/m+NiLXVOxnC2Wb1L627ko+yTOSnH92msEbKffOG3YYeGUGLgWeHHrpuBpGHlkl+d7uHCpJLmHwHKzWdX6GLz9xNfA3C4w5ClyR5ILu5eIV3bKJSrIL+B1gT1V9fcSYPs/9uHMNv5fzSyO21+fSH5NyOfD56q4AO9+kH7NuX34PcH9VvWNo1Zrua6NyTcN+tki2yexrk3j3eJI3Bu8q393djgNv6Za/DnhdNx0GX3TyEPA5YHYV8z2DQWk/a2jZcLYDXe67Gbzx81MTynETg5d832Bw3u4a4LuBTwEPAn8PPKcbOwv8xdB9XwPMdbdXr1K2OQbnGu/qbu/uxn4fcGSx537CuT7Q7UP3dP+YLpyfq5vfzeCvJB4ad65R2brl7z27bw2NXc3H7CUMTsXcM/Tc7V7rfW2RXNOwn43KNpF9zcsaSFLD1t3pGklSf5a8JDXMkpekhlnyktQwS16SGmbJS1LDLHlJatj/ATE0CnT9sSX3AAAAAElFTkSuQmCC\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# grid_world = reset_grid_world(21)\n",
        "grid_world, hash_state(grid_world)"
      ],
      "metadata": {
        "id": "iS0XOCEJC4sw"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "for i in N:\n",
        "  print(i, N[i])"
      ],
      "metadata": {
        "id": "Es7zxraVT0Hx"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "t = np.array([0,0,0,0,0])\n",
        "model(t.reshape(1, -1))[0][0].numpy().argmax()"
      ],
      "metadata": {
        "id": "8E_2jlHesbGG"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "grid_world= reset_grid_world()\n",
        "state_history, action_history, reward, possible_policy = run_episode(grid_world.copy(), model, eps=0.1)"
      ],
      "metadata": {
        "id": "sXb0afjjpSf5"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def show_movement(states):\n",
        "  grid_world = reset_grid_world()\n",
        "  goal_location = get_locations(grid_world, 2)\n",
        "  obstacle_locations = get_locations(grid_world, -1)\n",
        "  for state in states:\n",
        "    i, j = int(state[0]//5), int(state[0]%5)\n",
        "    grid_world[i,j] = 1\n",
        "    print(grid_world)\n",
        "    grid_world[i][j] = 0\n",
        "    print()\n",
        "\n",
        "print(model(np.reshape([0, -1, -1, -1, 0], (1, -1)))[0].numpy(), model(np.reshape([0, -1, -1, -1, 0], (1, -1)))[0].numpy().argmax())"
      ],
      "metadata": {
        "id": "YMGbeUdipSi_"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "show_movement(state_history)"
      ],
      "metadata": {
        "id": "oEj1GL9Q06fb"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# PROBLEMS:\n",
        "\n",
        "# infinite loops\n",
        "# too long tragectories, agent can't find the way\n",
        "# tree search helps it NAVIGATE, but it usually gets stuck\n",
        "\n",
        "# effect of positive/negative rewards on the path taken\n",
        "    # use value functions?  \n",
        "    # intermediate rewards?\n",
        "    # deep MCTS?\n",
        "\n",
        "# * turn this into a simple baseline to try out different methods (ranked rewards,...)"
      ],
      "metadata": {
        "id": "6Nl-a2OM1KLK"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# 1 goal\n",
        "results_1=[\n",
        "True, True, True, True, True, \n",
        "True, True, True, True, True, \n",
        "True, True, True, True, True, \n",
        "True, True, True, True, True, \n",
        "True, False, True, True, True, \n",
        "False, True, True, True, True, \n",
        "False, True, True, True, True, \n",
        "False, True, True, True, True, \n",
        "False, True, True, False, True, \n",
        "False, True, True, True, True, \n",
        "True, False, True, True, False, \n",
        "True, False, True, True, False, \n",
        "True, True, True, True, True, \n",
        "True, True, True, True, True, \n",
        "True, False, False, False, True, \n",
        "True, True, True, False, True, \n",
        "False, True, True, False, False]\n",
        "\n",
        "# 2 goals\n",
        "results_2=[\n",
        "True, True, True, True, True, \n",
        "True, True, True, True, True, \n",
        "True, True, True, True, True, \n",
        "True, True, True, True, True, \n",
        "True, False, True, True, False, \n",
        "True, True, False, False, True, \n",
        "True, True, True, True, True, \n",
        "False, True, True, True, False, \n",
        "True, False, True, False, True, \n",
        "True, True, True, True, True, \n",
        "False, True, False, True, False, \n",
        "False, False, True, False, True, \n",
        "False, False, False, True, True, \n",
        "False, False, True, False, False, \n",
        "True, True, False, True, False, \n",
        "False, False, False, False, True,\n",
        "False, False, False, False, False]\n",
        "\n",
        "# 3 goals\n",
        "results_3=[\n",
        "True, True, True, True, True, \n",
        "True, True, True, True, True, \n",
        "True, True, True, True, True, \n",
        "True, True, True, False, True, \n",
        "True, False, True, True, True, \n",
        "True, True, False, False, True, \n",
        "True, True, True, True, True, \n",
        "False, True, True, True, False, \n",
        "True, False, False, True, True, \n",
        "True, False, False, False, True, \n",
        "True, True, True, True, False, \n",
        "False, False, True, False, True, \n",
        "False, False, False, True, True, \n",
        "False, True, False, False, False, \n",
        "False, False, True, True, False, \n",
        "False, False, False, True, False,\n",
        "False, False, False, False, True]\n",
        "\n",
        "# ordered goals \n",
        "results_3_2=[\n",
        "True, False, False, True, True, \n",
        "True, True, True, True, True, \n",
        "True, False, False, True, True, \n",
        "True, True, False, False, True, \n",
        "True, False, True, True, True, \n",
        "False, True, False, False, True, \n",
        "False, False, False, True, True, \n",
        "False, True, True, True, False, \n",
        "True, False, False, False, True, \n",
        "False, False, False, False, True, \n",
        "False, True, False, True, False, \n",
        "False, False, False, True, False, \n",
        "True, False, False, False, False, \n",
        "False, False, False, False, False, \n",
        "True, False, False, False, False, \n",
        "False, False, True, False, False,\n",
        "False, False, False, False, False]"
      ],
      "metadata": {
        "id": "5zukkkTytFdv"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# stay at end goal results_1 \n",
        "array([[ True,  True,  True,  True,  True],\n",
        "        [ True,  True,  True,  True,  True],\n",
        "        [ True,  True,  True,  True,  True],\n",
        "        [ True,  True,  True,  True,  True],\n",
        "        [ True,  True,  True,  True,  True],\n",
        "        [ True,  True,  True,  True,  True],\n",
        "        [ True,  True,  True,  True,  True],\n",
        "        [ True,  True,  True,  True,  True],\n",
        "        [ True,  True,  True,  True, False],\n",
        "        [ True,  True,  True, False,  True],\n",
        "        [ True, False,  True, False,  True]]), [True, True])\n",
        "\n",
        "# stay at end goals (2)\n",
        "(array([[ True,  True, False,  True, False],\n",
        "        [False,  True,  True,  True,  True],\n",
        "        [ True,  True,  True,  True,  True],\n",
        "        [False,  True, False,  True, False],\n",
        "        [ True, False, False, False, False],\n",
        "        [False,  True, False, False, False],\n",
        "        [False, False,  True,  True, False],\n",
        "        [False, False, False,  True, False],\n",
        "        [False, False,  True, False,  True],\n",
        "        [False, False, False, False,  True]]),\n",
        " [False, False, False])\n",
        "\n",
        "# stay at end goals (3)\n",
        "array([[False, False, False,  True, False],\n",
        "        [ True,  True, False, False, False],\n",
        "        [False,  True, False,  True, False],\n",
        "        [False, False, False, False, False],\n",
        "        [False, False,  True, False, False],\n",
        "        [False, False,  True,  True, False],\n",
        "        [False, False, False, False, False],\n",
        "        [False, False, False, False, False],\n",
        "        [False, False, False, False, False]]),"
      ],
      "metadata": {
        "id": "78b5LyLohd-F"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "eX7FFYtKiEPA"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [],
      "metadata": {
        "id": "HoXIEAQFcze_"
      }
    }
  ]
}