Softmax_and_Q_learning_V2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "executionInfo": {
     "elapsed": 636,
     "status": "ok",
     "timestamp": 1624275244016,
     "user": {
      "displayName": "Avi Alok",
      "photoUrl": "",
      "userId": "13579507839502371538"
     },
     "user_tz": -330
    },
    "id": "0t0qCEamjPos"
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import random\n",
    "import matplotlib.pyplot as plt\n",
    "import random"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "u81bVkwSjPo4"
   },
   "source": [
    "Definition of environment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "executionInfo": {
     "elapsed": 12,
     "status": "ok",
     "timestamp": 1624275244016,
     "user": {
      "displayName": "Avi Alok",
      "photoUrl": "",
      "userId": "13579507839502371538"
     },
     "user_tz": -330
    },
    "id": "8QcA6P-cjPo8"
   },
   "outputs": [],
   "source": [
    "env=[0,1,2,3,4,5,6,7]\n",
    "dist1=[0,1,1.848,2.414,2.613,2.414,1.848,1]\n",
    "k=2.2\n",
    "dist=[i/k for i in dist1]\n",
    "total_score=[]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "8ARWjCd-jPpA"
   },
   "source": [
    "Actions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "executionInfo": {
     "elapsed": 9,
     "status": "ok",
     "timestamp": 1624275244016,
     "user": {
      "displayName": "Avi Alok",
      "photoUrl": "",
      "userId": "13579507839502371538"
     },
     "user_tz": -330
    },
    "id": "oE_QvxpWjPpA"
   },
   "outputs": [],
   "source": [
    "actions=[0,1,2,3,4,5,6,7]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "QU-APqnvjPpD"
   },
   "source": [
    "Rewards"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "executionInfo": {
     "elapsed": 9,
     "status": "ok",
     "timestamp": 1624275244017,
     "user": {
      "displayName": "Avi Alok",
      "photoUrl": "",
      "userId": "13579507839502371538"
     },
     "user_tz": -330
    },
    "id": "4SaD_1Z8jPpF"
   },
   "outputs": [],
   "source": [
    "rewards=[70,0,0,70,70,0,70,0]\n",
    "reward_update_rate = [8, 0, 0, 2, 2 , 0, 8, 0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "YBRX-QwJjPpH"
   },
   "source": [
    "Help Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "executionInfo": {
     "elapsed": 9,
     "status": "ok",
     "timestamp": 1624275244017,
     "user": {
      "displayName": "Avi Alok",
      "photoUrl": "",
      "userId": "13579507839502371538"
     },
     "user_tz": -330
    },
    "id": "mkmOXKBVjPpJ"
   },
   "outputs": [],
   "source": [
    "def get_starting_location():\n",
    "    current_place = np.random.randint(8)\n",
    "    return current_place\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "18f1-sLLhCDB"
   },
   "source": [
    "softmax action select"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "executionInfo": {
     "elapsed": 8,
     "status": "ok",
     "timestamp": 1624275244017,
     "user": {
      "displayName": "Avi Alok",
      "photoUrl": "",
      "userId": "13579507839502371538"
     },
     "user_tz": -330
    },
    "id": "E5K1AXjLhIT-"
   },
   "outputs": [],
   "source": [
    "def softmax(x,beta):\n",
    "\n",
    "    e_x = np.exp(beta*x - beta*np.max(x))\n",
    "    return e_x / e_x.sum()\n",
    "\n",
    "def get_next_action(current_env_index,beta):\n",
    "    A=softmax(Q[current_env_index],beta)\n",
    "    #print(A)\n",
    "    numberList = [0,1,2,3,4,5,6,7]\n",
    "    B=random.choices(numberList, weights=A, k=1)\n",
    "    #print(B)\n",
    "    return int(B[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "executionInfo": {
     "elapsed": 9,
     "status": "ok",
     "timestamp": 1624275244018,
     "user": {
      "displayName": "Avi Alok",
      "photoUrl": "",
      "userId": "13579507839502371538"
     },
     "user_tz": -330
    },
    "id": "U6tq630pjPpP"
   },
   "outputs": [],
   "source": [
    "def get_next_location(current_env_index,action_index):\n",
    "    new_env_index = (current_env_index+action_index)%8\n",
    "    index_difference=action_index\n",
    "    return new_env_index,index_difference"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "executionInfo": {
     "elapsed": 8,
     "status": "ok",
     "timestamp": 1624275244018,
     "user": {
      "displayName": "Avi Alok",
      "photoUrl": "",
      "userId": "13579507839502371538"
     },
     "user_tz": -330
    },
    "id": "RyNqLbF3jPpS"
   },
   "outputs": [],
   "source": [
    "def mesaure_and_update_rewards(current_env_index,time,index_difference,rewards):\n",
    "    time=time+1+dist[index_difference]\n",
    "    re=rewards[current_env_index]\n",
    "    for i in range(len(rewards)):\n",
    "        if(i!=current_env_index):\n",
    "            if(reward_update_rate[i]+rewards[i]>200):\n",
    "                rewards[i]=200\n",
    "            else:   \n",
    "                rewards[i]=reward_update_rate[i]+rewards[i]\n",
    "        else:\n",
    "            rewards[i]=0.9*rewards[i]\n",
    "    \n",
    "    #print(rewards)\n",
    "    return time,re,rewards"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "executionInfo": {
     "elapsed": 9,
     "status": "ok",
     "timestamp": 1624275244019,
     "user": {
      "displayName": "Avi Alok",
      "photoUrl": "",
      "userId": "13579507839502371538"
     },
     "user_tz": -330
    },
    "id": "_yJg2rRgjPpU"
   },
   "outputs": [],
   "source": [
    "def updateQtable(timetaken,prev_env_index,action_index,reward,avg_reward,beta):\n",
    "    da=0\n",
    "    current_patch=(prev_env_index+action_index)%8\n",
    "    next_action=get_next_action(current_patch,beta)\n",
    "    Q[prev_env_index,action_index]=(1-alpha)*Q[prev_env_index,action_index] + alpha*(reward+gamma*Q[current_patch, next_action])\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "executionInfo": {
     "elapsed": 9,
     "status": "ok",
     "timestamp": 1624275244019,
     "user": {
      "displayName": "Avi Alok",
      "photoUrl": "",
      "userId": "13579507839502371538"
     },
     "user_tz": -330
    },
    "id": "U2qS8fzVjPpU"
   },
   "outputs": [],
   "source": [
    "epsilon=1\n",
    "gamma=0.9\n",
    "alpha=0.02\n",
    "Q=np.zeros(shape=(8,8))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "0ZzZdsqOH8iI"
   },
   "source": [
    "TRAINING STEPS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "executionInfo": {
     "elapsed": 13169,
     "status": "ok",
     "timestamp": 1624275258534,
     "user": {
      "displayName": "Avi Alok",
      "photoUrl": "",
      "userId": "13579507839502371538"
     },
     "user_tz": -330
    },
    "id": "S327DduJjPpX"
   },
   "outputs": [],
   "source": [
    "for p in range(1):\n",
    "    beta=2\n",
    "    epsilon=0.3\n",
    "    gamma=0.9\n",
    "    alpha=0.8\n",
    "    Q=np.zeros(shape=(8,8))\n",
    "    TRAINING_EPISODES=1000\n",
    "    for i in range(TRAINING_EPISODES):\n",
    "        rewards=[70,0,0,70,70,0,70,0]\n",
    "        time=0\n",
    "        current_patch=get_starting_location()\n",
    "        cums=0\n",
    "        avg_reward=0\n",
    "        global_avg_reward=[]\n",
    "        D=[]\n",
    "        while(time<300):\n",
    "            #next_action,epsilon=get_next_action(current_patch,time,epsilon)\n",
    "            next_action=get_next_action(current_patch,beta)\n",
    "            #print(current_patch)\n",
    "            current_patch,indexdif=get_next_location(current_patch,next_action)\n",
    "            D.append(current_patch)\n",
    "            time1,re,rewards=mesaure_and_update_rewards(current_patch,time,indexdif,rewards)\n",
    "            cums=cums+re\n",
    "            updateQtable(time1-time,current_patch,next_action,re,avg_reward,beta)\n",
    "            time=time1\n",
    "            avg_reward=cums/time\n",
    "            global_avg_reward.append(avg_reward)\n",
    "        #Print(cums)\n",
    "    #plt.plot(global_avg_reward)\n",
    "    #plt.ylabel(\"global_avg_reward\")\n",
    "    #plt.xlabel(\"time\")\n",
    "    \n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "executionInfo": {
     "elapsed": 18,
     "status": "ok",
     "timestamp": 1624275258535,
     "user": {
      "displayName": "Avi Alok",
      "photoUrl": "",
      "userId": "13579507839502371538"
     },
     "user_tz": -330
    },
    "id": "m5COk2uC_ysv"
   },
   "outputs": [],
   "source": [
    "def final_episode():\n",
    "  \n",
    "  rewards=[70,0,0,70,70,0,70,0]\n",
    "  time=0\n",
    "  current_patch=get_starting_location()\n",
    "  cums=0\n",
    "  avg_reward=0\n",
    "  global_avg_reward=[]\n",
    "  Step_List=[]\n",
    "  A_steps=[0,0,0,0,0,0,0,0]\n",
    "  A_exploit=[0,0,0,0,0,0,0,0]\n",
    "  while(time<300):\n",
    "            #next_action,epsilon=get_next_action(current_patch,time,epsilon)\n",
    "            next_action=get_next_action(current_patch,beta)\n",
    "            #print(current_patch)\n",
    "            \n",
    "            Step_List.append(current_patch)\n",
    "            time1,re,rewards=mesaure_and_update_rewards(current_patch,time,indexdif,rewards)\n",
    "            for j in range(8):\n",
    "              if(current_patch==j):\n",
    "                A_steps[j]+=1\n",
    "                A_exploit[j]+=re\n",
    "            cums=cums+re\n",
    "            updateQtable(time1-time,current_patch,next_action,re,avg_reward,beta)\n",
    "            current_patch,indexdif=get_next_location(current_patch,next_action)\n",
    "            time=time1\n",
    "            avg_reward=cums/time\n",
    "            global_avg_reward.append(avg_reward)\n",
    "            print(rewards)\n",
    "            print(current_patch)\n",
    "  return Step_List,A_steps,A_exploit"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Mwjtdg3nIA0r"
   },
   "source": [
    "FINAL EPISODE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "BjEeiHpvEoS6"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "Softmax_and_Q_learning_V2.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}