A Practitioner’s Guide to Reinforcement Learning.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "81e36249-ad29-4d52-ae7b-b4d9e46edce8",
   "metadata": {},
   "source": [
    "# A Practitioner’s Guide to Reinforcement Learning\n",
    "As described [here](https://medium.com/towards-data-science/a-practitioners-guide-to-reinforcement-learning-1f1e249f8fa5)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a96db23f-d453-42cf-a1b1-a7eefe6b3c1a",
   "metadata": {},
   "outputs": [],
   "source": [
    "from itertools import product\n",
    "import random\n",
    "\n",
    "from gymnasium.spaces.tuple import Tuple\n",
    "\n",
    "\n",
    "def space_to_tuples(space):\n",
    "    if isinstance(space, Tuple):\n",
    "        for encoding in product(*[range(factor.n) for factor in space]):\n",
    "            yield encoding\n",
    "    else:\n",
    "        for encoding in range(space.n):\n",
    "            yield encoding\n",
    "\n",
    "def get_best_action(q_table, state): # for a given state, find the best action from the Q table\n",
    "    return max(((action, value) for action, value in q_table[state].items()), key=lambda x: x[1])[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1bf34c04-6da6-4d53-95ea-8cd577c926d2",
   "metadata": {},
   "outputs": [],
   "source": [
    "import gymnasium as gym\n",
    "from PIL import Image\n",
    "\n",
    "env = gym.make(\"FrozenLake-v1\", render_mode=\"rgb_array\", is_slippery=False)\n",
    "# env = gym.make(\"Taxi-v3\", render_mode=\"rgb_array\")\n",
    "# env = gym.make('Blackjack-v1', natural=True, sab=False, render_mode=\"rgb_array\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "00ee27fd-985a-44d2-a949-72a98345ee00",
   "metadata": {},
   "outputs": [],
   "source": [
    "alpha = 0.1\n",
    "gamma = 0.9\n",
    "n_episodes = 100_000\n",
    "max_steps = 100\n",
    "epsilon = 0.2\n",
    "\n",
    "q_table = {i: {j: 0 for j in space_to_tuples(env.action_space)} for i in space_to_tuples(env.observation_space)}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fd7aa95c-4999-49b4-a612-15eb407a7079",
   "metadata": {},
   "outputs": [],
   "source": [
    "for _ in range(n_episodes):\n",
    "    # new episode (game), so we need a reset\n",
    "    state, _ = env.reset()\n",
    "    \n",
    "    # play the game for max_steps\n",
    "    for step in range(max_steps):\n",
    "        # pick an action\n",
    "        if random.random() < epsilon:\n",
    "            # sometimes it is random to encourage exploration, otherwise we cannot find good policies ...\n",
    "            action = env.action_space.sample()\n",
    "        else:\n",
    "            # ... and sometimes we use the best action according to our Q-table\n",
    "            action = get_best_action(q_table, state)\n",
    "        \n",
    "        # we take that action and then get some data from the game\n",
    "        next_state, reward, terminated, _, _ = env.step(action)\n",
    "        \n",
    "        # update the Q-table according to the magic formula\n",
    "        q_table[state][action] = (1-alpha)*q_table[state][action] + alpha*(reward + gamma*max(q_table[next_state].values()))\n",
    "\n",
    "        # check if the game is finished, can be if you win or lose\n",
    "        if terminated:\n",
    "            # a new episode starts after this\n",
    "            break\n",
    "        else:\n",
    "            # if the episode continues, update the current state (we took an action!)\n",
    "            state = next_state"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cb50fb09-975b-45ad-8187-e601ae372f4c",
   "metadata": {},
   "outputs": [],
   "source": [
    "def play_episode(env, q_table):\n",
    "    state, _ = env.reset()\n",
    "\n",
    "    while True:\n",
    "        action = get_best_action(q_table, state)\n",
    "        next_state, reward, terminated, _, _ = env.step(action)\n",
    "        \n",
    "        if terminated:\n",
    "            break\n",
    "        else:\n",
    "            state = next_state\n",
    "   \n",
    "    return reward\n",
    "\n",
    "rewards = []\n",
    "for episode in range(10000):\n",
    "    rewards.append(play_episode(env, q_table))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b1220ac8-9de8-4aaf-b25f-d96353086f19",
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "\n",
    "print(Counter(rewards))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d3d30605-e63d-4a3c-ba7b-14eece501efd",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}