02_minerl_navigate_agent.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#default_exp agents.navigate"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Basic MineRL Agent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "\n",
    "import time\n",
    "\n",
    "import gym\n",
    "import minerl  # noqa\n",
    "\n",
    "# OpenCog\n",
    "from opencog.atomspace import *\n",
    "from opencog.logger import log\n",
    "from opencog.pln import *\n",
    "from opencog.type_constructors import *\n",
    "from opencog.utilities import set_default_atomspace\n",
    "\n",
    "# ROCCA\n",
    "from rocca.envs.wrappers import MineRLWrapper\n",
    "from rocca.envs.wrappers.utils import minerl_single_action\n",
    "from rocca.agents import OpencogAgent\n",
    "from rocca.agents.utils import *"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Agent definition"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Below is pretty much `OpencogAgent` but with the `step` method fixed to use the `minerl_single_action` helper."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "\n",
    "class NavigateAgent(OpencogAgent):\n",
    "    def __init__(self, env, action_space, p_goal, n_goal, log_level=\"info\"):\n",
    "        OpencogAgent.__init__(self, env, action_space, p_goal, n_goal, log_level)\n",
    "\n",
    "    def step(self):\n",
    "        \"\"\"Run one step of observation, decision and env update\n",
    "        \"\"\"\n",
    "        agent_log.debug(\"atomese_obs = {}\".format(self.observation))\n",
    "        obs_record = [self.record(o, self.step_count, tv=TRUE_TV)\n",
    "                      for o in self.observation]\n",
    "        agent_log.debug(\"obs_record = {}\".format(obs_record))\n",
    "\n",
    "        # Make the goal for that iteration\n",
    "        goal = self.make_goal()\n",
    "        agent_log.debug(\"goal = {}\".format(goal))\n",
    "\n",
    "        # Plan, i.e. come up with cognitive schematics as plans.  Here the\n",
    "        # goal expiry is 2, i.e. must be fulfilled set for the next two iterations.\n",
    "        cogscms = self.plan(goal, self.expiry)\n",
    "        agent_log.debug(\"cogscms = {}\".format(cogscms))\n",
    "\n",
    "        # Deduce the action distribution\n",
    "        mxmdl = self.deduce(cogscms)\n",
    "        agent_log.debug(\"mxmdl = {}\".format(mxmdl_to_str(mxmdl)))\n",
    "\n",
    "        # Select the next action\n",
    "        action, pblty = self.decide(mxmdl)\n",
    "        agent_log.debug(\"action with probability of success = {}\".format(act_pblt_to_str((action, pblty))))\n",
    "\n",
    "        # Timestamp the action that is about to be executed\n",
    "        action_record = self.record(action, self.step_count, tv=TRUE_TV)\n",
    "        agent_log.debug(\"action_record = {}\".format(action_record))\n",
    "        agent_log.debug(\"action = {}\".format(action))\n",
    "\n",
    "        # Increment the counter for that action and log it\n",
    "        self.action_counter[action] += 1\n",
    "        agent_log.debug(\"action_counter = {}\".format(self.action_counter))\n",
    "\n",
    "        # Increase the step count and run the next step of the environment\n",
    "        self.step_count += 1\n",
    "        # TODO gather environment info.\n",
    "        reward, self.observation, done = self.env.step(minerl_single_action(self.env, action))\n",
    "        \n",
    "        self.accumulated_reward += float(reward.out[1].name)\n",
    "        agent_log.debug(\"observation = {}\".format(self.observation))\n",
    "        agent_log.debug(\"reward = {}\".format(reward))\n",
    "        agent_log.debug(\"accumulated reward = {}\".format(self.accumulated_reward))\n",
    "\n",
    "        reward_record = self.record(reward, self.step_count, tv=TRUE_TV)\n",
    "        agent_log.debug(\"reward_record = {}\".format(reward_record))\n",
    "\n",
    "        if done:\n",
    "            return False\n",
    "\n",
    "        return True"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Experiment\n",
    "\n",
    "For this experiment we use the basic _MineRLNavigateDense-v0_ environment of MineRL. The task of the agent is to touch a diamond block placed somewhere in the vicinity. This is a _Dense_ version of the environment, which means the agent receives rewards all the time, based on whether the distance to the target decreases or increases."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "env = gym.make('MineRLNavigateDense-v0')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "env.action_space"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "env.observation_space"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "atomspace = AtomSpace()\n",
    "set_default_atomspace(atomspace)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "wrapped_env = MineRLWrapper(env)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create Goal\n",
    "pgoal = EvaluationLink(PredicateNode(\"Reward\"), NumberNode(\"100\"))\n",
    "ngoal = EvaluationLink(PredicateNode(\"Reward\"), NumberNode(\"0\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "action_space = {\n",
    "    ExecutionLink(SchemaNode(\"attack\"), NumberNode(\"0\")),\n",
    "    ExecutionLink(SchemaNode(\"attack\"), NumberNode(\"1\")),\n",
    "    ExecutionLink(SchemaNode(\"forward\"), NumberNode(\"0\")),\n",
    "    ExecutionLink(SchemaNode(\"forward\"), NumberNode(\"1\")),\n",
    "    ExecutionLink(SchemaNode(\"back\"), NumberNode(\"0\")),\n",
    "    ExecutionLink(SchemaNode(\"back\"), NumberNode(\"1\")),\n",
    "    ExecutionLink(SchemaNode(\"left\"), NumberNode(\"0\")),\n",
    "    ExecutionLink(SchemaNode(\"left\"), NumberNode(\"1\")),\n",
    "    ExecutionLink(SchemaNode(\"right\"), NumberNode(\"0\")),\n",
    "    ExecutionLink(SchemaNode(\"right\"), NumberNode(\"1\")),\n",
    "    ExecutionLink(SchemaNode(\"jump\"), NumberNode(\"0\")),\n",
    "    ExecutionLink(SchemaNode(\"jump\"), NumberNode(\"1\")),\n",
    "    ExecutionLink(SchemaNode(\"sprint\"), NumberNode(\"0\")),\n",
    "    ExecutionLink(SchemaNode(\"sprint\"), NumberNode(\"1\")),\n",
    "    ExecutionLink(SchemaNode(\"sneak\"), NumberNode(\"0\")),\n",
    "    ExecutionLink(SchemaNode(\"sneak\"), NumberNode(\"1\")),\n",
    "    ExecutionLink(SchemaNode(\"place\"), ConceptNode(\"dirt\")),\n",
    "    ExecutionLink(SchemaNode(\"place\"), ConceptNode(\"none\")),\n",
    "    ExecutionLink(SchemaNode(\"camera\"), ListLink(NumberNode(\"2.5\"), NumberNode(\"0.0\"))),\n",
    "    ExecutionLink(SchemaNode(\"camera\"), ListLink(NumberNode(\"0.0\"), NumberNode(\"-1.5\"))),\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "agent = NavigateAgent(wrapped_env, action_space, pgoal, ngoal)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lt_iterations = 1  # Number of learning-training iterations\n",
    "lt_period = 200  # Duration of a learning-training iteration\n",
    "\n",
    "for i in range(lt_iterations):\n",
    "    agent.reset_action_counter()\n",
    "    par = agent.accumulated_reward  # Keep track of the reward before\n",
    "    # Discover patterns to make more informed decisions\n",
    "    agent_log.info(\"Start learning ({}/{})\".format(i + 1, lt_iterations))\n",
    "    print(f\"Learning... ({i+1}/{lt_iterations})\")\n",
    "    agent.learn()\n",
    "    # Run agent to accumulate percepta\n",
    "    agent_log.info(\"Start training ({}/{})\".format(i + 1, lt_iterations))\n",
    "    print(f\"Running... ({i+1}/{lt_iterations})\")\n",
    "    for j in range(lt_period):\n",
    "        agent.step()\n",
    "        time.sleep(0.01)\n",
    "        # log.info(\"step_count = {}\".format(agent.step_count))\n",
    "    nar = agent.accumulated_reward - par\n",
    "    agent_log.info(\"Accumulated reward during {}th iteration = {}\".format(i + 1, nar))\n",
    "    print(f\"Reward for {i+1} iteration = {nar}.\")\n",
    "    agent_log.info(\"Action counter during {}th iteration:\\n{}\".format(i + 1, agent.action_counter))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}