Transformatics_v1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "32c7cf81-644e-4b06-9e9a-a4a6e8808458",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import random, shutil, pickle, sys\n",
    "import numpy as np\n",
    "from collections import deque\n",
    "from keras import Sequential\n",
    "from keras.optimizers import RMSprop\n",
    "from IPython.display import display\n",
    "from PIL import Image\n",
    "import pandas as pd\n",
    "from keras import Model\n",
    "import keras, os, glob\n",
    "import tensorflow as tf\n",
    "from keras.layers import Layer, Dense, Conv2D, Flatten, RepeatVector,Dropout, LayerNormalization, MultiHeadAttention, GlobalAveragePooling1D\n",
    "from keras.layers import Activation, LSTM, Bidirectional , Dropout\n",
    "from keras import layers\n",
    "# import cv2\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "import tensorflow as tf\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "import codecs\n",
    "import csv\n",
    "import secrets\n",
    "import sqlite3\n",
    "from IPython.display import display\n",
    "from IPython.display import clear_output\n",
    "import time\n",
    "from tabulate import tabulate\n",
    "import json\n",
    "import datetime\n",
    "from tensorflow.keras.optimizers import Adam"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "98d492ed-4faf-4aac-810e-4a1dcdb0ab51",
   "metadata": {},
   "source": [
    "Recommedations. add account balance on the chart the current trade its profit closing profit \n",
    "\n",
    "also add different consipicous icons for\n",
    "opening the trade == done\n",
    "holding the trade in profit == done\n",
    "holding the trade in loss == done\n",
    "show reversal arrows for a trade that was in profit and now it is in loss\n",
    "show reversal arrows for a trade that was in loss and now is in profit give a reward of 0.6 and holding a profitable trade 0.8 \n",
    "\n",
    "use cases for rewards\n",
    "\n",
    "first practise drawing on the chart ask chart gpt for the lines\n",
    "\n",
    "\n",
    "Rewards\n",
    "0 == idle = 0.2\n",
    "0 == close == profit ==  1\n",
    "0 == close == loss   == -1\n",
    "\n",
    "1 == open == 0.4\n",
    "1 == close == profit == 0.9\n",
    "1 == close == loss == -0.9\n",
    "1 == hold == profit == 0.7\n",
    "1 == hold == loss == -0.7\n",
    "\n",
    "2 == open == 0.4\n",
    "2 == close == profit == 0.9\n",
    "2 == close == loss == -0.9\n",
    "2 == hold == profit == 0.7\n",
    "2 == hold == loss == -0.7\n",
    "\n",
    "use case 2\n",
    "a trade that was in loss and now its in profit \n",
    "1 == hold == loss == profit == 0.8\n",
    "2 == hold == loss == profit == 0.8\n",
    "\n",
    "use case 3\n",
    "a trade that was in profit and now its in loss\n",
    "1 == hold == loss == profit == -0.8\n",
    "2 == hold == loss == profit == -0.8\n",
    "\n",
    "use case 2 and 3 need a bar that when it reverses the the bar widens to accomodate the change of a reversing arrow show strange graffiti on the bar\n",
    "\n",
    "\n",
    "use case 4\n",
    "account balance show == done\n",
    ":profit == done\n",
    ":accumulative profit == done\n",
    ":current_trade == done\n",
    "\n",
    "use case to achieve above objectives we need to have a history of the current trade called \n",
    "current_history()\n",
    ":highest _ price\n",
    ":lowest _ price\n",
    "\n",
    "use case 7:\n",
    "when the price goes really high above a certain point we need a different animation on the chart\n",
    "so that it can know that its really time to be ready for the next time == done\n",
    "\n",
    "This usecase commands that the change in price for the algorithm the rise of profit in the chart should be consipicous\n",
    "such that it reflects different bars\n",
    "\n",
    "1:/:2 == profit == 10 different bar\n",
    "1:/:2 == loss == -10 different bar\n",
    "\n",
    "1:/:2 == profit == 20 different bar\n",
    "1:/:2 == loss == -20 different bar\n",
    "\n",
    "1:/:2 == profit == 30 different bar\n",
    "1:/:2 == loss == -30 different bar\n",
    "\n",
    "1:/:2 == profit == 40 different bar\n",
    "1:/:2 == loss == -40 different bar\n",
    "\n",
    "\n",
    "1:/:2 == profit == 50 different bar\n",
    "1:/:2 == loss == -50 different bar\n",
    "\n",
    "\n",
    "1:/:2 == profit == 60 different bar\n",
    "1:/:2 == loss == -60 different bar\n",
    "\n",
    "\n",
    "1:/:2 == profit == 70 different bar\n",
    "1:/:2 == loss == -70 different bar\n",
    "\n",
    "1:/:2 == profit == 80 different bar\n",
    "1:/:2 == loss == -80 different bar\n",
    "\n",
    "1:/:2 == profit == 90 different bar\n",
    "1:/:2 == loss == -90 different bar\n",
    "\n",
    "1:/:2 == profit >= 100 different bar\n",
    "1:/:2 == loss == 100 different bar\n",
    "\n",
    "\n",
    "use case 8 \n",
    "\n",
    "create better entry colors on the chart == done\n",
    "\n",
    "the empty parts of the chart should include the effects that are happening on the forex environment == done\n",
    "\n",
    "use case 9\n",
    "\n",
    "the time steps should have the most recent image so that the missing aspect of loss presented after a timestep can be eliminated\n",
    "\n",
    "so the next model with have a shape of (1,5,224,224,3) the fifth frame should have the most recent time step for a step of 1\n",
    "\n",
    "frame1 = fifth image  =  0   += 1 = 1\n",
    "frame2 = fourth image =  32  += 1 = 33\n",
    "frame3 = third image  =  64  += 1 = 65\n",
    "frame4 = second image =  128 += 1 = 129\n",
    "frame5 = first image  =  129 += 1 = 130\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8e8baf5a-a20d-472f-ad93-9ed106fd5cee",
   "metadata": {},
   "source": [
    "#Chart Decorator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "64b4ac15-6e77-43c0-afb4-21e81e1d7ff8",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "class ChartDecorator:\n",
    "    def __init__(self):\n",
    "        self.balance_limit = 300\n",
    "        self.standard_balance = 1000\n",
    "        \n",
    "    def reset_chart(self, img, draw_context):\n",
    "        # Define the coordinates of the black bar\n",
    "        x1, y1 = 0, 204\n",
    "        x2, y2 = 200, 224\n",
    "        # Draw the black bar bottom bar\n",
    "        draw_context.rectangle([x1, y1, x2, y2], fill=(0, 0, 0))\n",
    "        # Draw the black bar top bar\n",
    "        x1, y1 = 0, 0\n",
    "        x2, y2 = 224, 14\n",
    "        draw_context.rectangle([x1, y1, x2, y2], fill=(0, 0, 0))\n",
    "\n",
    "\n",
    "    def draw_account_balance(self, img, draw_context, account_balance = 1000):\n",
    "        balance = self.balance_limit + (account_balance - self.standard_balance)\n",
    "        if balance == self.balance_limit:\n",
    "            x1, y1 = 3, 204\n",
    "            x2, y2 = 75 , 224\n",
    "            # Draw the green bar for account\n",
    "            draw_context.rectangle([x1, y1, x2, y2], fill=(110, 235, 131))\n",
    "\n",
    "        elif balance < self.balance_limit:\n",
    "            health_bar = int((balance * 75) / self.balance_limit )\n",
    "            lost_bar = 75 - health_bar\n",
    "            #current_balnace\n",
    "            x1, y1 = 3, 204\n",
    "            x2, y2 = health_bar, 224\n",
    "            draw_context.rectangle([x1, y1, x2, y2], fill=(110, 235, 131))\n",
    "            #lost bar\n",
    "            x1, y1 = health_bar, 204\n",
    "            x2, y2 = 75 , 224\n",
    "            draw_context.rectangle([x1, y1, x2, y2], fill=(255, 0, 0))\n",
    "        elif balance > self.balance_limit:\n",
    "            x1, y1 = 3, 204\n",
    "            x2, y2 = 75 , 224\n",
    "            # Draw the green bar for account\n",
    "            draw_context.rectangle([x1, y1, x2, y2], fill=(110, 235, 131)) \n",
    "            #Draw the profit section on account_bar\n",
    "            profit = int(((balance - self.balance_limit) * 75) / self.balance_limit)\n",
    "\n",
    "            x1, y1 = 80, 204\n",
    "            x2, y2 = 80 + profit , 224\n",
    "            # Draw the green bar for account\n",
    "            draw_context.rectangle([x1, y1, x2, y2], fill=(71, 44, 27)) \n",
    "\n",
    "        #draw start separator\n",
    "        #draw Account Separator and end separator\n",
    "        x1, y1 = 0, 204\n",
    "        x2, y2 = 3 , 224\n",
    "        # Draw the green bar for account\n",
    "        draw_context.rectangle([x1, y1, x2, y2], fill=(255, 87, 20))     \n",
    "\n",
    "        #draw Account Separator and end separator\n",
    "        x1, y1 = 76, 204\n",
    "        x2, y2 = 80 , 224\n",
    "        # Draw the green bar for account\n",
    "        draw_context.rectangle([x1, y1, x2, y2], fill=(255, 87, 20)) \n",
    "\n",
    "        #draw Account Separator and end separator\n",
    "        x1, y1 = 155, 204\n",
    "        x2, y2 = 160 , 224\n",
    "        # Draw the green bar for account\n",
    "        draw_context.rectangle([x1, y1, x2, y2], fill=(255, 87, 20)) \n",
    "\n",
    "    def draw_current_trade(self,img, draw_context, position=None):\n",
    "        if position == \"buy\":\n",
    "            x1, y1 = 160, 204\n",
    "            x2, y2 = 175, 224\n",
    "            draw_context.rectangle([x1, y1, x2, y2], fill=(255, 255, 0)) \n",
    "        elif position == \"sell\":\n",
    "            x1, y1 = 180, 204\n",
    "            x2, y2 = 195, 224\n",
    "            draw_context.rectangle([x1, y1, x2, y2], fill=(0, 255, 0))  \n",
    "        else:\n",
    "            x1, y1 = 200, 204\n",
    "            x2, y2 = 215, 224\n",
    "            draw_context.rectangle([x1, y1, x2, y2], fill=(255,0,255)) \n",
    "\n",
    "        #Draw position separator icons\n",
    "        x1, y1 = 175, 204\n",
    "        x2, y2 = 180, 224\n",
    "        draw_context.rectangle([x1, y1, x2, y2], fill=(255, 255, 255)) \n",
    "        x1, y1 = 195, 204\n",
    "        x2, y2 = 200, 224\n",
    "        draw_context.rectangle([x1, y1, x2, y2], fill=(255, 255, 255)) \n",
    "        x1, y1 = 215, 204\n",
    "        x2, y2 = 220, 224\n",
    "        draw_context.rectangle([x1, y1, x2, y2], fill=(255, 255, 255)) \n",
    "\n",
    "    def draw_profit_bar(self, img, draw_context, profit = 0, account_balance = 1000):\n",
    "        pnl_parts = int(abs(profit) / 10)\n",
    "        for segment in range(0, 200, 10):\n",
    "            f = segment + 10\n",
    "            x1, y1 = segment+10, 0\n",
    "            x2, y2 = f + 1, 9\n",
    "            draw_context.rectangle([x1, y1, x2, y2], fill=(255, 255, 255)) \n",
    "            #display profit on the chart\n",
    "            if profit > 0 and pnl_parts > 0 and segment >= 100:\n",
    "                if segment < pnl_parts * 10 + 100:\n",
    "                    x1, y1 = segment, 0\n",
    "                    x2, y2 = segment + 10, 9\n",
    "                    draw_context.rectangle([x1, y1, x2, y2], fill=(27, 152, 224)) \n",
    "            elif profit < 0 and pnl_parts > 0 and segment <= 90 and segment >= 100 - abs(profit):           \n",
    "                x1, y1 = segment, 0\n",
    "                x2, y2 = segment + 10, 9\n",
    "                draw_context.rectangle([x1, y1, x2, y2], fill=(255, 0, 0))  \n",
    "\n",
    "            account_balance_bar = (200*account_balance)/2000\n",
    "            x1, y1 = 0, 10\n",
    "            x2, y2 = account_balance_bar, 14\n",
    "            draw_context.rectangle([x1, y1, x2, y2], fill=(255, 119, 0)) \n",
    "\n",
    "\n",
    "    def draw_buy_bar(self, img2, start_lines, end_lines, position = None, profit = 0, prev_trade=None, account_balance = 1100):\n",
    "\n",
    "        from PIL import Image, ImageDraw\n",
    "        d = ImageDraw.Draw(img2)\n",
    "        \n",
    "        if start_lines[\"ask_line\"] == None:\n",
    "            start_lines[\"ask_line\"] = start_lines[\"bid_line\"] + 2\n",
    "        elif start_lines[\"bid_line\"] == None:\n",
    "            start_lines[\"bid_line\"] == start_lines[\"ask_line\"] - 2\n",
    "        elif end_lines[\"ask_line\"] == None:\n",
    "            end_lines[\"ask_line\"] = end_lines[\"bid_line\"] + 2\n",
    "        elif end_lines[\"bid_line\"] == None:\n",
    "            end_lines[\"bid_line\"] == end_lines[\"ask_line\"] - 2\n",
    "        \n",
    "        self.reset_chart(img2, d)\n",
    "        self.draw_account_balance(img2, d, account_balance)\n",
    "\n",
    "        if position == \"sell\":\n",
    "            #draw icon for trade setup\n",
    "            self.draw_current_trade(img2, d, \"sell\")\n",
    "            # Draw indicator of trade start\n",
    "            line_color = (200,200,200)\n",
    "            top = (200, end_lines[\"ask_line\"])\n",
    "            bottom = (200, end_lines[\"bid_line\"])\n",
    "            d.line([top, bottom], fill=line_color, width=20)\n",
    "\n",
    "            #Draw start marker and line\n",
    "            line_color = (0, 255, 0)\n",
    "            top = (200, start_lines[\"ask_line\"])\n",
    "            bottom = (200, start_lines[\"bid_line\"])\n",
    "            d.line([top, bottom], fill=line_color, width=30) \n",
    "            #Draw start line\n",
    "            line_color = (0, 255, 0)#(252, 81, 48)\n",
    "            top = (50, start_lines[\"ask_line\"])\n",
    "            bottom = (50, start_lines[\"bid_line\"])\n",
    "            d.line([top, bottom], fill=line_color, width=250) \n",
    "\n",
    "            #Draw profit indicator of sell\n",
    "            if profit < 0:\n",
    "                line_color = (255, 0, 0)\n",
    "                top = (200, start_lines[\"ask_line\"])\n",
    "                bottom = (200,end_lines[\"bid_line\"])\n",
    "                d.line([top, bottom], fill=line_color, width=10)\n",
    "            else:\n",
    "                line_color = (0, 255, 0)\n",
    "                top = (200, start_lines[\"ask_line\"])\n",
    "                bottom = (200,end_lines[\"bid_line\"])\n",
    "                d.line([top, bottom], fill=line_color, width=10)\n",
    "            #Draw loss indicator for sell\n",
    "            self.draw_profit_bar(img2, d, profit)\n",
    "        elif position == \"buy\":\n",
    "            #draw bottom right trade status sell , buy , close\n",
    "            self.draw_current_trade(img2, d, \"buy\") # \n",
    "            #Draw indicator of trade start\n",
    "            #Draw end marker\n",
    "            line_color = (65, 64, 102)\n",
    "            top = (200, end_lines[\"ask_line\"])\n",
    "            bottom = (200, end_lines[\"bid_line\"])\n",
    "            d.line([top, bottom], fill=line_color, width=20) \n",
    "\n",
    "            #Draw start marker for buy\n",
    "            line_color = (255, 255, 0)\n",
    "            top = (200, start_lines[\"ask_line\"])\n",
    "            bottom = (200, start_lines[\"bid_line\"])\n",
    "            d.line([top, bottom], fill=line_color, width=30) \n",
    "\n",
    "            #Draw start buy line across the chart\n",
    "            line_color = (255, 255, 0)#(255,0,200)\n",
    "            top = (50, start_lines[\"ask_line\"])\n",
    "            bottom = (50, start_lines[\"bid_line\"])\n",
    "            d.line([top, bottom], fill=line_color, width=250)         \n",
    "\n",
    "\n",
    "            if profit < 0:\n",
    "                line_color = (255, 0, 0)\n",
    "                top = (200, start_lines[\"ask_line\"])\n",
    "                bottom = (200, end_lines[\"bid_line\"])\n",
    "                d.line([top, bottom], fill=line_color, width=10)\n",
    "            else:\n",
    "                line_color = (255, 255, 0)\n",
    "                top = (200, start_lines[\"ask_line\"])\n",
    "                bottom = (200, end_lines[\"bid_line\"])\n",
    "                d.line([top, bottom], fill=line_color, width=10)\n",
    "\n",
    "            self.draw_profit_bar(img2, d, profit)\n",
    "\n",
    "        elif position == \"close\" and prev_trade == None:\n",
    "            self.draw_current_trade(img2, d, \"close\")\n",
    "            \n",
    "            line_color = (255,0,255)\n",
    "            top = (200, end_lines[\"ask_line\"])\n",
    "            bottom = (200, end_lines[\"bid_line\"])\n",
    "            d.line([top, bottom], fill=line_color, width=30)  \n",
    "\n",
    "            self.draw_profit_bar(img2, d, profit)\n",
    "        else:\n",
    "            #draw main vertical bar\n",
    "            self.draw_current_trade(img2, d, \"close\")\n",
    "\n",
    "            line_color = (255,0,255)\n",
    "            top = (200, start_lines[\"ask_line\"])\n",
    "            bottom = (200, end_lines[\"bid_line\"])\n",
    "            d.line([top, bottom], fill=line_color, width=15)\n",
    "            #draw end bar\n",
    "            line_color = (255,0,255)\n",
    "            top = (200, end_lines[\"ask_line\"])\n",
    "            bottom = (200, end_lines[\"bid_line\"])\n",
    "            d.line([top, bottom], fill=line_color, width=30) \n",
    "            #Draw start marker and line\n",
    "            line_color = (255,0,255)\n",
    "            top = (200, start_lines[\"ask_line\"])\n",
    "            bottom = (200, start_lines[\"bid_line\"])\n",
    "            d.line([top, bottom], fill=line_color, width=30)\n",
    "\n",
    "            line_color = (2, 8, 135)\n",
    "            top = (50, start_lines[\"ask_line\"])\n",
    "            bottom = (50, start_lines[\"bid_line\"])\n",
    "            d.line([top, bottom], fill=line_color, width=250)\n",
    "\n",
    "            self.draw_profit_bar(img2, d, profit)\n",
    "\n",
    "            if prev_trade == \"sell\" and profit > 0:   \n",
    "                #print right bar for the trade that has closed\n",
    "                line_color = (0, 255, 0)\n",
    "                top = (213, start_lines[\"ask_line\"])\n",
    "                bottom = (213, end_lines[\"bid_line\"])\n",
    "                d.line([top, bottom], fill=line_color, width=10) \n",
    "\n",
    "                line_color = (241, 254, 198)\n",
    "                top = (190, start_lines[\"ask_line\"])\n",
    "                bottom = (190, end_lines[\"bid_line\"])\n",
    "                d.line([top, bottom], fill=line_color, width=10)             \n",
    "\n",
    "            elif prev_trade == \"sell\" and profit < 0:\n",
    "                line_color = (0, 255, 0)\n",
    "                top = (213, start_lines[\"ask_line\"])\n",
    "                bottom = (213, end_lines[\"bid_line\"])\n",
    "                d.line([top, bottom], fill=line_color, width=10) \n",
    "\n",
    "                line_color = (255,0,0)\n",
    "                top = (190, start_lines[\"ask_line\"])\n",
    "                bottom = (190, end_lines[\"bid_line\"])\n",
    "                d.line([top, bottom], fill=line_color, width=10)    \n",
    "            elif prev_trade == \"buy\" and profit > 0:\n",
    "                line_color = (255, 255, 0)\n",
    "                top = (213, start_lines[\"ask_line\"])\n",
    "                bottom = (213, end_lines[\"bid_line\"])\n",
    "                d.line([top, bottom], fill=line_color, width=10) \n",
    "\n",
    "                line_color = (241, 254, 198)\n",
    "                top = (190, start_lines[\"ask_line\"])\n",
    "                bottom = (190, end_lines[\"bid_line\"])\n",
    "                d.line([top, bottom], fill=line_color, width=10)    \n",
    "\n",
    "            elif prev_trade == \"buy\" and profit < 0:\n",
    "                line_color = (255, 255, 0)\n",
    "                top = (213, start_lines[\"ask_line\"])\n",
    "                bottom = (213, end_lines[\"bid_line\"])\n",
    "                d.line([top, bottom], fill=line_color, width=10) \n",
    "\n",
    "                line_color = (255,0,0)\n",
    "                top = (190, start_lines[\"ask_line\"])\n",
    "                bottom = (190, end_lines[\"bid_line\"])\n",
    "                d.line([top, bottom], fill=line_color, width=10) \n",
    "\n",
    "        # display(img2)\n",
    "        return img2\n",
    "# def get_ask_bid_lines(image_path):\n",
    "#     ask_bid_lines = {\"ask_line\":None, \"bid_line\":None, \"key\": 0}\n",
    "#     img = Image.open(image_path)\n",
    "#     img_data = np.array(img)\n",
    "#     # print(img_data.shape)\n",
    "#     last_part = Image.fromarray(img_data[:,200:201,:])\n",
    "#     # display(last_part)\n",
    "#     img1_pixels = last_part.load()\n",
    "#     ask_color = (255,0,0)\n",
    "#     bid_color = (119, 136, 153)\n",
    "\n",
    "#     for y in range(224):\n",
    "#         if img1_pixels[0,y] == ask_color:\n",
    "#             ask_bid_lines[\"ask_line\"] = y\n",
    "#             # print(y)\n",
    "#         if img1_pixels[0,y] == bid_color:\n",
    "#             ask_bid_lines[\"bid_line\"] = y\n",
    "#             # print(y)\n",
    "#     return ask_bid_lines , img\n",
    "\n",
    "# chart_decorator = ChartDecorator()\n",
    "# ask_bid1 , img1 = get_ask_bid_lines(\"colabM1M5/episode2/Volatility_100_Index_M1_1158233112023.04.11-04:48:38_7330_7328.png\")\n",
    "# ask_bid2 , img2 = get_ask_bid_lines(\"colabM1M5/episode2/Volatility_100_Index_M1_1158233112023.04.11-04:58:14_7356_7354.png\")\n",
    "# img2 = chart_decorator.draw_buy_bar(img2, ask_bid1, ask_bid2, prev_trade=\"buy\", position=\"buy\" , profit = 0, account_balance = 1300 )"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dfc318b9-8231-487c-9adb-87fe3890bfb3",
   "metadata": {},
   "source": [
    "#Define Agent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "ec568662-7b2c-443e-95a6-2c4645cb0ec1",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# m5long = pd.read_csv(\"final_m5long.csv\")\n",
    "# m5short = pd.read_csv(\"final_m5short.csv\")\n",
    "# m1long = pd.read_csv(\"final_m1long.csv\")\n",
    "# m1short = pd.read_csv(\"final_complete_m1short.csv\")\n",
    "m1short = pd.read_csv(\"final_m1short.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "a802b90f-b3c8-4044-ac77-bcd0fba87040",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# def display_row_images(long_short_dict_now , long_short_dict_next):\n",
    "from collections import deque\n",
    "class ForexCustomEnv:\n",
    "    def __init__(self, m1short, chart_decorator):\n",
    "        self.m1short = m1short\n",
    "        self.trade_queue = deque(maxlen=4)\n",
    "        self.image_queue = deque(maxlen=4)\n",
    "        self.dataset_directory = \"colabM1M5/episode2/\"\n",
    "        self.account_balance = 1000\n",
    "        self.env_draw = False\n",
    "        self.chart_decorator = chart_decorator\n",
    "        self.prev_trade = None\n",
    "        \n",
    "        \n",
    "\n",
    "    def get_ask_bid_lines(self, image_path):\n",
    "        ask_bid_lines = {\"ask_line\":None, \"bid_line\":None, \"key\": 0}\n",
    "        img = Image.open(image_path)\n",
    "        img_data = np.array(img)\n",
    "        last_part = Image.fromarray(img_data[:,200:201,:])\n",
    "        img1_pixels = last_part.load()\n",
    "        ask_color = (255,0,0)\n",
    "        bid_color = (119, 136, 153)\n",
    "        for y in range(224):\n",
    "            if img1_pixels[0,y] == ask_color:\n",
    "                ask_bid_lines[\"ask_line\"] = y\n",
    "            if img1_pixels[0,y] == bid_color:\n",
    "                ask_bid_lines[\"bid_line\"] = y\n",
    "                \n",
    "        return ask_bid_lines , img          \n",
    "    \n",
    "    def reset(self, current_step = 4, key1 = 0, key2 = 1, key3 = 2, key4 = 3, slot1 = 4, slot2 = 3, slot3 = 2, slot4 = 1):\n",
    "        self.current_step = 4\n",
    "        linesq, imageq = self.get_ask_bid_lines(f\"{self.dataset_directory}{self.m1short.iloc[self.current_step-slot1]['image_path']}\")\n",
    "        linesq[\"key\"] = key1\n",
    "        linest, imaget = self.get_ask_bid_lines(f\"{self.dataset_directory}{self.m1short.iloc[self.current_step-slot2]['image_path']}\")\n",
    "        linest[\"key\"] = key2\n",
    "        liness, images = self.get_ask_bid_lines(f\"{self.dataset_directory}{self.m1short.iloc[self.current_step-slot3]['image_path']}\")\n",
    "        liness[\"key\"] = key3\n",
    "        linesc, imagec = self.get_ask_bid_lines(f\"{self.dataset_directory}{self.m1short.iloc[self.current_step-slot4]['image_path']}\")\n",
    "        linesc[\"key\"] = key4\n",
    "        self.trade_queue.append(linesq)\n",
    "        self.trade_queue.append(linest)\n",
    "        self.trade_queue.append(liness)\n",
    "        self.trade_queue.append(linesc)\n",
    "        self.image_queue.append(imageq)\n",
    "        self.image_queue.append(imaget)\n",
    "        self.image_queue.append(images)\n",
    "        self.image_queue.append(imagec)    \n",
    "        self.done = False\n",
    "        self.reward = 0\n",
    "        self.position = None\n",
    "        self.reset_current_trade()\n",
    "        self.account_balance = 1000\n",
    "        next_state = self.get_obs()\n",
    "        self.episode_profit = 0\n",
    "        self.prev_trade = None\n",
    "        \n",
    "        #reset standard limit in chart decorator\n",
    "        self.chart_decorator.standard_limit = self.account_balance\n",
    "        \n",
    "        return next_state, self.reward, self.done, self.current_trade \n",
    "        \n",
    "    def reset_current_trade(self):\n",
    "        self.current_trade = {}\n",
    "        self.current_trade[\"current_step\"] = self.current_step\n",
    "        self.current_trade[\"ask_bid_lines\"] = None\n",
    "        self.current_trade[\"entry_price\"] = None\n",
    "        self.current_trade[\"current_price\"] = None\n",
    "        self.current_trade[\"timesteps\"] = 0\n",
    "        self.current_trade[\"position\"] = None\n",
    "        self.current_trade[\"profit\"] = 0\n",
    "        self.current_trade[\"position\"] = None\n",
    "        self.current_trade[\"parent_ask_bid_lines\"] = None\n",
    "        self.current_trade[\"reward\"] = 0\n",
    "        self.current_trade[\"balance\"] = self.account_balance\n",
    "        self.current_trade[\"episode_profit\"] = 0\n",
    "        \n",
    "        \n",
    "    def display_stacked_horizontally(self,imageq, imaget,images,imagec):\n",
    "        # # Calculate the required dimensions for the new image\n",
    "        new_width = imageq.width * 4\n",
    "        new_height = imageq.height\n",
    "        # # Create a new image with the required dimensions\n",
    "        new_image = Image.new(\"RGB\", (new_width, new_height))\n",
    "        # # Paste the individual images side by side\n",
    "        new_image.paste(imageq, (0, 0))\n",
    "        new_image.paste(imaget, (imageq.width, 0))\n",
    "        new_image.paste(images, (imageq.width + imaget.width, 0))\n",
    "        new_image.paste(imagec, (imageq.width + imaget.width + images.width, 0))\n",
    "        \n",
    "\n",
    "    \n",
    "    def reset_collection(self, t_q, i_q):\n",
    "        _queue = deque(maxlen=4)\n",
    "        for i in range(0, 4):\n",
    "            _lines_c, _image_c = self.get_ask_bid_lines(f\"{self.dataset_directory}{self.m1short.iloc[t_q[i]['key']]['image_path']}\")\n",
    "            _image_c = self.draw_buy_bar(_image_c, _lines_c, _lines_c, position=\"close\", profit = 0)\n",
    "            _queue.append(_image_c)\n",
    "        return t_q, _queue\n",
    "    \n",
    "    def draw_collection(self,q):\n",
    "        # # Calculate the required dimensions for the new image\n",
    "        new_width = q[0].width * 4\n",
    "        new_height =q[0].height\n",
    "        # # Create a new image with the required dimensions\n",
    "        new_image = Image.new(\"RGB\", (new_width, new_height))\n",
    "        # # Paste the individual images side by side\n",
    "        new_image.paste(q[0], (0, 0))\n",
    "        new_image.paste(q[1], (q[0].width, 0))\n",
    "        new_image.paste(q[2], (q[0].width + q[0].width, 0))\n",
    "        new_image.paste(q[3], (q[0].width + q[0].width + q[0].width, 0))\n",
    "        display(new_image)\n",
    "        \n",
    "    def draw_buy_bar(self, current_image, start_lines, end_lines, position = None, profit = 0):\n",
    "        # def draw_buy_bar(self, img2, start_lines, end_lines, position = None, profit = 0, prev_trade=None, account_balance = 1100):\n",
    "        return self.chart_decorator.draw_buy_bar(current_image, start_lines, end_lines, self.position, profit, self.prev_trade, self.account_balance)\n",
    "#         try:\n",
    "#             if start_lines[\"ask_line\"] == None:\n",
    "#                 start_lines[\"ask_line\"] = start_lines[\"bid_line\"] + 2\n",
    "#             elif start_lines[\"bid_line\"] == None:\n",
    "#                 start_lines[\"bid_line\"] == start_lines[\"ask_line\"] - 2\n",
    "#             elif end_lines[\"ask_line\"] == None:\n",
    "#                 end_lines[\"ask_line\"] = end_lines[\"bid_line\"] + 2\n",
    "#             elif end_lines[\"bid_line\"] == None:\n",
    "#                 end_lines[\"bid_line\"] == end_lines[\"ask_line\"] - 2\n",
    "                \n",
    "#             from PIL import Image, ImageDraw\n",
    "#             d = ImageDraw.Draw(current_image)\n",
    "#             if self.position == \"sell\":\n",
    "#                 # Draw indicator of trade start\n",
    "#                 line_color = (200,200,200)\n",
    "#                 top = (200, end_lines[\"ask_line\"])\n",
    "#                 bottom = (200, end_lines[\"bid_line\"])\n",
    "\n",
    "#                 d.line([top, bottom], fill=line_color, width=20)\n",
    "#             #Draw profit indicator of sell\n",
    "#                 if profit < 0:\n",
    "#                     line_color = (0, 0, 255)\n",
    "#                     top = (200, start_lines[\"ask_line\"])\n",
    "#                     bottom = (200,end_lines[\"bid_line\"])\n",
    "#                     d.line([top, bottom], fill=line_color, width=10)\n",
    "#                 else:\n",
    "#                     line_color = (0, 255, 0)\n",
    "#                     top = (200, start_lines[\"ask_line\"])\n",
    "#                     bottom = (200,end_lines[\"bid_line\"])\n",
    "#                     d.line([top, bottom], fill=line_color, width=10)\n",
    "#             #Draw loss indicator for sell\n",
    "#             elif self.position == \"buy\":\n",
    "#                 #Draw indicator of trade start\n",
    "#                 line_color = (0,255,0)\n",
    "#                 top = (200, end_lines[\"ask_line\"])\n",
    "#                 bottom = (200, end_lines[\"bid_line\"])\n",
    "#                 d.line([top, bottom], fill=line_color, width=20)        \n",
    "#                 if profit < 0:\n",
    "#                     line_color = (255, 0, 0)\n",
    "#                     top = (200, start_lines[\"ask_line\"])\n",
    "#                     bottom = (200, end_lines[\"bid_line\"])\n",
    "#                     d.line([top, bottom], fill=line_color, width=10)\n",
    "#                 else:\n",
    "#                     line_color = (255, 255, 0)\n",
    "#                     top = (200, start_lines[\"ask_line\"])\n",
    "#                     bottom = (200, end_lines[\"bid_line\"])\n",
    "#                     d.line([top, bottom], fill=line_color, width=10)\n",
    "#             else:\n",
    "#                 line_color = (255,0,255)\n",
    "#                 top = (200, start_lines[\"ask_line\"])\n",
    "#                 bottom = (200, end_lines[\"bid_line\"])\n",
    "#                 d.line([top, bottom], fill=line_color, width=40)\n",
    "#         except Exception as e:\n",
    "#             print(f\"\\n\\n\\nstart_lines: {start_lines} end_lines:{end_lines}{e}\\n\\n\\n\")\n",
    "#         return current_image\n",
    "    \n",
    "    '''\n",
    "    @description: Enters a trade in the forex environment\n",
    "                  Enters the trade in the current time step, sets the position of the trade. sets the ask bid lines for the current frame\n",
    "                  sets the entry price for the trade, and profit = 0. Then draws on the image the entry icon on the chart\n",
    "                  and saves the updated current_image with modifications to the image queue with its corresponding ask and bid lines. \n",
    "                  then initializes the timesteps of the current trade to 0. sets the reward to 0.3\n",
    "    @params\n",
    "                  @position = None the position of the current_trade\n",
    "                  @ask_bid_lines of the current_image\n",
    "                  @current_image the current frame of the environment\n",
    "                  \n",
    "    '''\n",
    "    \n",
    "    def enter_trade(self, ask_bid_lines,current_image):\n",
    "        self.current_trade[\"position\"] = self.position\n",
    "        self.current_trade[\"current_step\"] = self.current_step\n",
    "        self.current_trade[\"ask_bid_lines\"] = ask_bid_lines\n",
    "        self.current_trade[\"entry_price\"] = float(f\"{self.m1short.iloc[self.current_step]['Ask']}\") if self.position == \"buy\" else float(f\"{self.m1short.iloc[self.current_step]['Bid']}\")\n",
    "        self.current_trade[\"profit\"] = 0\n",
    "        self.current_trade[\"current_price\"] = float(f\"{self.m1short.iloc[self.current_step]['Ask']}\")\n",
    "        self.current_trade[\"timesteps\"] = 0\n",
    "        self.current_trade[\"reward\"] = 0.4\n",
    "        self.current_trade[\"parent_ask_bid_lines\"] = ask_bid_lines\n",
    "        self.current_trade[\"balance\"] = self.account_balance\n",
    "        current_image = self.draw_buy_bar(current_image, ask_bid_lines, ask_bid_lines, self.position , self.current_trade[\"profit\"])\n",
    "        self.trade_queue.append(ask_bid_lines)\n",
    "        self.image_queue.append(current_image)  \n",
    "        if self.env_draw:\n",
    "            self.draw_collection(self.image_queue) \n",
    "        self.reward = 0.4\n",
    "        \n",
    "    def enter_idle_mode(self, ask_bid_lines, current_image ):\n",
    "        self.current_trade[\"current_step\"] = self.current_step\n",
    "        self.current_trade[\"ask_bid_lines\"] = ask_bid_lines\n",
    "        self.current_trade[\"entry_price\"] = float(f\"{self.m1short.iloc[self.current_step]['Ask']}\")\n",
    "        self.current_trade[\"current_price\"] = float(f\"{self.m1short.iloc[self.current_step]['Ask']}\")\n",
    "        self.current_trade[\"position\"] = self.position\n",
    "        self.current_trade[\"reward\"] = 0.2\n",
    "        self.current_trade[\"timesteps\"] += 1\n",
    "        self.current_trade[\"profit\"] = 0\n",
    "        self.current_trade[\"parent_ask_bid_lines\"] = None\n",
    "        self.current_trade[\"balance\"] = self.account_balance\n",
    "        current_image = self.draw_buy_bar(current_image, ask_bid_lines, ask_bid_lines, self.position, self.current_trade[\"profit\"])\n",
    "        self.trade_queue.append(ask_bid_lines)\n",
    "        self.image_queue.append(current_image)  \n",
    "        if self.env_draw:\n",
    "            self.draw_collection(self.image_queue)            \n",
    "        self.reward = 0.2 \n",
    "        \n",
    "    def hold_position(self,ask_bid_lines, current_image):\n",
    "        \n",
    "        self.current_trade[\"parent_ask_bid_lines\"] = self.parent_ask_bid_lines\n",
    "        self.current_trade[\"current_price\"] = float(f\"{self.m1short.iloc[self.current_step]['Bid']}\") if self.position == \"buy\" else float(f\"{self.m1short.iloc[self.current_step]['Ask']}\")\n",
    "        self.current_trade[\"profit\"] = self.current_trade[\"current_price\"] - self.current_trade[\"entry_price\"] if self.position == \"buy\" else self.current_trade[\"entry_price\"] - self.current_trade[\"current_price\"]\n",
    "        self.current_trade[\"timesteps\"] += 1\n",
    "        self.current_trade[\"balance\"] = self.account_balance\n",
    "        current_image = self.draw_buy_bar(current_image, self.current_trade[\"parent_ask_bid_lines\"], ask_bid_lines, self.position , self.current_trade[\"profit\"])\n",
    "        self.trade_queue.append(ask_bid_lines)\n",
    "        self.image_queue.append(current_image)\n",
    "        if self.env_draw:\n",
    "            self.draw_collection(self.image_queue) \n",
    "        if self.current_trade[\"profit\"] >= 0:\n",
    "            self.reward = 0.6\n",
    "            self.current_trade[\"reward\"] = self.reward\n",
    "        else:\n",
    "            self.reward = -0.6\n",
    "            self.current_trade[\"reward\"] = self.reward\n",
    "\n",
    "        \n",
    "                \n",
    "    def close_position(self, ask_bid_lines, current_image):\n",
    "        \n",
    "        self.current_trade[\"current_price\"] = float(f\"{self.m1short.iloc[self.current_step]['Bid']}\") if self.position == \"buy\" else float(f\"{self.m1short.iloc[self.current_step]['Ask']}\")\n",
    "        self.current_trade[\"profit\"] = self.current_trade[\"current_price\"] - self.current_trade[\"entry_price\"] if self.position == \"buy\" else self.current_trade[\"entry_price\"] - self.current_trade[\"current_price\"]\n",
    "        current_image = self.draw_buy_bar(current_image, self.parent_ask_bid_lines, ask_bid_lines, self.position, self.current_trade[\"profit\"] )\n",
    "        self.trade_queue.append(ask_bid_lines)\n",
    "        self.image_queue.append(current_image)\n",
    "        if self.env_draw:\n",
    "            self.draw_collection(self.image_queue) \n",
    "        if self.current_trade[\"profit\"] >= 0:\n",
    "            self.reward = 0.9\n",
    "        else:\n",
    "            self.reward = -0.9         \n",
    "        self.trade_queue, self.image_queue = self.reset_collection(self.trade_queue,self.image_queue)\n",
    "        self.account_balance += self.current_trade[\"profit\"]\n",
    "        self.current_trade[\"current_timestep\"] = self.current_step\n",
    "        self.current_trade[\"timesteps\"] += 1\n",
    "        self.current_trade[\"position\"] = None\n",
    "        self.current_trade[\"balance\"] = self.account_balance\n",
    "        self.current_trade[\"episode_profit\"] = self.episode_profit\n",
    "        self.position = None\n",
    "        self.parent_ask_bid = None\n",
    "        \n",
    "    def step(self, action):\n",
    "        self.reward = 0\n",
    "        self.done = False\n",
    "        ask_bid_lines, current_image = self.get_ask_bid_lines(f\"{self.dataset_directory}{self.m1short.iloc[self.current_step]['image_path']}\")\n",
    "        ask_bid_lines[\"key\"] = self.current_step\n",
    "        if self.position == None:\n",
    "            if action == 1:\n",
    "                self.position = \"buy\"\n",
    "                self.prev_trade = \"None\"\n",
    "                self.parent_ask_bid_lines = ask_bid_lines\n",
    "                self.enter_trade(ask_bid_lines,current_image)\n",
    "            elif action == 2:\n",
    "                self.position = \"sell\"\n",
    "                self.prev_trade = \"None\"\n",
    "                self.parent_ask_bid_lines = ask_bid_lines\n",
    "                self.enter_trade(ask_bid_lines,current_image)\n",
    "            elif action == 0:\n",
    "                self.position = None\n",
    "                self.prev_trade = None\n",
    "                self.parent_ask_bid_lines = ask_bid_lines\n",
    "                self.enter_idle_mode(ask_bid_lines, current_image)\n",
    "        elif self.position == \"buy\":\n",
    "            if action == 1:\n",
    "                self.prev_trade = \"buy\"\n",
    "                self.hold_position(ask_bid_lines, current_image)\n",
    "            elif action in [0,2]:\n",
    "                self.close_position(ask_bid_lines, current_image)\n",
    "                self.prev_trade = None\n",
    "                if self.account_balance > int((2/3) * self.chart_decorator.balance_limit) + self.chart_decorator.standard_balance:\n",
    "                    self.chart_decorator.standard_balance += self.chart_decorator.balance_limit\n",
    "        elif self.position == \"sell\":\n",
    "            if action == 2:\n",
    "                self.prev_trade = \"sell\"\n",
    "                self.hold_position(ask_bid_lines, current_image)\n",
    "                \n",
    "            elif action in [1,0]:\n",
    "                self.close_position(ask_bid_lines, current_image)\n",
    "                self.position = None\n",
    "                self.prev_trade = None\n",
    "                if self.account_balance > int((2/3) * self.chart_decorator.balance_limit) + self.chart_decorator.standard_balance:\n",
    "                    self.chart_decorator.standard_balance += self.chart_decorator.balance_limit\n",
    "        self.current_step+=1\n",
    "        next_state = self.get_obs()\n",
    "        if self.account_balance < 970:\n",
    "            self.done = True\n",
    "        print(self.trade_queue)\n",
    "        return next_state, self.reward, self.done, self.current_trade\n",
    "    \n",
    "    def get_obs(self):\n",
    "        stacked_obs = np.expand_dims(np.stack([np.asarray(self.image_queue[0])/255, np.asarray(self.image_queue[1])/255, np.asarray(self.image_queue[2])/255, np.asarray(self.image_queue[3])/255]), axis=0)\n",
    "        \n",
    "        return np.expand_dims(stacked_obs, axis=0)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "533f0f69-8de9-484f-84f8-d8db286b70e3",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "class TimeSformerBlock(tf.keras.layers.Layer):\n",
    "    def __init__(self, hidden_dim, num_heads):\n",
    "        super(TimeSformerBlock, self).__init__()\n",
    "        \n",
    "        # Define the self-attention layer\n",
    "        self.self_attention = tf.keras.layers.MultiHeadAttention(num_heads=num_heads, key_dim=hidden_dim)\n",
    "        \n",
    "        # Define the feedforward layer\n",
    "        self.feedforward = tf.keras.Sequential([\n",
    "            tf.keras.layers.Dense(hidden_dim * 4, activation='relu'),\n",
    "            tf.keras.layers.Dense(hidden_dim)\n",
    "        ])\n",
    "        \n",
    "        # Define the layer normalization layers\n",
    "        self.norm1 = tf.keras.layers.LayerNormalization()\n",
    "        self.norm2 = tf.keras.layers.LayerNormalization()\n",
    "        \n",
    "    def call(self, x):\n",
    "        # Apply layer normalization and self-attention\n",
    "        norm_x = self.norm1(x)\n",
    "        attention_output = self.self_attention(norm_x, norm_x)\n",
    "        x = x + attention_output\n",
    "        \n",
    "        # Apply layer normalization and feedforward layer\n",
    "        norm_x = self.norm2(x)\n",
    "        feedforward_output = self.feedforward(norm_x)\n",
    "        x = x + feedforward_output\n",
    "        \n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "d3610e83-fa64-4105-ba45-8a07a7cb4eed",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "class Agent:\n",
    "    def __init__(self, state_size, action_size, env, learning_rate=0.001):\n",
    "        self.state_size = state_size\n",
    "        self.action_size = action_size\n",
    "        self.learning_rate = learning_rate\n",
    "        self.epsilon = 1.0\n",
    "        self.epsilon_min = 0.01\n",
    "        self.epsilon_decay = 0.99\n",
    "        self.gamma = 0.01\n",
    "        self.env = env\n",
    "        self.actor = self._actor_model()\n",
    "        self.critic = self._critic_model()\n",
    "        # self.critic.set_weights(self.actor.get_weights())\n",
    "        self.models_dir = \"models/attention/\"\n",
    "        self.actor_model_name = \"att_actor.h5\"\n",
    "        self.critic_model_name = \"att_critic.h5\"\n",
    "        self.actor.summary()\n",
    "        \n",
    "        \n",
    "    def _actor_model(self):\n",
    "        cnn = Sequential()\n",
    "        cnn.add(tf.keras.layers.Conv3D(32, (3,3,3), strides=(1,4,4), padding=\"same\"))\n",
    "        cnn.add(Activation('relu'))\n",
    "        cnn.add(tf.keras.layers.MaxPooling3D(pool_size=(1,2,2), strides=(1,2,2), padding='same'))\n",
    "        cnn.add(tf.keras.layers.Conv3D(64, (3,3,3), padding=\"same\"))\n",
    "        cnn.add(Activation('relu'))\n",
    "        cnn.add(tf.keras.layers.MaxPooling3D(pool_size=(1,2,2), strides=(1,2,2), padding='same'))\n",
    "        cnn.add(tf.keras.layers.Conv3D(64, (3,3,3), padding=\"same\"))\n",
    "        cnn.add(Activation('relu'))\n",
    "        cnn.add(Flatten())\n",
    "        cnn.add(Dense(512, activation='relu'))\n",
    "        transformer = TimeSformerBlock( num_heads=8, hidden_dim=512)\n",
    "        model=Sequential()\n",
    "        model.add(tf.keras.layers.TimeDistributed(cnn,input_shape=self.state_size))\n",
    "        model.add(transformer)\n",
    "        model.add(Dense(32, activation='relu'))\n",
    "        # model.add(Dense(16, activation='relu'))\n",
    "        model.add(Dense(self.action_size, activation='softmax'))\n",
    "        model.compile(loss='categorical_crossentropy',optimizer=Adam(learning_rate=1e-4))\n",
    "        return model\n",
    "    \n",
    "    def _critic_model(self):\n",
    "        cnn = Sequential()\n",
    "        cnn.add(tf.keras.layers.Conv3D(32, (3,3,3), strides=(1,4,4), padding=\"same\"))\n",
    "        cnn.add(Activation('relu'))\n",
    "        cnn.add(tf.keras.layers.MaxPooling3D(pool_size=(1,2,2), strides=(1,2,2), padding='same'))\n",
    "        cnn.add(tf.keras.layers.Conv3D(64, (3,3,3), padding=\"same\"))\n",
    "        cnn.add(Activation('relu'))\n",
    "        cnn.add(tf.keras.layers.MaxPooling3D(pool_size=(1,2,2), strides=(1,2,2), padding='same'))\n",
    "        cnn.add(tf.keras.layers.Conv3D(64, (3,3,3), padding=\"same\"))\n",
    "        cnn.add(Activation('relu'))\n",
    "        cnn.add(Flatten())\n",
    "        cnn.add(Dense(512, activation='relu'))\n",
    "        transformer = TimeSformerBlock( num_heads=8, hidden_dim=512)\n",
    "        model=Sequential()\n",
    "        model.add(tf.keras.layers.TimeDistributed(cnn,input_shape=self.state_size))\n",
    "        model.add(transformer)\n",
    "        model.add(Dense(32, activation='relu'))\n",
    "        # model.add(Dense(16, activation='relu'))\n",
    "        model.add(Dense(self.action_size, activation='softmax'))\n",
    "        model.add(Dense(1, activation='linear'))\n",
    "        model.compile(loss='mse',optimizer=Adam(learning_rate=1e-4))\n",
    "        return model\n",
    "    \n",
    "    def predict(self, state):\n",
    "        return self.actor.predict(state)\n",
    "        \n",
    "    def save(self):\n",
    "        self.actor.save(f\"{self.models_dir}{self.actor_model_name}\")\n",
    "        self.critic.save(f\"{self.models_dir}{self.critic_model_name}\")\n",
    "        \n",
    "    def act(self, state):\n",
    "        if np.random.rand() <= self.epsilon:\n",
    "            return secrets.randbelow(self.action_size) if np.random.choice(20) % 2 == 0 else np.random.choice(self.action_size) , [[0,0,0]]\n",
    "        else:\n",
    "            pred = self.actor.predict(state)\n",
    "            return np.argmax(pred[0][0]), pred \n",
    "        \n",
    "    def load(self):\n",
    "        self.actor.load_weights(f\"{self.models_dir}{self.actor_model_name}\")\n",
    "        self.critic.load_weights(f\"{self.models_dir}{self.critic_model_name}\")\n",
    "        \n",
    "    def test_agent(self):\n",
    "        header = [\"step\",\"action\", \"reward\", \"done\",\"profit\", \"Episode Profit\", \"balance\",  \"actor\", \"Eprice\", \"Cprice\", ]\n",
    "        state, reward, done, current_trade = self.env.reset()\n",
    "        profits = []\n",
    "        for step in range(1225):\n",
    "            rows = []\n",
    "            action, pred = self.act(state)\n",
    "            next_state, reward, done, current_trade = self.env.step(action)\n",
    "            row = [step, action, reward,done, current_trade[\"profit\"], current_trade[\"episode_profit\"], current_trade[\"balance\"], \",\".join(str(\"{:.2f}\".format(x)) for x in list(pred)[0][0]), current_trade[\"entry_price\"], current_trade[\"current_price\"]]\n",
    "            rows.append(row)\n",
    "            print(tabulate(rows, headers=header, tablefmt='grid'))\n",
    "            print(f\"current_trade {current_trade}\\n\")\n",
    "            state = next_state\n",
    "            profits.append(current_trade[\"profit\"])\n",
    "            \n",
    "        # import matplotlib.pyplot as plt\n",
    "        # plt.plot(profits)\n",
    "        # plt.xlabel('Step')\n",
    "        # plt.ylabel('Total Profits')\n",
    "        # plt.show()\n",
    "    def train(self, num_episodes):\n",
    "        \n",
    "        header = [\"episode\",\"epsilon\", \"counter\",\"action\", \"done\" ,\"profit\", \"balance\",\"Eprice\",\"Cprice\", \"critic target_f_values\",\"target_f__reward\", \"actor pred qvls\", \"fit actor q_values\", \"total_profits\"]\n",
    "        self.episode_rewards = []\n",
    "        target_f_values = []\n",
    "        for episode in range(num_episodes):\n",
    "            state, reward, done, _ = self.env.reset()\n",
    "            total_rewards = 0\n",
    "            counter = 0\n",
    "            total_profits = 0\n",
    "            while not done:\n",
    "                rows = []\n",
    "                \n",
    "                action, pred = self.act(state)\n",
    "                next_state, reward, done, current_trade = self.env.step(action)\n",
    "                total_rewards += reward\n",
    "                total_profits += current_trade[\"profit\"]\n",
    "                if not done:\n",
    "                    target_f_values = self.critic.predict(next_state)\n",
    "                    \n",
    "                    target_f = reward + self.gamma * target_f_values[0][0]\n",
    "                    \n",
    "                else:\n",
    "                    target_f = reward\n",
    "                \n",
    "                q_values = self.actor.predict(state)\n",
    "                \n",
    "                try:\n",
    "                    q_values_copy = np.array(q_values, copy=True)\n",
    "                    q_values_copy[0][0][action] = target_f\n",
    "                    row = [episode,self.epsilon,counter, action, done,  current_trade[\"profit\"], current_trade[\"balance\"],current_trade[\"entry_price\"], current_trade[\"current_price\"], target_f_values[0][0], target_f, \",\".join(str(\"{:.2f}\".format(x)) for x in list(q_values)[0][0]),\",\".join(str(\"{:.2f}\".format(x)) for x in list(q_values_copy)[0][0]), total_profits]\n",
    "                    rows.append(row)\n",
    "                    print(tabulate(rows,headers = header, tablefmt=\"grid\"))\n",
    "                except Exception as e:\n",
    "                    print(f\"Exception {e}\")\n",
    "                    \n",
    "            \n",
    "                self.critic.fit(next_state, np.array(q_values[0][0][action]).reshape(1,1,1) , verbose=2)\n",
    "                \n",
    "                q_values[0][0][action] = target_f\n",
    "                \n",
    "\n",
    "                \n",
    "                self.actor.fit(state, q_values, verbose=2)\n",
    "                \n",
    "                state = next_state\n",
    "                \n",
    "                if counter % 50 == 0:\n",
    "                    self.save()\n",
    "                    \n",
    "                # Decay the exploration rate\n",
    "                if self.epsilon > self.epsilon_min and counter % 20 == 0:\n",
    "                    self.epsilon *= self.epsilon_decay\n",
    "                if counter > 1200:\n",
    "                    done = True\n",
    "                \n",
    "                counter += 1\n",
    "                \n",
    "                \n",
    "            clear_output(wait=True)\n",
    "            if episode % 3 == 0:\n",
    "                self.save()\n",
    "            # Decay the exploration rate\n",
    "            if self.epsilon > self.epsilon_min:\n",
    "                self.epsilon *= self.epsilon_decay\n",
    "            self.episode_rewards.append(total_rewards)\n",
    "            print(self.episode_rewards)\n",
    "        return self.episode_rewards\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "f4e0fd43-5295-4f45-9485-307909af95cd",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# agent.save()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "3e1d6f24-7fd9-4cb8-9132-5fa5b12eeeba",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[-0.3999999999999999]\n"
     ]
    }
   ],
   "source": [
    "if __name__ == \"__main__\":\n",
    "    action = 3\n",
    "    state_size = (1, 4, 224, 224, 3)\n",
    "    chart_decorator = ChartDecorator()\n",
    "    env = ForexCustomEnv(m1short, chart_decorator)\n",
    "    agent = Agent(state_size, action, env)\n",
    "    agent.env = env\n",
    "    agent.epsilon = 1.0\n",
    "    agent.env.env_draw = False\n",
    "    # agent.load()\n",
    "    episode_rewards = agent.train(num_episodes=1)\n",
    "    # agent.test_agent()\n",
    " \n",
    "# import matplotlib.pyplot as plt\n",
    "# plt.plot(episode_rewards)\n",
    "# plt.xlabel('Episode')\n",
    "# plt.ylabel('Total Reward')\n",
    "# plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f0849bfc-3f63-4374-b42d-86490872807c",
   "metadata": {},
   "source": [
    "# real work"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "280077fd-9868-4b9c-80a8-8c30478e2040",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9509900498999999"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# agent.epsilon"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ecb03557-fd26-48ee-9281-f8a9707c0295",
   "metadata": {},
   "outputs": [],
   "source": [
    "ticks = [ 0,0,0,0,1,1,1,1,1,1,1,0,2,2,2,2,2,2,2,2,2,2,2,2,1,1,1,1,1,1,1,1,1,1,1,1,1, 0, 2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bc5ea8fe-d87e-4cb1-84f3-12049594d34a",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# episode_rewards = agent.train(num_episodes=1)\n",
    "# agent.epsilon = 0.001\n",
    "# import matplotlib.pyplot as plt\n",
    "# plt.plot(episode_rewards)\n",
    "# plt.xlabel('Episode')\n",
    "# plt.ylabel('Total Reward')\n",
    "# plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "9a6b0a54-8eb7-4f27-8712-fc7a022487bf",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# agent.save()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "7d92aab0-b148-41a7-87c2-b1534880c5f6",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[59.20000000000001]"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# episode_rewards"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "831a9d45-af02-4bc2-aa0e-620958a836cc",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}