DRL_trader.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "O9O5DMVJv1UN",
    "colab_type": "text"
   },
   "source": [
    "# Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab_type": "code",
    "id": "aGKy8rnjtmho",
    "outputId": "a650b56e-4702-444e-c218-c93ad85b3ea5",
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 139.0
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: ta==0.3.8 in /usr/local/lib/python3.6/dist-packages (0.3.8)\n",
      "Requirement already satisfied: pandas in /usr/local/lib/python3.6/dist-packages (from ta==0.3.8) (0.24.2)\n",
      "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from ta==0.3.8) (1.16.3)\n",
      "Requirement already satisfied: pytz>=2011k in /usr/local/lib/python3.6/dist-packages (from pandas->ta==0.3.8) (2018.9)\n",
      "Requirement already satisfied: python-dateutil>=2.5.0 in /usr/local/lib/python3.6/dist-packages (from pandas->ta==0.3.8) (2.5.3)\n",
      "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.6/dist-packages (from python-dateutil>=2.5.0->pandas->ta==0.3.8) (1.12.0)\n"
     ]
    }
   ],
   "source": [
    "!pip install ta==0.3.8"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "mqEAZRnBv5rm",
    "colab_type": "code",
    "colab": {}
   },
   "outputs": [],
   "source": [
    "import pandas\n",
    "import numpy\n",
    "import datetime\n",
    "from datetime import timedelta\n",
    "from enum import Enum\n",
    "import argparse\n",
    "from ta import *\n",
    "\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.patches as patches\n",
    "\n",
    "import random\n",
    "import copy\n",
    "import numpy as np\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "import os\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import torch.nn.functional as F\n",
    "from torch.distributions import Normal, Categorical\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "EeBLdmpcxE3D",
    "colab_type": "text"
   },
   "source": [
    "# Classes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "id": "_245I4bNxGrp",
    "colab_type": "code",
    "colab": {}
   },
   "outputs": [],
   "source": [
    "class Singleton(object):\n",
    "    def __new__(cls, *args, **kw):\n",
    "        if not hasattr(cls, '_instance'):\n",
    "            orig = super(Singleton, cls)\n",
    "            cls._instance = orig.__new__(cls, *args, **kw)\n",
    "        return cls._instance\n",
    "\n",
    "\n",
    "      \n",
    "      \n",
    "#Config enviromnet elements       \n",
    "class EnvironmentConfiguration():\n",
    "    Symbol = \"\"\n",
    "    TrainStartDate = datetime.datetime.now()\n",
    "    TrainEndDate = datetime.datetime.now()\n",
    "      \n",
    "#Data Of every Frame  \n",
    "class StockData:\n",
    "    Symbol = \"\"\n",
    "    IndexDate = \"\"\n",
    "    Close = 0\n",
    "    Low = 0\n",
    "    High = 0\n",
    "    Volume = 0\n",
    "    volatility_atr=0\n",
    "    trend_macd=0\n",
    "    momentum_rsi=0\n",
    "\n",
    "\n",
    "class StockDataList(Singleton):\n",
    "    df = pandas.DataFrame()\n",
    "\n",
    "    \n",
    "# fill the data    \n",
    "class State():\n",
    "    HoldStock = False\n",
    "    StockDataFrame = pandas.DataFrame()\n",
    "    IndexDate = datetime.datetime.now()\n",
    "    stockData = StockData()\n",
    "    lastBuyStockData = StockData()\n",
    "    pastStockData = StockData()\n",
    "\n",
    "    def setDataFrame(self, indexDate):\n",
    "        self.IndexDate = indexDate\n",
    "        StockDatalist = StockDataList()\n",
    "        StartDate = self.IndexDate \n",
    "        \n",
    "        df = StockDatalist.df\n",
    "        df = df[df['Date'] <= self.IndexDate]\n",
    "        df = df[df['Date'] >= StartDate]\n",
    "        self.StockDataFrame = df\n",
    "\n",
    "        df2 = df[df['Date'] == self.IndexDate]\n",
    "        if len(df) > 0:\n",
    "            nstockdata = StockData()\n",
    "            nstockdata.Close = df2['Close'].values[0]\n",
    "            nstockdata.High = df2['High'].values[0]\n",
    "            nstockdata.Volume = df2['V'].values[0]\n",
    "            nstockdata.IndexDate = df2['Date'].values[0]\n",
    "            nstockdata.Low = df2['Low'].values[0]\n",
    "            nstockdata.Open = df2['Open'].values[0]\n",
    "            nstockdata.volatility_atr7  = df2['ATR7'].values[0]\n",
    "            nstockdata.volatility_atr14 = df2['ATR14'].values[0]\n",
    "            nstockdata.volatility_atr21 = df2['ATR21'].values[0]\n",
    "            nstockdata.volatility_atr32 = df2['ATR32'].values[0]\n",
    "            \n",
    "            \n",
    "            nstockdata.trend_macd = df2['MACD'].values[0]\n",
    "            nstockdata.momentum_rsi = df2['RSI'].values[0]\n",
    "            \n",
    "            \n",
    "            self.stockData = nstockdata\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "#  Action type\n",
    "class ActionType(Enum):\n",
    "    Buy = -1\n",
    "    Stay = 0\n",
    "    Sell = 1\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "class Reward:\n",
    "    TotalPoint = 0\n",
    "    LastState = State()\n",
    "    NextAction = ActionType\n",
    "\n",
    "\n",
    "\n",
    "# frame for rewards\n",
    "class PreviousRewards():\n",
    "    name = \"Rewards\"\n",
    "    dty = {\n",
    "        \"TotalPoint\": \"float64\",\n",
    "        \"LastBuyDate\": \"datetime64\",\n",
    "        \"Profits\": \"float64\"\n",
    "    }\n",
    "    Rewards = pandas.DataFrame(columns=dty.keys())\n",
    "    profit=0\n",
    "\n",
    " \n",
    "    def __init__(self):\n",
    "        self.Rewards.index.names = ['index']\n",
    "\n",
    "\n",
    " \n",
    "\n",
    "  \n",
    "\n",
    "    def Append(self, pastState, current_state, actionType):\n",
    "        printText = \"\"\n",
    "        reward = Reward()\n",
    "        reward.LastState = copy.deepcopy(current_state)\n",
    "        reward.NextAction = actionType\n",
    "        reward.TotalPoint, ptext,profit = self.Calculate_reward(pastState, current_state, actionType)\n",
    "      \n",
    "        if current_state.lastBuyStockData is None:\n",
    "            LastBuyDate = None\n",
    "        else:\n",
    "            LastBuyDate = current_state.lastBuyStockData.IndexDate\n",
    "\n",
    "        row = [\n",
    "               reward.TotalPoint,\n",
    "               LastBuyDate,\n",
    "               profit\n",
    "               ]\n",
    "        self.Rewards.loc[len(self.Rewards)] = row\n",
    "        return reward, printText\n",
    "\n",
    "\n",
    "        \n",
    "\n",
    "\n",
    "        \n",
    "        \n",
    "        \n",
    "        \n",
    "        \n",
    "        \n",
    "        \n",
    "       \n",
    "        \n",
    "#  reward Part    \n",
    "        \n",
    "    # reward for after buy action     \n",
    "    def calculate_stay_reward(self,holdStock,currentState,pastState):\n",
    "       rewardPoint=0\n",
    "       if holdStock:  # stok Varsa\n",
    "          currentStockData = copy.deepcopy(currentState.stockData)        \n",
    "          rewardPoint = (currentStockData.Close - pastState.stockData.Close)\n",
    "          if rewardPoint > 0:\n",
    "            rewardPoint=0.1\n",
    "          else:\n",
    "            rewardPoint=-0.1\n",
    "              \n",
    "       else:  # Stok yoksa\n",
    "           rewardPoint = -0.00\n",
    "           \n",
    "       return rewardPoint  \n",
    "     \n",
    "      \n",
    "      \n",
    "        \n",
    "    \n",
    "    def Calculate_reward(self, pastState, currentState,actionType):\n",
    "        rewardType=''\n",
    "        printText = \"\"\n",
    "        \n",
    "        \n",
    "        if (currentState.HoldStock == True and actionType == ActionType.Buy):\n",
    "            rewardPoint=self.calculate_stay_reward(currentState.HoldStock,currentState,pastState)\n",
    "            \n",
    "        elif (currentState.HoldStock == False and actionType == ActionType.Sell):\n",
    "            rewardPoint = -0.01 \n",
    "            \n",
    "            \n",
    "        else:\n",
    "\n",
    "            if actionType == ActionType.Sell:\n",
    "                buyStockData = copy.deepcopy(currentState.lastBuyStockData)\n",
    "                sellStockData = copy.deepcopy(currentState.stockData)\n",
    "\n",
    "                self.profit=self.profit + (sellStockData.Close - buyStockData.Close)\n",
    "              \n",
    "              \n",
    "                \n",
    "                profit = (sellStockData.Close - buyStockData.Close)\n",
    "\n",
    "                \n",
    "                if self.holdSteps!=0:\n",
    "                    rewardPoint = profit \n",
    "                else:\n",
    "                    rewardPoint=0.0\n",
    "                   \n",
    "                print(\n",
    "                      'dayprofit=',sellStockData.Close-buyStockData.Close,'self.profit=',self.profit,\n",
    "                      'BuyDate=',str(buyStockData.IndexDate)[2:10],str(buyStockData.IndexDate)[11:16] ,'SellDate=',str(sellStockData.IndexDate)[2:10],str(sellStockData.IndexDate)[11:16],\n",
    "                      'buyPrice=',buyStockData.Close,'sellPrice=',sellStockData.Close,\n",
    "                      'reward=',rewardPoint\n",
    "                      )    \n",
    "                if rewardPoint>0:\n",
    "                  rewardPoint=0.5\n",
    "                else:\n",
    "                  rewardPoint=-0.5\n",
    "                \n",
    "            elif actionType == ActionType.Stay:\n",
    "             \n",
    "              rewardPoint=self.calculate_stay_reward(currentState.HoldStock,currentState,pastState)\n",
    "\n",
    "            else:  # Buy\n",
    "                self.holdSteps=0\n",
    "               \n",
    "                rewardPoint = 0.00\n",
    "\n",
    "        rewardscaled = rewardPoint * 250\n",
    "#         print('rewardscaled',rewardscaled)\n",
    "        return rewardscaled, printText,self.profit\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "nlfsMDRowwA_",
    "colab_type": "text"
   },
   "source": [
    "# Enviroment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "id": "_drG4I9QwzFY",
    "colab_type": "code",
    "colab": {}
   },
   "outputs": [],
   "source": [
    "\n",
    "# enviroment class to handle data and return every step\n",
    "class Environment:\n",
    "    config = EnvironmentConfiguration()\n",
    "    Data = StockDataList()  \n",
    "    CurrentState = State()\n",
    "    pastState= State()\n",
    "\n",
    "    Info = \"\"\n",
    "    IterationID = 0 \n",
    "    Rewards = PreviousRewards()\n",
    "    stepCounter = 0\n",
    "    render_on = 0\n",
    "    EnvironmentID = 0\n",
    "   \n",
    "#reset Enviroment to start From Start Date of data\n",
    "    def reset(self, IterationNumber):\n",
    "        mindate = self.Data.df['Date'].min()\n",
    "        self.CurrentState = State()\n",
    "        self.pastState = State()\n",
    "\n",
    "        self.CurrentState.setDataFrame(mindate)\n",
    "        self.Info = \"\"\n",
    "        self.IterationID = IterationNumber\n",
    "        self.Rewards = PreviousRewards()\n",
    "        self.stepCounter = 0\n",
    "\n",
    "#     intialize Enviroment First Step and config    \n",
    "    def initializeEnv(self, config): \n",
    "        self.EnvironmentID = int(datetime.datetime.now().strftime(\"%Y%m%d%H%M%S\"))\n",
    "        self.config = config\n",
    "        self.Data = self.getData(config.DataLocation, config.Symbol, config.TrainStartDate, config.TrainEndDate)\n",
    "        mindate = self.Data.df['Date'].min()\n",
    "        self.CurrentState.setDataFrame(mindate)\n",
    "        \n",
    "        \n",
    "#   return next State,Reward,Done,and summary   \n",
    "    def step(self , actionType):\n",
    "        done = False\n",
    "        reward, ptext = self.Rewards.Append(self.pastState,self.CurrentState, actionType)\n",
    "        self.Info = \"Next Step in State \" + str(self.CurrentState) + \" with Action \" + str(actionType) + \" and Gain Reward \" + str(reward.TotalPoint)\n",
    "        self.pastState=copy.deepcopy(self.CurrentState)\n",
    "        nextState = State() \n",
    "        nextState.HoldStock = self.CurrentState.HoldStock\n",
    "        nextState.lastBuyStockData = copy.deepcopy(self.CurrentState.lastBuyStockData) \n",
    "\n",
    "        if actionType == ActionType.Sell:\n",
    "            if self.CurrentState.HoldStock==True:\n",
    "                done=True\n",
    "            nextState.HoldStock = False\n",
    "            nextState.lastBuyStockData = StockData()\n",
    "            nextState.Quantity = 0\n",
    "        elif actionType == ActionType.Buy:\n",
    "            if self.CurrentState.HoldStock==False:\n",
    "                nextState.HoldStock = True\n",
    "                nextState.lastBuyStockData = copy.deepcopy(self.CurrentState.stockData)\n",
    "                nextState.Quantity = 1\n",
    "\n",
    "\n",
    "        else: #Stay\n",
    "            nextState.lastBuyStockData = copy.deepcopy(self.CurrentState.lastBuyStockData)\n",
    "        tempDF= str(self.Data.df['Date'].values[0]) + ' ' + str(self.Data.df['Time'].values[0])\n",
    "        DF= pandas.to_datetime(str(self.Data.df['Date'].values[0]) + ' ' + str(self.Data.df['Time'].values[0]))\n",
    "      \n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "        newIndexDate = (self.Data.df[self.Data.df['Date']  > self.CurrentState.IndexDate])['Date'].min()\n",
    "\n",
    "     \n",
    "\n",
    "        if  pandas.isnull(newIndexDate): \n",
    "            done = True\n",
    "        nextState.IndexDate = newIndexDate\n",
    "        nextState.setDataFrame(newIndexDate)\n",
    "        \n",
    "        self.CurrentState = copy.deepcopy(nextState)\n",
    "        self.stepCounter = self.stepCounter  + 1\n",
    "        return self.CurrentState, reward, done, self.Info\n",
    "      \n",
    "      \n",
    "      \n",
    "      \n",
    "      \n",
    "    def getDataPointCount(self):\n",
    "        return len(self.Data.df)\n",
    "      \n",
    "      \n",
    "      \n",
    "      \n",
    "      \n",
    "    # import csv file\n",
    "    def getData(self, datalocation, Symbol='', StartDate = datetime.datetime.strptime('2001/1/1 00:00', \"%Y/%m/%d %H:%M\"), EndDate=datetime.datetime.strptime('2101/1/1 00:00', \"%Y/%m/%d %H:%M\")):\n",
    "        df = pandas.read_csv(datalocation, sep=\",\")\n",
    "\n",
    "        if Symbol != '':\n",
    "            df = df[df['Symbol'] == Symbol]\n",
    "        df['Date'] = pandas.to_datetime(df['Date'] + ' ' + df['Time'])\n",
    "        \n",
    "        df['ATR7'] = average_true_range(df[\"High\"], df[\"Low\"], df[\"Close\"], n=7,fillna=True)\n",
    "        df['ATR14'] = average_true_range(df[\"High\"], df[\"Low\"], df[\"Close\"], n=14,fillna=True)\n",
    "        df['ATR21'] = average_true_range(df[\"High\"], df[\"Low\"], df[\"Close\"], n=21,fillna=True)\n",
    "        df['ATR32'] = average_true_range(df[\"High\"], df[\"Low\"], df[\"Close\"], n=32,fillna=True)\n",
    "         \n",
    "        df['MACD'] = macd(df['Close'], n_fast=12, n_slow=26, fillna=False)\n",
    "        df['RSI'] = rsi(df['Close'], n=14, fillna=False)\n",
    "        \n",
    "        \n",
    "        df = df[df['Date'] <= EndDate]\n",
    "        df = df[df['Date'] >= StartDate]\n",
    "        df.reset_index()\n",
    "        sdl = StockDataList()\n",
    "        #sdl.MountData(df)\n",
    "        sdl.df = df\n",
    "        return sdl\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "   "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "bE8nMDFZxkG-",
    "colab_type": "text"
   },
   "source": [
    "# Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code",
    "id": "dB6bA8bOvwSZ",
    "colab": {}
   },
   "outputs": [],
   "source": [
    "class Actor(nn.Module):\n",
    "    def __init__(self, num_inputs,hidden_size):\n",
    "      \n",
    "        super(Actor, self).__init__()\n",
    "        self.L1hiddenActor=torch.zeros(1,1,hidden_size)\n",
    "        self.L1memmoryActor=torch.zeros(1,1,hidden_size)\n",
    "      \n",
    "        \n",
    "        self.actorL1 =nn.LSTM(\n",
    "            input_size=num_inputs,\n",
    "            hidden_size=hidden_size,\n",
    "            num_layers=1,\n",
    "        )\n",
    "\n",
    "        self.actorLinearL2 = nn.Linear(hidden_size, 2)\n",
    "\n",
    "    def forward(self, input):\n",
    "            \n",
    "        muL1,muStatesL1=self.actorL1(input,(self.L1hiddenActor,self.L1memmoryActor))\n",
    "        muOutL2 = self.actorLinearL2(muL1)\n",
    "        \n",
    "        dist=Categorical(muOutL2)\n",
    " \n",
    "        return muL1,dist\n",
    "      \n",
    "\n",
    "class Critic(nn.Module):\n",
    "    def __init__(self, num_inputs,hidden_size):\n",
    "        super(Critic, self).__init__()\n",
    "      \n",
    "      \n",
    "        self.L1hiddenCritic=torch.zeros(1,1,hidden_size)\n",
    "        self.L1memmoryCritic=torch.zeros(1,1,hidden_size)\n",
    "        \n",
    "\n",
    "        self.criticL1 = nn.LSTM(\n",
    "            input_size=num_inputs,\n",
    "            hidden_size=hidden_size,\n",
    "            num_layers=1,\n",
    "        )\n",
    "        \n",
    "        self.criticLinearL2 = nn.Linear(hidden_size, 1)\n",
    "\n",
    "\n",
    "    def forward(self, input):\n",
    "      \n",
    "        valueL1 ,valueStatesL1 = self.criticL1(input,(self.L1hiddenCritic,self.L1memmoryCritic))       \n",
    "        valueOutL2 = self.criticLinearL2(valueL1)\n",
    "\n",
    "\n",
    "        return valueOutL2\n",
    "      "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "id": "1uTuyWIEfDdC",
    "colab_type": "code",
    "colab": {}
   },
   "outputs": [],
   "source": [
    ""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "DNfP8mFGxpQI",
    "colab_type": "text"
   },
   "source": [
    "# Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code",
    "id": "R6BG0qPnXRYX",
    "colab": {}
   },
   "outputs": [],
   "source": [
    "from google.colab import files\n",
    "uploaded = files.upload()\n",
    "# "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab_type": "code",
    "outputId": "ffa7aef3-6651-4c5d-90ed-7bf260ee09ea",
    "id": "6RMhjTefgryv",
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 587591.0
    },
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "# HyperParameters\n",
    "\n",
    "GAMMA = 0.6   # Cares about Closest reward or future reward 0 myopic 1 farsighted\n",
    "GAE_LAMBDA = 0.95\n",
    "PPO_EPSILON = 0.2\n",
    "CRITIC_DISCOUNT = 1\n",
    "ENTROPY_BETA = 0.1\n",
    "PPO_STEPS = 40\n",
    "MINI_BATCH_SIZE = 16\n",
    "PPO_EPOCHS = 1\n",
    "TEST_EPOCHS =10\n",
    "NUM_TESTS = 1\n",
    "TARGET_REWARD = 100e6\n",
    "\n",
    "\n",
    "\n",
    "# ENV parameters\n",
    "\n",
    "config = EnvironmentConfiguration()\n",
    "config.DataLocation = 'EURUSD.e15.csv'\n",
    "config.TrainStartDate = datetime.datetime.strptime('2017/7/5 00:00', \"%Y/%m/%d %H:%M\") # it should be same with test_env it cant change singlely\n",
    "config.TrainEndDate = datetime.datetime.strptime('2019/3/1 23:47', \"%Y/%m/%d %H:%M\")\n",
    "env = Environment()\n",
    "env.initializeEnv(config)\n",
    "TestEnv=copy.deepcopy(env)\n",
    "print(env.Info)\n",
    "\n",
    "\n",
    "\n",
    "previousDays=30    # number Of previous Days feed To lstm\n",
    "dataCount = env.getDataPointCount() - (previousDays + 1)\n",
    "\n",
    "\n",
    "# Autodetect CUDA\n",
    "use_cuda = torch.cuda.is_available()\n",
    "device = torch.device(\"cuda\" if use_cuda else \"cpu\")\n",
    "print('Device:', device)\n",
    "\n",
    "\n",
    "# PPO model \n",
    "\n",
    "modelActor = Actor(12,100).to(device)\n",
    "modelCritic = Critic(12,10).to(device)\n",
    "\n",
    "LEARNING_RATE_ACTOR = 0.05\n",
    "LEARNING_RATE_CRITIC = 0.01\n",
    "\n",
    "optimizerActor = optim.Adadelta(modelActor.parameters(), lr = LEARNING_RATE_ACTOR) \n",
    "optimizerCritic = optim.Adadelta(modelCritic.parameters(), lr=LEARNING_RATE_CRITIC)\n",
    "\n",
    "\n",
    "scaler = MinMaxScaler(feature_range=(-1,1))\n",
    "\n",
    "\n",
    "\n",
    "\n",
    " \n",
    "\n",
    "# make Directory for saving Trained Models\n",
    "\n",
    "def mkdir(base, name):\n",
    "    path = os.path.join(base, name)\n",
    "    if not os.path.exists(path):\n",
    "        os.makedirs(path)\n",
    "    return path\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "#Returns the log of the probability density/mass function evaluated at value.\n",
    "def getLogProbFromDist(dist,actions):\n",
    "    logProbTemp=[]\n",
    " \n",
    "    for action in actions:\n",
    "        \n",
    "        \n",
    "        logProbTemp.append(dist.log_prob(action)[0][0])\n",
    "\n",
    "    return torch.tensor(logProbTemp)\n",
    "\n",
    "#  add Last State of enviroment to our state list and remove oldest one\n",
    "def addLastStateToStateList(states,newState):\n",
    "    newLSTMStates=[]\n",
    "    newLSTMStates.append([newState])\n",
    "\n",
    "    for i in  (range(states.size()[0])):\n",
    "        if i==states.size()[0]-1:\n",
    "            break\n",
    "        newLSTMStates.append(states[i].tolist())\n",
    "\n",
    "    return torch.tensor(newLSTMStates)\n",
    "\n",
    "# normalize single Number beetween range \n",
    "def singleNormalize(number,_range):\n",
    "    tempList=[number,-_range,_range]\n",
    "#     if number>_range :\n",
    "#       print(\"warning number is Bigger  than range\",' number = ',number ,' range = ',_range)\n",
    "#     if number<-_range :\n",
    "#       print(\"warning number is Smaller than range\",' number = ',number ,' range = ',_range)\n",
    "    numpyTemp = np.asarray([tempList])\n",
    "    numpyTemp = numpyTemp.reshape(-1, 1)\n",
    "    scaler.fit(numpyTemp)\n",
    "    tempList = scaler.transform(numpyTemp)\n",
    "    return tempList[0][0]\n",
    "\n",
    "# Normalize OHLC percentages\n",
    "def normalizePercentages(openPercentage,closePercentage,highPercentage,lowPercentage,_range=0):\n",
    "    openPercentageNormalize=singleNormalize(openPercentage,_range)\n",
    "    closePercentageNormalize=singleNormalize(closePercentage,_range)\n",
    "    highPercentageNormalize=singleNormalize(highPercentage,_range)\n",
    "    lowPercentageNormalize=singleNormalize(lowPercentage,_range)\n",
    "\n",
    "    return openPercentageNormalize,closePercentageNormalize,highPercentageNormalize,lowPercentageNormalize\n",
    "  \n",
    "# reseting delta Price To seperate Trades from Each Other\n",
    "def resetDeltaPriceHistory(envStateList):\n",
    "    envStateToList=envStateList.tolist()\n",
    "    envStateToListTemp=[]\n",
    "    for i in range(envStateList.size()[0]):\n",
    "      envStateToList[i][0][11]=0.0\n",
    "      \n",
    "      envStateToListTemp.append(envStateToList[i])\n",
    "    envStateList=torch.tensor(envStateToListTemp)\n",
    "    return envStateList  \n",
    "  \n",
    "  \n",
    "  \n",
    "  \n",
    "  \n",
    "# get State Array List in size Of previous Days\n",
    "def getStateListForLSTM(env,previousDays):\n",
    "    envStates=[]\n",
    "    envStatesTemp=[]\n",
    "    getState_volume=0\n",
    "    for i in range(previousDays):\n",
    "        getState, reward, done, _ = env.step(ActionType.Stay)\n",
    "        tempState=getStateFromEnviroment(getState,env)\n",
    "        \n",
    "        getState_volume=getState.stockData.Volume\n",
    "        tempstate=[tempState]\n",
    "        envStates.append(tempstate)\n",
    "    for state in  envStates:\n",
    "      envStatesTemp.insert(0,state)\n",
    "\n",
    "    return getState_volume,torch.tensor(envStatesTemp)\n",
    "  \n",
    "# Get percentage of part inside whole\n",
    "def percentage(part, whole):\n",
    "  return 100 * float(part)/float(whole)\n",
    "\n",
    "\n",
    "#  get OHLC delta percentage beetwen currentState and pastState\n",
    "def getOCHLpercentage(currentState,pastState):\n",
    "    closePercentage=percentage(currentState.stockData.Close-pastState.stockData.Close,pastState.stockData.Close)\n",
    "    highPercentage=percentage(currentState.stockData.High-pastState.stockData.High,pastState.stockData.Close)\n",
    "    lowPercentage=percentage(currentState.stockData.Low-pastState.stockData.Low,pastState.stockData.Close)\n",
    "    openPercentage=percentage(currentState.stockData.Open-pastState.stockData.Open,pastState.stockData.Close)\n",
    "    \n",
    "    return openPercentage,closePercentage,highPercentage,lowPercentage\n",
    "\n",
    "# Normalize and Fit every Single State come From ENV\n",
    "def getStateFromEnviroment(state, env):\n",
    "    VolumeDelta = percentage(state.stockData.Volume - env.pastState.stockData.Volume,env.pastState.stockData.Volume)\n",
    "\n",
    "    ATR7 = percentage(state.stockData.volatility_atr7 - env.pastState.stockData.volatility_atr7,env.pastState.stockData.volatility_atr7)\n",
    "    ATR7 =ATR7 *3\n",
    "    \n",
    "    ATR14= percentage(state.stockData.volatility_atr14 - env.pastState.stockData.volatility_atr14,env.pastState.stockData.volatility_atr14)\n",
    "    ATR14=ATR14*3.5\n",
    "    \n",
    "    ATR21= percentage(state.stockData.volatility_atr21 - env.pastState.stockData.volatility_atr21,env.pastState.stockData.volatility_atr21)\n",
    "    ATR21=ATR21*4\n",
    "\n",
    "    ATR32= percentage(state.stockData.volatility_atr32 - env.pastState.stockData.volatility_atr32,env.pastState.stockData.volatility_atr32)\n",
    "    ATR32=ATR32*5\n",
    "    \n",
    "    RSI= percentage(state.stockData.momentum_rsi - env.pastState.stockData.momentum_rsi,env.pastState.stockData.momentum_rsi)\n",
    "    RSI=RSI/5\n",
    "    \n",
    "    \n",
    "    MACD= percentage(state.stockData.trend_macd - env.pastState.stockData.trend_macd,env.pastState.stockData.trend_macd)\n",
    "    MACD=MACD/50\n",
    "    \n",
    "    ATR7 =singleNormalize(ATR7 ,35)\n",
    "    ATR14=singleNormalize(ATR14,35)\n",
    "    ATR21=singleNormalize(ATR21,35)\n",
    "    ATR32=singleNormalize(ATR32,35)\n",
    "    \n",
    "    RSI=singleNormalize(RSI,14)\n",
    "    \n",
    "    MACD=singleNormalize(MACD,5)\n",
    "\n",
    "    \n",
    "    VolumeDelta = singleNormalize(VolumeDelta, 800)\n",
    "    openPercentage, closePercentage, highPercentage, lowPercentage=getOCHLpercentage(state, env.pastState)\n",
    "    openPercentageNormalize, closePercentageNormalize, highPercentageNormalize, lowPercentageNormalize= normalizePercentages(openPercentage,closePercentage,highPercentage,lowPercentage,0.75)\n",
    " \n",
    "\n",
    "    if state.lastBuyStockData.Close == 0:\n",
    "        priceDelta = 0.0\n",
    "    else:\n",
    "        priceDelta = percentage(state.stockData.Close - state.lastBuyStockData.Close,\n",
    "                                state.lastBuyStockData.Close)\n",
    "        priceDelta = priceDelta * 6\n",
    "\n",
    "\n",
    "\n",
    "    if state.HoldStock:\n",
    "        holdStock = 0.5\n",
    "    else:\n",
    "        holdStock = -0.5\n",
    "    fitState = [openPercentageNormalize, closePercentageNormalize, highPercentageNormalize, lowPercentageNormalize, VolumeDelta ,RSI,MACD,ATR7,ATR14,ATR21,ATR32,holdStock, priceDelta]\n",
    "\n",
    "    fitState = np.around(np.asarray(fitState), decimals=4)\n",
    "\n",
    "    fitState = [fitState[0], fitState[1], fitState[2], fitState[3], fitState[4],\n",
    "                fitState[5],fitState[6], fitState[7], fitState[8], fitState[9],fitState[10],fitState[11],fitState[12]]\n",
    "    \n",
    "    \n",
    "    fitState = [fitState[0], fitState[1], fitState[2], fitState[3], fitState[4],\n",
    "                fitState[5],fitState[6], fitState[7],fitState[8],fitState[9],fitState[10],fitState[12]]\n",
    "#     print('O =',fitState[0],'C =',fitState[1],'H =',fitState[2],'L =',fitState[3],'V =',fitState[4],'RSI =',fitState[5],'MACD =',fitState[6],'ATR =',fitState[7],'HS =',fitState[8],'PrD =',fitState[9],'St',fitState[10])\n",
    "    return fitState\n",
    "\n",
    "\n",
    "# Change Action Format From Numpy To enum for Env.step(actionType)\n",
    "def getActionForEnviroment(action,holdStock):\n",
    "    actionType=getActionFromActionNumpy(action)\n",
    "    defaultResult = actionType,torch.tensor(action)\n",
    "#     if actionType==ActionType.Sell and holdStock==False :\n",
    "#         # print(\"if 1  ActionType.Sell  holdStock==false\")\n",
    "#         defaultResult= ActionType.Stay,torch.tensor([0])\n",
    "#     elif actionType==ActionType.Buy and holdStock==True :\n",
    "#         # print(\"if 2  ActionType.Buy  holdStock==True\")\n",
    "#         defaultResult= ActionType.Stay,torch.tensor([0])\n",
    "    # print('inputACtion=',action,'defaultResult',defaultResult,'holdStock=',holdStock)\n",
    "    return defaultResult\n",
    "\n",
    "\n",
    "def getActionFromActionNumpy(action):\n",
    "    if action==[0]:\n",
    "        return ActionType.Buy\n",
    "    elif action==[1]:\n",
    "        return ActionType.Sell\n",
    "\n",
    "\n",
    "\n",
    "# test ENV is enviroment for test Model performance\n",
    "def test_env(env, modelActor,modelCritic, device, deterministic=True):\n",
    "  \n",
    "#   config ENV to Test Data \n",
    "    config = EnvironmentConfiguration()\n",
    "    config.DataLocation = 'EURUSD.e15.csv'\n",
    "    config.TrainStartDate = datetime.datetime.strptime('2019/3/1 00:00', \"%Y/%m/%d %H:%M\") # it should be same with test_env it cant change singlely\n",
    "    config.TrainEndDate = datetime.datetime.strptime('2019/3/27 23:47', \"%Y/%m/%d %H:%M\")\n",
    "    env = Environment()\n",
    "    env.initializeEnv(config)\n",
    "    env.reset(1)\n",
    "    \n",
    "    \n",
    "    done = False\n",
    "    total_reward = 0\n",
    "#   GetState and prepair for lstm\n",
    "    lastStateVolumeTest,startStateListTest=getStateListForLSTM(env, previousDays=previousDays)\n",
    "    torchTestState = startStateListTest\n",
    "    envStateListTest = startStateListTest\n",
    "    holdStockTest=False\n",
    "    \n",
    "    \n",
    "    \n",
    "    for i in range(100):\n",
    "        torchTestState = torch.FloatTensor(torchTestState).to(device)  # send state to NN\n",
    "\n",
    "        L1Out,dist= modelActor(torchTestState)\n",
    "        _= modelCritic(torchTestState)\n",
    "        \n",
    "        print('valueOut',_[0][0])\n",
    "\n",
    "        print('dist.probs',dist.probs[0][0])\n",
    "        action=torch.argmax(dist.probs[0][0])  # get the action with highest probality\n",
    "        actiontype,editedAction = getActionForEnviroment(action.cpu().numpy(),holdStockTest)\n",
    "        print(\"actiontype\",actiontype)\n",
    "        envNextTestState, reward, done, _ = env.step(actiontype)\n",
    "        \n",
    "        envStateListTest = addLastStateToStateList(envStateListTest, getStateFromEnviroment(envNextTestState,env))\n",
    "        holdStockTest=envNextTestState.HoldStock\n",
    "        lastStateVolumeTest=envNextTestState.stockData.Volume\n",
    "        \n",
    "        print(\"reward\",reward.TotalPoint)\n",
    "        reward=reward.TotalPoint\n",
    "        total_reward += reward\n",
    "          \n",
    "        if done:\n",
    "          envStateListTest=resetDeltaPriceHistory(envStateListTest)\n",
    "\n",
    "        print('\\n')\n",
    "          \n",
    "        torchTestState = envStateListTest\n",
    "\n",
    "    print('Test total_reward',total_reward)\n",
    "    \n",
    "#   return ENV to train State\n",
    "    config.TrainStartDate = datetime.datetime.strptime('2017/7/5 00:00', \"%Y/%m/%d %H:%M\") # it should be same with test_env it cant change singlely\n",
    "    config.TrainEndDate = datetime.datetime.strptime('2019/3/1 23:47', \"%Y/%m/%d %H:%M\")\n",
    "    \n",
    "    env.initializeEnv(config)\n",
    "    \n",
    "    return total_reward\n",
    "\n",
    "\n",
    "def normalize(x):\n",
    "    x -= x.mean()\n",
    "    x /= (x.std() + 1e-1)\n",
    "    return x\n",
    "\n",
    "# Compute General Advance Estimation\n",
    "def compute_gae(next_value, rewards, masks, values, gamma=GAMMA, lam=GAE_LAMBDA):\n",
    "    values = values + [next_value]\n",
    "    gae = 0\n",
    "    returns = []\n",
    "    for step in reversed(range(len(rewards))):\n",
    "\n",
    "        delta = rewards[step] + gamma * values[step + 1] * masks[step] - values[step]\n",
    "        gae = delta + gamma * lam * masks[step] * gae\n",
    "        returns.insert(0, gae + values[step])\n",
    "    return returns\n",
    "\n",
    "# prepare batch data For update Model\n",
    "def ppo_iter(states, actions, log_probs, returns, advantage):\n",
    "    for i in range(len(actions)):\n",
    "        start = i*previousDays\n",
    "        stop = start + previousDays\n",
    "        \n",
    "        \n",
    "        yield states[start:stop, :], actions[i], log_probs[i], returns[i,:], advantage[i, :]\n",
    "\n",
    "# update Model\n",
    "def ppo_update(frame_idx, states, actions, log_probs, returns, advantages, clip_param=PPO_EPSILON):\n",
    "    count_steps = 0\n",
    "    sum_returns = 0.0\n",
    "    sum_advantage = 0.0\n",
    "    sum_loss_actor = 0.0\n",
    "    sum_loss_critic = 0.0\n",
    "    sum_entropy = 0.0\n",
    "    sum_loss_total = 0.0\n",
    "    running_loss=0\n",
    "    # PPO EPOCHS is the number of times we will go through ALL the training data to make updates\n",
    "    for _ in range(PPO_EPOCHS):\n",
    "        # grabs batche until we have covered all data\n",
    "        for state, action, old_log_probs, return_, advantage in ppo_iter(states, actions, log_probs,\n",
    "                                                                         returns, advantages):\n",
    "            L1Out,dist = modelActor(state)\n",
    "            value = modelCritic(state)\n",
    "            \n",
    "            entropy = dist.entropy().mean()   # ACTOR\n",
    "            new_log_probs = dist.log_prob(action)[0][0].to(device) # ACTOR\n",
    "            \n",
    "            ratio = (new_log_probs - old_log_probs).exp()  # ACTOR\n",
    "            \n",
    "           \n",
    "          \n",
    "          \n",
    "            surr1 = ratio * advantage  # ACTOR & CRITIC \n",
    "            surr2 = torch.clamp(ratio, 1.0 - clip_param, 1.0 + clip_param) * advantage\n",
    "\n",
    "            actor_loss = - torch.min(surr1, surr2).mean()\n",
    "            critic_loss = (return_ - torch.FloatTensor([[value[0][0].item()]])).pow(2).mean().to(device)\n",
    "\n",
    "            loss = CRITIC_DISCOUNT * critic_loss + actor_loss - ENTROPY_BETA * entropy\n",
    "            \n",
    "            # update NN\n",
    "            \n",
    "            optimizerActor.zero_grad()  \n",
    "            optimizerCritic.zero_grad()  \n",
    "            \n",
    "            \n",
    "            loss.backward()\n",
    "            \n",
    "            optimizerActor.step()\n",
    "            optimizerCritic.step()\n",
    "            \n",
    "            \n",
    "            \n",
    "            \n",
    "            running_loss += loss.item()\n",
    "    \n",
    "    print('running_loss',loss.item())\n",
    "    \n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    mkdir('.', 'checkpointsForBorsa')\n",
    "\n",
    "\n",
    "    print(modelActor)\n",
    "    print(modelCritic)\n",
    "    \n",
    "    frame_idx = 0\n",
    "    train_epoch = 0\n",
    "    best_reward = None\n",
    "    env.reset(1)\n",
    "    envRange=0\n",
    "#   Prepare State\n",
    "    lastStateVolume,startStateList=getStateListForLSTM(env, previousDays=previousDays)\n",
    "    torchState=startStateList\n",
    "    envStateList=startStateList\n",
    "    \n",
    "    \n",
    "    holdStock=False\n",
    "    early_stop = False\n",
    "    itearationNumber=1\n",
    "    while not early_stop:\n",
    "\n",
    "        log_probs = []\n",
    "        values = []\n",
    "        states = []\n",
    "        actions = []\n",
    "        rewards = []\n",
    "        masks = []\n",
    "        actionSeries=0\n",
    "        \n",
    "        for _ in range(PPO_STEPS):\n",
    "            \n",
    "            if envRange >= (dataCount-previousDays):\n",
    "\n",
    "                print('env.reset')\n",
    "\n",
    "                env.reset(itearationNumber)\n",
    "                itearationNumber+=1\n",
    "                lastStateVolumeReset, startStateList = getStateListForLSTM(env, previousDays=previousDays)\n",
    "                envStateList=startStateList\n",
    "                torchState=startStateList\n",
    "                envRange = 0\n",
    "            envRange += 1\n",
    "            torchState = torch.FloatTensor(torchState).to(device)\n",
    "            \n",
    "            L1Out,dist = modelActor(torchState) # get action from Actor\n",
    "            value = modelCritic(torchState) # get value from Critic\n",
    "          \n",
    "            action=torch.argmax(dist.probs[0][0])  # get the action with highest probality\n",
    "            action=action.item()\n",
    "            action=torch.tensor([action]).to(device)\n",
    "         \n",
    "            \n",
    "            print('dist.probs',dist.probs[0][0],value[0][0])\n",
    "            \n",
    "\n",
    "            actiontype, editedAction = getActionForEnviroment(action.cpu().numpy(),holdStock)\n",
    "\n",
    "            envNextState, reward, done, __ = env.step(actiontype)  # each state, reward, done \n",
    "            holdStock=envNextState.HoldStock\n",
    "\n",
    "            # append Stuffs\n",
    "\n",
    "            log_prob = getLogProbFromDist(dist,action)  #Returns the log of the probability density/mass function evaluated at value.\n",
    "            log_probs.append(log_prob)\n",
    "            values.append(torch.tensor([[value[0][0].item()]]))\n",
    "            rewards.append(torch.tensor([[reward.TotalPoint]]))\n",
    "            states.append(torchState)\n",
    "            actions.append(editedAction)\n",
    "            frame_idx += 1\n",
    "            \n",
    "            if done:\n",
    "                masks.append(torch.FloatTensor([0]))\n",
    "                envStateList=resetDeltaPriceHistory(envStateList)\n",
    "                torchState = envStateList\n",
    "                break\n",
    "            else:\n",
    "                masks.append(torch.FloatTensor([1]))\n",
    "                \n",
    "                \n",
    "                \n",
    "            if _>30:\n",
    "                env.step(ActionType.Sell) \n",
    "                envStateList=resetDeltaPriceHistory(envStateList)\n",
    "                torchState = envStateList\n",
    "                break\n",
    "                \n",
    "                \n",
    "                \n",
    "            envStateList=addLastStateToStateList(envStateList,getStateFromEnviroment(envNextState,env))\n",
    "            lastStateVolume = envNextState.stockData.Volume\n",
    "            torchState = envStateList\n",
    "            \n",
    "\n",
    "            \n",
    "        # Add next V(s) as 0.0\n",
    "        returns = compute_gae(torch.tensor([0.0]), rewards, masks, values) #compute general advance estimation\n",
    "        returns = torch.cat(returns).detach() #returns flat tensor without details\n",
    "\n",
    "        log_probs = torch.cat(log_probs).detach()\n",
    "        log_probs=np.around(log_probs,decimals=3)\n",
    "\n",
    "        values = torch.cat(values)\n",
    "        states = torch.cat(states)\n",
    "        actions = torch.cat(actions)\n",
    "\n",
    "        advantage = returns - values\n",
    "        advantage = normalize(advantage)\n",
    "        \n",
    "        ppo_update(frame_idx, states, actions, log_probs, returns, advantage)\n",
    "        \n",
    "#       Empty All Lists\n",
    "        log_probs = []\n",
    "        values = []\n",
    "        states = []\n",
    "        actions = []\n",
    "        rewards = []\n",
    "        masks = []\n",
    "       \n",
    "      \n",
    "      \n",
    "      \n",
    "      \n",
    "#       Go to test model performance and save its better than old Model\n",
    "        train_epoch += 1\n",
    "        print('train_epoch',train_epoch)\n",
    "        if train_epoch % TEST_EPOCHS == 0:\n",
    "            print('best_reward',best_reward)\n",
    "\n",
    "            test_reward = np.mean([test_env(TestEnv, modelActor,modelCritic, device) for _ in range(NUM_TESTS)])\n",
    "\n",
    "            print('Frame %s. reward: %s' % (frame_idx, test_reward))\n",
    "            # Save a checkpoint every time we achieve a best reward\n",
    "            if best_reward is None or best_reward < test_reward:\n",
    "                if best_reward is not None:\n",
    "                    print(\"Best reward updated: %.3f -> %.3f\" % (best_reward, test_reward))\n",
    "                    nameAc = \"%s_best_%+.3f_%d.dat\" % ('Actor', test_reward, frame_idx)\n",
    "                    nameCr = \"%s_best_%+.3f_%d.dat\" % ('Critic', test_reward, frame_idx)\n",
    "                    \n",
    "                    Actorname = os.path.join('.', 'checkpointsForBorsa', nameAc)\n",
    "                    Criticname = os.path.join('.', 'checkpointsForBorsa', nameCr)\n",
    "                    \n",
    "                    torch.save(modelActor.state_dict(), Actorname)\n",
    "                    torch.save(modelCritic.state_dict(), Criticname)\n",
    "                    \n",
    "                best_reward = test_reward\n",
    "\n",
    "            if test_reward > TARGET_REWARD: early_stop = True"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "aIkXoSWCGq8u",
    "colab_type": "text"
   },
   "source": [
    ""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "aIkXoSWCGq8u",
    "colab_type": "text"
   },
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "name": "DRL trader.ipynb",
   "version": "0.3.2",
   "provenance": [],
   "collapsed_sections": [],
   "toc_visible": true
  },
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3"
  },
  "accelerator": "TPU"
 },
 "nbformat": 4,
 "nbformat_minor": 0
}