Multi exchange broker mappings

README.md

@@ -5,7 +5,7 @@ I have been working on.
 Some additions have been made to this repo that I find useful. Your mileage may
 vary.
 
-Check out the example script to see how to setup and run on Kraken.
+Check out the sample scripts in `./samples` to see how to setup and run on Kraken and Binance.
 
 
 # Additions / Changes
@@ -15,11 +15,6 @@ Check out the example script to see how to setup and run on Kraken.
 - Added check for broker fills (complete notification, Cancel notification).
   Note that some exchanges may send different notification data
 
-- Broker mapping added as I noticed that there differences between the expected
-  order_types and retuned status's from canceling an order
-
-- Added a new mappings parameter to the script with defaults.
-
 - Added a new get_wallet_balance method. This will allow manual checking of the balance.
   The method will allow setting parameters. Useful for getting margin balances
 
@@ -28,28 +23,28 @@ Check out the example script to see how to setup and run on Kraken.
       with rest calls. As such, these will just return the last values called from getbalance().
       Because getbalance() will not be called by cerebro, you need to do this manually as and when  
       you want the information.
+- Broker mapping:
+     Naturally Backtrader uses general order types and doesn't know about specifics of 
+     individual crypto exchanges. E.g. status names, order type names, parameter names,....
+     This is why a mapping is needed between the parameter Backtrader passes on 
+     and the ones CCXT sends to the crypto exchanges.
+
+    Currently we support three means of mappings:
+
+    1. The static internal mappings of order types and status.
+        It can be found in `ccxtbt/resources/broker_mappings.json` and can be refined and
+        extended on demand.
+
+    2. The overwritable static mapping via `store.getbroker(broker_mapping=broker_mapping)` 
+        see `test/ccxtbt/mapping/test_mapping_file_handling.py:test_overwritten_mapping()` 
+
+    3. The overwritable broker methods to extend the mapping dynamically at runtime with custom logic. 
+        This has been needed for Binance e.g. because it doesn't have a 1:1 mapping from the
+        `stop-limit` Backtrader execution type to a single Binance order type constant. 
+        The Binance constants depend on the stop price being above or below the market price 
+        and on being a stop-limit buy order or a stop-limit sell order. 
+        See `test/ccxtbt/mapping/test_binance_broker.py`.
 
-- **Note:** The broker mapping should contain a new dict for order_types and mappings like below:
-
-```
-  broker_mapping = {
-      'order_types': {
-          bt.Order.Market: 'market',
-          bt.Order.Limit: 'limit',
-          bt.Order.Stop: 'stop-loss', #stop-loss for kraken, stop for bitmex
-          bt.Order.StopLimit: 'stop limit'
-      },
-      'mappings':{
-          'closed_order':{
-              'key': 'status',
-              'value':'closed'
-              },
-          'canceled_order':{
-              'key': 'result',
-              'value':1}
-              }
-      }
-```
 
   - Added new private_end_point method to allow using any private non-unified end point.
     An example for getting a list of postions and then closing them on Bitfinex

ccxtbt/__init__.py

@@ -1,3 +1,4 @@
 from .ccxtbroker import *
 from .ccxtfeed import *
 from .ccxtstore import *
+from .mapping import *

ccxtbt/ccxtbroker.py

@@ -20,12 +20,16 @@
 ###############################################################################
 from __future__ import (absolute_import, division, print_function,
                         unicode_literals)
+
 import collections
+import json
+
+from backtrader import BrokerBase, OrderBase
 from backtrader.position import Position
-from backtrader import BrokerBase, OrderBase, Order
 from backtrader.utils.py3 import queue, with_metaclass
+
 from .ccxtstore import CCXTStore
-import json
+
 
 class CCXTOrder(OrderBase):
     def __init__(self, owner, data, ccxt_order):
@@ -37,12 +41,15 @@ def __init__(self, owner, data, ccxt_order):
 
         super(CCXTOrder, self).__init__()
 
+
 class MetaCCXTBroker(BrokerBase.__class__):
     def __init__(cls, name, bases, dct):
         '''Class has already been created ... register'''
         # Initialize the class
         super(MetaCCXTBroker, cls).__init__(name, bases, dct)
-        CCXTStore.BrokerCls = cls
+        if name == 'CCXTBroker':
+            CCXTStore.BrokerCls = cls
+
 
 class CCXTBroker(with_metaclass(MetaCCXTBroker, BrokerBase)):
     '''Broker implementation for CCXT cryptocurrency trading library.
@@ -52,7 +59,7 @@ class CCXTBroker(with_metaclass(MetaCCXTBroker, BrokerBase)):
     Broker mapping added as I noticed that there differences between the expected
     order_types and retuned status's from canceling an order
 
-    Added a new mappings parameter to the script with defaults.
+    Added a new mapping parameter to the script with defaults.
 
     Added a get_balance function. Manually check the account balance and update brokers
     self.cash and self.value. This helps alleviate rate limit issues.
@@ -64,7 +71,7 @@ class CCXTBroker(with_metaclass(MetaCCXTBroker, BrokerBase)):
         Backtrader will call getcash and getvalue before and after next, slowing things down
         with rest calls. As such, th
 
-    The broker mapping should contain a new dict for order_types and mappings like below:
+    The broker mapping should contain a new dict for order_types and order_status like below:
 
     broker_mapping = {
         'order_types': {
@@ -73,7 +80,7 @@ class CCXTBroker(with_metaclass(MetaCCXTBroker, BrokerBase)):
             bt.Order.Stop: 'stop-loss', #stop-loss for kraken, stop for bitmex
             bt.Order.StopLimit: 'stop limit'
         },
-        'mappings':{
+        'order_status':{
             'closed_order':{
                 'key': 'status',
                 'value':'closed'
@@ -88,50 +95,27 @@ class CCXTBroker(with_metaclass(MetaCCXTBroker, BrokerBase)):
 
     '''
 
-    order_types = {Order.Market: 'market',
-                   Order.Limit: 'limit',
-                   Order.Stop: 'stop', #stop-loss for kraken, stop for bitmex
-                   Order.StopLimit: 'stop limit'}
-
-    mappings = {
-        'closed_order':{
-            'key': 'status',
-            'value':'closed'
-            },
-        'canceled_order':{
-            'key': 'status',
-            'value':'canceled'}
-            }
-
+    broker_mapping = None
 
     def __init__(self, broker_mapping=None, debug=False, **kwargs):
         super(CCXTBroker, self).__init__()
 
-        if broker_mapping is not None:
-            try:
-                self.order_types = broker_mapping['order_types']
-            except KeyError: # Might not want to change the order types
-                pass
-            try:
-                self.mappings = broker_mapping['mappings']
-            except KeyError: # might not want to change the mappings
-                pass
-
-
         self.store = CCXTStore(**kwargs)
 
+        self.broker_mapping = broker_mapping
+
         self.currency = self.store.currency
 
         self.positions = collections.defaultdict(Position)
 
         self.debug = debug
-        self.indent = 4 # For pretty printing dictionaries
+        self.indent = 4  # For pretty printing dictionaries
 
         self.notifs = queue.Queue()  # holds orders which are notified
 
         self.open_orders = list()
 
-        self.startingcash  = self.store._cash
+        self.startingcash = self.store._cash
         self.startingvalue = self.store._value
 
     def get_balance(self):
@@ -149,12 +133,12 @@ def get_wallet_balance(self, currency, params={}):
     def getcash(self):
         # Get cash seems to always be called before get value
         # Therefore it makes sense to add getbalance here.
-        #return self.store.getcash(self.currency)
+        # return self.store.getcash(self.currency)
         self.cash = self.store._cash
         return self.cash
 
     def getvalue(self, datas=None):
-        #return self.store.getvalue(self.currency)
+        # return self.store.getvalue(self.currency)
         self.value = self.store._value
         return self.value
 
@@ -194,21 +178,28 @@ def next(self):
                 print(json.dumps(ccxt_order, indent=self.indent))
 
             # Check if the order is closed
-            if ccxt_order[self.mappings['closed_order']['key']] == self.mappings['closed_order']['value']:
+            closed_order_exchange_value, closed_order_success_value = self.get_order_status_values(ccxt_order,
+                                                                                                   'closed_order',
+                                                                                                   self.broker_mapping)
+            if closed_order_exchange_value == closed_order_success_value:
                 pos = self.getposition(o_order.data, clone=False)
                 pos.update(o_order.size, o_order.price)
                 o_order.completed()
                 self.notify(o_order)
                 self.open_orders.remove(o_order)
 
     def _submit(self, owner, data, exectype, side, amount, price, params):
-        order_type = self.order_types.get(exectype) if exectype else 'market'
+        # order_type = self._get_order_type(exectype)
+        if 'broker_class' in self.broker_mapping:
+            order_type = exectype
+        else:
+            order_type = self.get_order_type(exectype, self.broker_mapping)
 
         # Extract CCXT specific params if passed to the order
         params = params['params'] if 'params' in params else params
 
         ret_ord = self.store.create_order(symbol=data.symbol, order_type=order_type, side=side,
-                                         amount=amount, price=price, params=params)
+                                          amount=amount, price=price, params=params)
 
         _order = self.store.fetch_order(ret_ord['id'], data.symbol)
 
@@ -218,6 +209,13 @@ def _submit(self, owner, data, exectype, side, amount, price, params):
         self.notify(order)
         return order
 
+    def get_order_type(self, exectype, broker_mapping):
+        local_exec_type = exectype if exectype else OrderBase.Market
+        order_type_name = OrderBase.ExecTypes[local_exec_type]
+        order_types = broker_mapping['order_types']
+        order_type = order_types.get(order_type_name)
+        return order_type
+
     def buy(self, owner, data, size, price=None, plimit=None,
             exectype=None, valid=None, tradeid=0, oco=None,
             trailamount=None, trailpercent=None,
@@ -242,31 +240,43 @@ def cancel(self, order):
             print('Broker cancel() called')
             print('Fetching Order ID: {}'.format(oID))
 
-
         # check first if the order has already been filled otherwise an error
         # might be raised if we try to cancel an order that is not open.
         ccxt_order = self.store.fetch_order(oID, order.data.symbol)
 
         if self.debug:
             print(json.dumps(ccxt_order, indent=self.indent))
 
-
-        if ccxt_order[self.mappings['closed_order']['key']] == self.mappings['closed_order']['value']:
+        closed_order_exchange_value, closed_order_success_value = self.get_order_status_values(ccxt_order,
+                                                                                               'closed_order',
+                                                                                               self.broker_mapping)
+        if closed_order_exchange_value == closed_order_success_value:
             return order
 
         ccxt_order = self.store.cancel_order(oID, order.data.symbol)
 
+        canceled_order_exchange_value, canceled_order_success_value = self.get_order_status_values(ccxt_order,
+                                                                                                   'canceled_order',
+                                                                                                   self.broker_mapping)
+
         if self.debug:
             print(json.dumps(ccxt_order, indent=self.indent))
-            print('Value Received: {}'.format(ccxt_order[self.mappings['canceled_order']['key']]))
-            print('Value Expected: {}'.format(self.mappings['canceled_order']['value']))
+            print('Value Received: {}'.format(canceled_order_exchange_value))
+            print('Value Expected: {}'.format(canceled_order_success_value))
 
-        if ccxt_order[self.mappings['canceled_order']['key']] == self.mappings['canceled_order']['value']:
-            self.open_orders.remove(order)
+        if canceled_order_exchange_value == canceled_order_success_value:
+            if order in self.open_orders:
+                self.open_orders.remove(order)
             order.cancel()
             self.notify(order)
         return order
 
+    def get_order_status_values(self, ccxt_order, status_type, broker_mapping):
+        exchange_status_key = broker_mapping['order_status'][status_type]['key']
+        success_order_status_value = broker_mapping['order_status'][status_type]['value']
+        exchange_status_value = ccxt_order[exchange_status_key]
+        return exchange_status_value, success_order_status_value
+
     def get_orders_open(self, safe=False):
         return self.store.fetch_open_orders()
 
@@ -287,9 +297,9 @@ def private_end_point(self, type, endpoint, params):
 
         print(dir(ccxt.hitbtc()))
         '''
-        endpoint_str = endpoint.replace('/','_')
-        endpoint_str = endpoint_str.replace('{','')
-        endpoint_str = endpoint_str.replace('}','')
+        endpoint_str = endpoint.replace('/', '_')
+        endpoint_str = endpoint_str.replace('{', '')
+        endpoint_str = endpoint_str.replace('}', '')
 
         method_str = 'private_' + type.lower() + endpoint_str.lower()