Add TimeInForce.GoodTilDate (GTD) support in backtesting and IB brokerage

Algorithm.CSharp/TimeInForceAlgorithm.cs

@@ -13,6 +13,8 @@
  * limitations under the License.
 */
 
+using System;
+using System.Collections.Generic;
 using QuantConnect.Data;
 using QuantConnect.Orders;
 
@@ -27,8 +29,10 @@ namespace QuantConnect.Algorithm.CSharp
     public class TimeInForceAlgorithm : QCAlgorithm
     {
         private Symbol _symbol;
-        private OrderTicket _gtcOrderTicket;
-        private OrderTicket _dayOrderTicket;
+        private OrderTicket _gtcOrderTicket1, _gtcOrderTicket2;
+        private OrderTicket _dayOrderTicket1, _dayOrderTicket2;
+        private OrderTicket _gtdOrderTicket1, _gtdOrderTicket2;
+        private readonly Dictionary<int, OrderStatus> _expectedOrderStatuses = new Dictionary<int, OrderStatus>();
 
         /// <summary>
         /// Initialise the data and resolution required, as well as the cash and start-end dates for your algorithm. All algorithms must initialized.
@@ -41,7 +45,7 @@ public override void Initialize()
 
             // The default time in force setting for all orders is GoodTilCancelled (GTC),
             // uncomment this line to set a different time in force.
-            // We currently only support GTC and DAY.
+            // We currently only support GTC, DAY, GTD.
             // DefaultOrderProperties.TimeInForce = TimeInForce.Day;
 
             _symbol = AddEquity("SPY", Resolution.Minute).Symbol;
@@ -53,22 +57,51 @@ public override void Initialize()
         /// <param name="data">Slice object keyed by symbol containing the stock data</param>
         public override void OnData(Slice data)
         {
-            if (_gtcOrderTicket == null)
+            if (_gtcOrderTicket1 == null)
             {
-                // This order has a default time in force of GoodTilCanceled,
-                // it will never expire and will not be canceled automatically.
+                // These GTC orders will never expire and will not be canceled automatically.
 
                 DefaultOrderProperties.TimeInForce = TimeInForce.GoodTilCanceled;
-                _gtcOrderTicket = LimitOrder(_symbol, 10, 160m);
+
+                // this order will not be filled before the end of the backtest
+                _gtcOrderTicket1 = LimitOrder(_symbol, 10, 100m);
+                _expectedOrderStatuses.Add(_gtcOrderTicket1.OrderId, OrderStatus.Submitted);
+
+                // this order will be filled before the end of the backtest
+                _gtcOrderTicket2 = LimitOrder(_symbol, 10, 160m);
+                _expectedOrderStatuses.Add(_gtcOrderTicket2.OrderId, OrderStatus.Filled);
             }
 
-            if (_dayOrderTicket == null)
+            if (_dayOrderTicket1 == null)
             {
-                // This order will expire at market close,
-                // if not filled by then it will be canceled automatically.
+                // These DAY orders will expire at market close,
+                // if not filled by then they will be canceled automatically.
 
                 DefaultOrderProperties.TimeInForce = TimeInForce.Day;
-                _dayOrderTicket = LimitOrder(_symbol, 10, 160m);
+
+                // this order will not be filled before market close and will be canceled
+                _dayOrderTicket1 = LimitOrder(_symbol, 10, 160m);
+                _expectedOrderStatuses.Add(_dayOrderTicket1.OrderId, OrderStatus.Canceled);
+
+                // this order will be filled before market close
+                _dayOrderTicket2 = LimitOrder(_symbol, 10, 180m);
+                _expectedOrderStatuses.Add(_dayOrderTicket2.OrderId, OrderStatus.Filled);
+            }
+
+            if (_gtdOrderTicket1 == null)
+            {
+                // These GTD orders will expire on October 10th at market close,
+                // if not filled by then they will be canceled automatically.
+
+                DefaultOrderProperties.TimeInForce = TimeInForce.GoodTilDate(new DateTime(2013, 10, 10));
+
+                // this order will not be filled before expiry and will be canceled
+                _gtdOrderTicket1 = LimitOrder(_symbol, 10, 100m);
+                _expectedOrderStatuses.Add(_gtdOrderTicket1.OrderId, OrderStatus.Canceled);
+
+                // this order will be filled before expiry
+                _gtdOrderTicket2 = LimitOrder(_symbol, 10, 160m);
+                _expectedOrderStatuses.Add(_gtdOrderTicket2.OrderId, OrderStatus.Filled);
             }
         }
 
@@ -82,5 +115,23 @@ public override void OnOrderEvent(OrderEvent orderEvent)
             Debug($"{Time} {orderEvent}");
         }
 
+        /// <summary>
+        /// End of algorithm run event handler. This method is called at the end of a backtest or live trading operation.
+        /// </summary>
+        public override void OnEndOfAlgorithm()
+        {
+            foreach (var kvp in _expectedOrderStatuses)
+            {
+                var orderId = kvp.Key;
+                var expectedStatus = kvp.Value;
+
+                var order = Transactions.GetOrderById(orderId);
+
+                if (order.Status != expectedStatus)
+                {
+                    throw new Exception($"Invalid status for order {orderId} - Expected: {expectedStatus}, actual: {order.Status}");
+                }
+            }
+        }
     }
 }

Algorithm.Python/TimeInForceAlgorithm.py

@@ -20,6 +20,8 @@
 from QuantConnect import *
 from QuantConnect.Algorithm import *
 from QuantConnect.Orders import *
+from QuantConnect.Orders.TimeInForces import *
+from datetime import datetime
 
 ### <summary>
 ### Demonstration algorithm of time in force order settings.
@@ -41,31 +43,70 @@ def Initialize(self):
         # We currently only support GTC and DAY.
         # self.DefaultOrderProperties.TimeInForce = TimeInForce.Day;
 
-        self.symbol = self.AddEquity("SPY", Resolution.Second).Symbol
+        self.symbol = self.AddEquity("SPY", Resolution.Minute).Symbol
 
-        self.gtcOrderTicket = None
-        self.dayOrderTicket = None
+        self.gtcOrderTicket1 = None
+        self.gtcOrderTicket2 = None
+        self.dayOrderTicket1 = None
+        self.dayOrderTicket2 = None
+        self.gtdOrderTicket1 = None
+        self.gtdOrderTicket2 = None
+        self.expectedOrderStatuses = {}
 
     # OnData event is the primary entry point for your algorithm. Each new data point will be pumped in here.
     # Arguments:
     #    data: Slice object keyed by symbol containing the stock data
     def OnData(self, data):
 
-        if self.gtcOrderTicket is None:
-            # This order has a default time in force of GoodTilCanceled,
-            # it will never expire and will not be canceled automatically.
+        if self.gtcOrderTicket1 is None:
+            # These GTC orders will never expire and will not be canceled automatically.
 
             self.DefaultOrderProperties.TimeInForce = TimeInForce.GoodTilCanceled
-            self.gtcOrderTicket = self.LimitOrder(self.symbol, 10, 160)
 
-        if self.dayOrderTicket is None:
-            # This order will expire at market close,
-            # if not filled by then it will be canceled automatically.
+            # this order will not be filled before the end of the backtest
+            self.gtcOrderTicket1 = self.LimitOrder(self.symbol, 10, 100)
+            self.expectedOrderStatuses[self.gtcOrderTicket1.OrderId] = OrderStatus.Submitted
+
+            # this order will be filled before the end of the backtest
+            self.gtcOrderTicket2 = self.LimitOrder(self.symbol, 10, 160)
+            self.expectedOrderStatuses[self.gtcOrderTicket2.OrderId] = OrderStatus.Filled
+
+        if self.dayOrderTicket1 is None:
+            # These DAY orders will expire at market close,
+            # if not filled by then they will be canceled automatically.
 
             self.DefaultOrderProperties.TimeInForce = TimeInForce.Day
-            self.dayOrderTicket = self.LimitOrder(self.symbol, 10, 160)
+
+            # this order will not be filled before market close and will be canceled
+            self.dayOrderTicket1 = self.LimitOrder(self.symbol, 10, 160)
+            self.expectedOrderStatuses[self.dayOrderTicket1.OrderId] = OrderStatus.Canceled
+
+            # this order will be filled before market close
+            self.dayOrderTicket2 = self.LimitOrder(self.symbol, 10, 180)
+            self.expectedOrderStatuses[self.dayOrderTicket2.OrderId] = OrderStatus.Filled
+
+        if self.gtdOrderTicket1 is None:
+            # These GTD orders will expire on October 10th at market close,
+            # if not filled by then they will be canceled automatically.
+
+            self.DefaultOrderProperties.TimeInForce = TimeInForce.GoodTilDate(datetime(2013, 10, 10))
+
+            # this order will not be filled before expiry and will be canceled
+            self.gtdOrderTicket1 = self.LimitOrder(self.symbol, 10, 100)
+            self.expectedOrderStatuses[self.gtdOrderTicket1.OrderId] = OrderStatus.Canceled
+
+            # this order will be filled before expiry
+            self.gtdOrderTicket2 = self.LimitOrder(self.symbol, 10, 160)
+            self.expectedOrderStatuses[self.gtdOrderTicket2.OrderId] = OrderStatus.Filled
 
     # Order event handler. This handler will be called for all order events, including submissions, fills, cancellations.
     # This method can be called asynchronously, ensure you use proper locks on thread-unsafe objects
     def OnOrderEvent(self, orderEvent):
         self.Debug(f"{self.Time} {orderEvent}")
+
+    # End of algorithm run event handler. This method is called at the end of a backtest or live trading operation.
+    def OnEndOfAlgorithm(self):
+        for orderId, expectedStatus in self.expectedOrderStatuses.items():
+            order = self.Transactions.GetOrderById(orderId)
+            if order.Status != expectedStatus:
+                raise Exception(f"Invalid status for order {orderId} - Expected: {expectedStatus}, actual: {order.Status}")

Brokerages/Backtesting/BacktestingBrokerage.cs

@@ -20,7 +20,6 @@
 using QuantConnect.Interfaces;
 using QuantConnect.Logging;
 using QuantConnect.Orders;
-using QuantConnect.Orders.TimeInForces;
 using QuantConnect.Securities;
 using QuantConnect.Securities.Option;
 
@@ -40,14 +39,6 @@ public class BacktestingBrokerage : Brokerage
         private readonly object _needsScanLock = new object();
         private readonly HashSet<Symbol> _pendingOptionAssignments = new HashSet<Symbol>();
 
-        private readonly Dictionary<TimeInForce, ITimeInForceHandler> _timeInForceHandlers = new Dictionary<TimeInForce, ITimeInForceHandler>
-        {
-            { TimeInForce.GoodTilCanceled, new GoodTilCanceledTimeInForceHandler() },
-            { TimeInForce.Day, new DayTimeInForceHandler() },
-            // Custom time in force will be renamed to GTD soon and will have its own new handler
-            { TimeInForce.Custom, new GoodTilCanceledTimeInForceHandler() }
-        };
-
         /// <summary>
         /// This is the algorithm under test
         /// </summary>
@@ -286,10 +277,8 @@ public void Scan()
                         continue;
                     }
 
-                    var timeInForceHandler = _timeInForceHandlers[order.TimeInForce];
-
                     // check if the time in force handler allows fills
-                    if (timeInForceHandler.IsOrderExpired(security, order))
+                    if (order.TimeInForce.IsOrderExpired(security, order))
                     {
                         OnOrderEvent(new OrderEvent(order, Algorithm.UtcTime, 0m)
                         {
@@ -386,7 +375,7 @@ public void Scan()
                     foreach (var fill in fills)
                     {
                         // check if the fill should be emitted
-                        if (!timeInForceHandler.IsFillValid(security, order, fill))
+                        if (!order.TimeInForce.IsFillValid(security, order, fill))
                         {
                             break;
                         }

Brokerages/InteractiveBrokers/InteractiveBrokersBrokerage.cs

@@ -35,6 +35,7 @@
 using IB = QuantConnect.Brokerages.InteractiveBrokers.Client;
 using IBApi;
 using NodaTime;
+using QuantConnect.Orders.TimeInForces;
 using Bar = QuantConnect.Data.Market.Bar;
 using HistoryRequest = QuantConnect.Data.HistoryRequest;
 
@@ -1626,6 +1627,30 @@ private IBApi.Order ConvertOrder(Order order, Contract contract, int ibOrderId)
                 Rule80A = _agentDescription
             };
 
+            var gtdTimeInForce = order.TimeInForce as GoodTilDateTimeInForce;
+            if (gtdTimeInForce != null)
+            {
+                DateTime expiryUtc;
+                if (order.SecurityType == SecurityType.Forex)
+                {
+                    expiryUtc = gtdTimeInForce.GetForexOrderExpiryDateTime(order);
+                }
+                else
+                {
+                    var exchangeHours = MarketHoursDatabase.FromDataFolder()
+                        .GetExchangeHours(order.Symbol.ID.Market, order.Symbol, order.SecurityType);
+
+                    var expiry = exchangeHours.GetNextMarketClose(gtdTimeInForce.Expiry.Date, false);
+                    expiryUtc = expiry.ConvertToUtc(exchangeHours.TimeZone);
+                }
+
+                // The IB format for the GoodTillDate order property is "yyyymmdd hh:mm:ss xxx" where yyyymmdd and xxx are optional.
+                // E.g.: 20031126 15:59:00 EST
+                // If no date is specified, current date is assumed. If no time-zone is specified, local time-zone is assumed.
+
+                ibOrder.GoodTillDate = expiryUtc.ToString("yyyyMMdd HH:mm:ss UTC", CultureInfo.InvariantCulture);
+            }
+
             var limitOrder = order as LimitOrder;
             var stopMarketOrder = order as StopMarketOrder;
             var stopLimitOrder = order as StopLimitOrder;
@@ -1750,7 +1775,7 @@ private Order ConvertOrder(IBApi.Order ibOrder, Contract contract)
 
             order.BrokerId.Add(ibOrder.OrderId.ToString());
 
-            order.Properties.TimeInForce = ConvertTimeInForce(ibOrder.Tif);
+            order.Properties.TimeInForce = ConvertTimeInForce(ibOrder.Tif, ibOrder.GoodTillDate);
 
             return order;
         }
@@ -1894,15 +1919,15 @@ private static OrderType ConvertOrderType(IBApi.Order order)
         /// <summary>
         /// Maps TimeInForce from IB to LEAN
         /// </summary>
-        private static TimeInForce ConvertTimeInForce(string timeInForce)
+        private static TimeInForce ConvertTimeInForce(string timeInForce, string expiryDateTime)
         {
             switch (timeInForce)
             {
                 case IB.TimeInForce.Day:
                     return TimeInForce.Day;
 
-                //case IB.TimeInForce.GoodTillDate:
-                //    return TimeInForce.GoodTilDate;
+                case IB.TimeInForce.GoodTillDate:
+                    return TimeInForce.GoodTilDate(ParseExpiryDateTime(expiryDateTime));
 
                 //case IB.TimeInForce.FillOrKill:
                 //    return TimeInForce.FillOrKill;
@@ -1917,6 +1942,30 @@ private static TimeInForce ConvertTimeInForce(string timeInForce)
             }
         }
 
+        private static DateTime ParseExpiryDateTime(string expiryDateTime)
+        {
+            // NOTE: we currently ignore the time zone in this method for a couple of reasons:
+            // - TZ abbreviations are ambiguous and unparsable to a unique time zone
+            //   see this article for more info:
+            //   https://codeblog.jonskeet.uk/2015/05/05/common-mistakes-in-datetime-formatting-and-parsing/
+            // - IB seems to also have issues with Daylight Saving Time zones
+            //   Example: an order submitted from Europe with GoodTillDate property set to "20180524 21:00:00 UTC"
+            //   when reading the open orders, the same property will have this value: "20180524 23:00:00 CET"
+            //   which is incorrect: should be CEST (UTC+2) instead of CET (UTC+1)
+
+            // We can ignore this issue, because the method is only called by GetOpenOrders,
+            // we only call GetOpenOrders during live trading, which means we won't be simulating time in force
+            // and instead will rely on brokerages to apply TIF properly.
+
+            var parts = expiryDateTime.Split(' ');
+            if (parts.Length == 3)
+            {
+                expiryDateTime = expiryDateTime.Substring(0, expiryDateTime.LastIndexOf(" ", StringComparison.Ordinal));
+            }
+
+            return DateTime.ParseExact(expiryDateTime, "yyyyMMdd HH:mm:ss", CultureInfo.InvariantCulture).Date;
+        }
+
         /// <summary>
         /// Maps TimeInForce from LEAN to IB
         /// </summary>
@@ -1931,24 +1980,27 @@ private static string ConvertTimeInForce(Order order)
                 return IB.TimeInForce.Day;
             }
 
-            switch (order.TimeInForce)
+            if (order.TimeInForce is DayTimeInForce)
             {
-                case TimeInForce.Day:
-                    return IB.TimeInForce.Day;
+                return IB.TimeInForce.Day;
+            }
 
-                //case TimeInForce.GoodTilDate:
-                //    return IB.TimeInForce.GoodTillDate;
+            if (order.TimeInForce is GoodTilDateTimeInForce)
+            {
+                return IB.TimeInForce.GoodTillDate;
+            }
 
-                //case TimeInForce.FillOrKill:
-                //    return IB.TimeInForce.FillOrKill;
+            //if (order.TimeInForce is FillOrKillTimeInForce)
+            //{
+            //    return IB.TimeInForce.FillOrKill;
+            //}
 
-                //case TimeInForce.ImmediateOrCancel:
-                //    return IB.TimeInForce.ImmediateOrCancel;
+            //if (order.TimeInForce is ImmediateOrCancelTimeInForce)
+            //{
+            //    return IB.TimeInForce.ImmediateOrCancel;
+            //}
 
-                case TimeInForce.GoodTilCanceled:
-                default:
-                    return IB.TimeInForce.GoodTillCancel;
-            }
+            return IB.TimeInForce.GoodTillCancel;
         }
 
         /// <summary>