Fall back from open interest mapping for KRX futures and warn on unavailable mapping modes

Algorithm.CSharp/FutureDataMappingModeFallbackRegressionAlgorithm.cs

@@ -0,0 +1,164 @@
+/*
+ * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+ * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+*/
+
+using System.Collections.Generic;
+using System.Linq;
+using QuantConnect.Data;
+using QuantConnect.Interfaces;
+using QuantConnect.Securities;
+using QuantConnect.Securities.Future;
+
+namespace QuantConnect.Algorithm.CSharp
+{
+    /// <summary>
+    /// Regression algorithm asserting that a future added with a data mapping mode its market has no data for (open interest on EUREX)
+    /// falls back to the market default, and that the related warnings are sent.
+    /// </summary>
+    public class FutureDataMappingModeFallbackRegressionAlgorithm : QCAlgorithm, IRegressionAlgorithmDefinition
+    {
+        private Future _future;
+        private bool _checkedAfterInitialize;
+
+        public override void Initialize()
+        {
+            SetStartDate(2024, 6, 3);
+            SetEndDate(2024, 6, 4);
+            SetAccountCurrency(Currencies.EUR);
+
+            _future = AddFuture(Futures.Indices.EuroStoxx50, Resolution.Minute, dataMappingMode: DataMappingMode.OpenInterest);
+            AssertWarning("Warning: OpenInterest data mapping mode is not available for EUREX futures, using LastTradingDay instead.");
+
+            // Open interest resolves to the mode already in use, so this is not a conflicting re-add
+            AddFuture(Futures.Indices.EuroStoxx50, Resolution.Minute, dataMappingMode: DataMappingMode.LastTradingDay);
+            if (DebugMessages.Any(message => message.Contains("already added")))
+            {
+                throw new RegressionTestException("Unexpected re-add warning for a future added again with the same data mapping mode");
+            }
+        }
+
+        public override void OnData(Slice slice)
+        {
+            if (_checkedAfterInitialize || _future.Mapped == null)
+            {
+                return;
+            }
+            _checkedAfterInitialize = true;
+
+            // Last trading day maps to the June contract, first day of the month would map to September
+            if (_future.Mapped.ID.Date.Month != 6)
+            {
+                throw new RegressionTestException($"Unexpected mapped contract {_future.Mapped}, expected the June contract");
+            }
+
+            var history = History(_future.Symbol, 10, Resolution.Minute).Select(x => x.Close).ToList();
+            if (history.Count == 0)
+            {
+                throw new RegressionTestException("Expected history for the continuous future using the fallback data mapping mode");
+            }
+
+            // explicit open interest falls back to the same mode, the market warning was already sent by AddFuture
+            var openInterestHistory = History(_future.Symbol, 10, Resolution.Minute, dataMappingMode: DataMappingMode.OpenInterest).Select(x => x.Close).ToList();
+            if (!openInterestHistory.SequenceEqual(history))
+            {
+                throw new RegressionTestException("Expected the explicit open interest history to match the fallback data mapping mode history");
+            }
+            AssertWarning("Warning: OpenInterest data mapping mode is not available for EUREX futures, using LastTradingDay instead.");
+
+            AddFuture(Futures.Indices.EuroStoxx50, Resolution.Minute, dataMappingMode: DataMappingMode.FirstDayMonth);
+            AssertWarning("Warning: /FESX already added, ignoring data mapping mode FirstDayMonth. Remove it first to change its settings.");
+        }
+
+        public override void OnEndOfAlgorithm()
+        {
+            if (!_checkedAfterInitialize)
+            {
+                throw new RegressionTestException("The continuous future was never mapped");
+            }
+            if (_future.Mapped.ID.Date.Month != 6)
+            {
+                throw new RegressionTestException($"Unexpected mapped contract {_future.Mapped} after the ignored re-add, expected the June contract");
+            }
+        }
+
+        private void AssertWarning(string warning)
+        {
+            if (DebugMessages.Count(message => message.EndsWith(warning)) != 1)
+            {
+                throw new RegressionTestException($"Expected the warning '{warning}' to be sent once");
+            }
+        }
+
+        /// <summary>
+        /// This is used by the regression test system to indicate if the open source Lean repository has the required data to run this algorithm.
+        /// </summary>
+        public bool CanRunLocally { get; } = true;
+
+        /// <summary>
+        /// This is used by the regression test system to indicate which languages this algorithm is written in.
+        /// </summary>
+        public List<Language> Languages { get; } = new() { Language.CSharp, Language.Python };
+
+        /// <summary>
+        /// Data Points count of all timeslices of algorithm
+        /// </summary>
+        public long DataPoints => 5010;
+
+        /// <summary>
+        /// Data Points count of the algorithm history
+        /// </summary>
+        public int AlgorithmHistoryDataPoints => 20;
+
+        /// <summary>
+        /// Final status of the algorithm
+        /// </summary>
+        public AlgorithmStatus AlgorithmStatus => AlgorithmStatus.Completed;
+
+        /// <summary>
+        /// This is used by the regression test system to indicate what the expected statistics are from running the algorithm
+        /// </summary>
+        public Dictionary<string, string> ExpectedStatistics => new Dictionary<string, string>
+        {
+            {"Total Orders", "0"},
+            {"Average Win", "0%"},
+            {"Average Loss", "0%"},
+            {"Compounding Annual Return", "0%"},
+            {"Drawdown", "0%"},
+            {"Expectancy", "0"},
+            {"Start Equity", "100000"},
+            {"End Equity", "100000"},
+            {"Net Profit", "0%"},
+            {"Sharpe Ratio", "0"},
+            {"Sortino Ratio", "0"},
+            {"Probabilistic Sharpe Ratio", "0%"},
+            {"Loss Rate", "0%"},
+            {"Win Rate", "0%"},
+            {"Profit-Loss Ratio", "0"},
+            {"Alpha", "0"},
+            {"Beta", "0"},
+            {"Annual Standard Deviation", "0"},
+            {"Annual Variance", "0"},
+            {"Information Ratio", "0"},
+            {"Tracking Error", "0"},
+            {"Treynor Ratio", "0"},
+            {"Total Fees", "€0.00"},
+            {"Estimated Strategy Capacity", "€0"},
+            {"Lowest Capacity Asset", ""},
+            {"Portfolio Turnover", "0%"},
+            {"Drawdown Recovery", "0"},
+            {"OrderListHash", "d41d8cd98f00b204e9800998ecf8427e"},
+        };
+    }
+}

Algorithm.Python/FutureDataMappingModeFallbackRegressionAlgorithm.py

@@ -0,0 +1,66 @@
+# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals.
+# Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from AlgorithmImports import *
+
+### <summary>
+### Regression algorithm asserting that a future added with a data mapping mode its market has no data for (open interest on EUREX)
+### falls back to the market default, and that the related warnings are sent.
+### </summary>
+class FutureDataMappingModeFallbackRegressionAlgorithm(QCAlgorithm):
+    def initialize(self):
+        self.set_start_date(2024, 6, 3)
+        self.set_end_date(2024, 6, 4)
+        self.set_account_currency(Currencies.EUR)
+
+        self._future = self.add_future(Futures.Indices.EURO_STOXX_50, Resolution.MINUTE, data_mapping_mode=DataMappingMode.OPEN_INTEREST)
+        self._assert_warning("Warning: OpenInterest data mapping mode is not available for EUREX futures, using LastTradingDay instead.")
+
+        # Open interest resolves to the mode already in use, so this is not a conflicting re-add
+        self.add_future(Futures.Indices.EURO_STOXX_50, Resolution.MINUTE, data_mapping_mode=DataMappingMode.LAST_TRADING_DAY)
+        if any("already added" in message for message in self.debug_messages):
+            raise AssertionError("Unexpected re-add warning for a future added again with the same data mapping mode")
+
+        self._checked_after_initialize = False
+
+    def on_data(self, slice):
+        if self._checked_after_initialize or self._future.mapped is None:
+            return
+        self._checked_after_initialize = True
+
+        # Last trading day maps to the June contract, first day of the month would map to September
+        if self._future.mapped.id.date.month != 6:
+            raise AssertionError(f"Unexpected mapped contract {self._future.mapped}, expected the June contract")
+
+        history = self.history(self._future.symbol, 10, Resolution.MINUTE)
+        if history.empty:
+            raise AssertionError("Expected history for the continuous future using the fallback data mapping mode")
+
+        # explicit open interest falls back to the same mode, the market warning was already sent by add_future
+        open_interest_history = self.history(self._future.symbol, 10, Resolution.MINUTE, data_mapping_mode=DataMappingMode.OPEN_INTEREST)
+        if list(open_interest_history["close"]) != list(history["close"]):
+            raise AssertionError("Expected the explicit open interest history to match the fallback data mapping mode history")
+        self._assert_warning("Warning: OpenInterest data mapping mode is not available for EUREX futures, using LastTradingDay instead.")
+
+        self.add_future(Futures.Indices.EURO_STOXX_50, Resolution.MINUTE, data_mapping_mode=DataMappingMode.FIRST_DAY_MONTH)
+        self._assert_warning("Warning: /FESX already added, ignoring data mapping mode FirstDayMonth. Remove it first to change its settings.")
+
+    def on_end_of_algorithm(self):
+        if not self._checked_after_initialize:
+            raise AssertionError("The continuous future was never mapped")
+        if self._future.mapped.id.date.month != 6:
+            raise AssertionError(f"Unexpected mapped contract {self._future.mapped} after the ignored re-add, expected the June contract")
+
+    def _assert_warning(self, warning):
+        if sum(1 for message in self.debug_messages if message.endswith(warning)) != 1:
+            raise AssertionError(f"Expected the warning '{warning}' to be sent once")

Algorithm/QCAlgorithm.History.cs

@@ -1106,10 +1106,10 @@ protected IEnumerable<HistoryRequest> CreateDateRangeHistoryRequests(IEnumerable
             {
                 var requests = new List<HistoryRequest>();
 
-                foreach (var config in GetMatchingSubscriptions(x, requestedType, resolution))
+                foreach (var config in GetMatchingSubscriptions(x, requestedType, resolution, dataMappingMode: dataMappingMode))
                 {
                     var request = _historyRequestFactory.CreateHistoryRequest(config, startAlgoTz, endAlgoTz, GetExchangeHours(x, requestedType), resolution,
-                        fillForward, extendedMarketHours, dataMappingMode, dataNormalizationMode, contractDepthOffset);
+                        fillForward, extendedMarketHours, GetHistoryDataMappingMode(config, dataMappingMode), dataNormalizationMode, contractDepthOffset);
                     requests.Add(request);
                 }
 
@@ -1149,7 +1149,7 @@ private IEnumerable<HistoryRequest> CreateBarCountHistoryRequests(IEnumerable<Sy
             return symbols.Where(HistoryRequestValid).SelectMany(symbol =>
             {
                 // Match or create configs for the symbol
-                var configs = GetMatchingSubscriptions(symbol, requestedType, resolution, useAllSubscriptions).ToList();
+                var configs = GetMatchingSubscriptions(symbol, requestedType, resolution, useAllSubscriptions, dataMappingMode).ToList();
                 if (configs.Count == 0)
                 {
                     return Enumerable.Empty<HistoryRequest>();
@@ -1166,17 +1166,33 @@ private IEnumerable<HistoryRequest> CreateBarCountHistoryRequests(IEnumerable<Sy
                     var end = Time;
 
                     return _historyRequestFactory.CreateHistoryRequest(config, start, end, exchange, res, fillForward,
-                        extendedMarketHours, dataMappingMode, dataNormalizationMode, contractDepthOffset);
+                        extendedMarketHours, GetHistoryDataMappingMode(config, dataMappingMode), dataNormalizationMode, contractDepthOffset);
                 });
             });
         }
 
+        /// <summary>
+        /// Gets the data mapping mode for a history request, falling back to the market default if the requested one is not available
+        /// </summary>
+        private DataMappingMode? GetHistoryDataMappingMode(SubscriptionDataConfig config, DataMappingMode? dataMappingMode)
+        {
+            var symbol = config.Symbol;
+            // only continuous futures data is mapped, not contracts nor chain universe data
+            if (!dataMappingMode.HasValue || symbol.SecurityType != SecurityType.Future || !symbol.IsCanonical()
+                || !LeanData.IsCommonLeanDataType(config.Type))
+            {
+                return dataMappingMode;
+            }
+            return GetDataMappingModeOrDefault(symbol, dataMappingMode);
+        }
+
         private int GetTickTypeOrder(SecurityType securityType, TickType tickType)
         {
             return SubscriptionManager.AvailableDataTypes[securityType].IndexOf(tickType);
         }
 
-        private IEnumerable<SubscriptionDataConfig> GetMatchingSubscriptions(Symbol symbol, Type type, Resolution? resolution = null, bool useAllSubscriptions = false)
+        private IEnumerable<SubscriptionDataConfig> GetMatchingSubscriptions(Symbol symbol, Type type, Resolution? resolution = null, bool useAllSubscriptions = false,
+            DataMappingMode? dataMappingMode = null)
         {
             var subscriptions = SubscriptionManager.SubscriptionDataConfigService
                 // we add internal subscription so that history requests are covered, this allows us to warm them up too
@@ -1293,7 +1309,9 @@ private IEnumerable<SubscriptionDataConfig> GetMatchingSubscriptions(Symbol symb
                 // Inherit values from existing subscriptions or use defaults
                 var extendedMarketHours = userConfigIfAny?.ExtendedMarketHours ?? UniverseSettings.ExtendedMarketHours;
                 var dataNormalizationMode = userConfigIfAny?.DataNormalizationMode ?? UniverseSettings.GetUniverseNormalizationModeOrDefault(symbol.SecurityType);
-                var dataMappingMode = userConfigIfAny?.DataMappingMode ?? UniverseSettings.GetUniverseMappingModeOrDefault(symbol.SecurityType, symbol.ID.Market);
+                var requestedDataMappingMode = dataMappingMode ?? userConfigIfAny?.DataMappingMode;
+                // only continuous futures data falls back from an unavailable mapping mode, other data types are not mapped
+                var unmappedDataMappingMode = requestedDataMappingMode ?? UniverseSettings.GetUniverseMappingModeOrDefault(symbol.SecurityType, symbol.ID.Market);
                 var contractDepthOffset = userConfigIfAny?.ContractDepthOffset ?? (uint)Math.Abs(UniverseSettings.ContractDepthOffset);
 
                 // If type was specified and not a lean data type and also not abstract, we create a new subscription
@@ -1325,7 +1343,7 @@ private IEnumerable<SubscriptionDataConfig> GetMatchingSubscriptions(Symbol symb
                         LeanData.GetCommonTickTypeForCommonDataTypes(dataType, symbol.SecurityType),
                         true,
                         dataNormalizationMode,
-                        dataMappingMode,
+                        unmappedDataMappingMode,
                         contractDepthOffset)};
                 }
 
@@ -1341,6 +1359,9 @@ private IEnumerable<SubscriptionDataConfig> GetMatchingSubscriptions(Symbol symb
                         // Use the config type to get an accurate mhdb entry
                         var entry = MarketHoursDatabase.GetEntry(symbol, new[] { configType });
                         var res = GetResolution(symbol, resolution, configType);
+                        var configDataMappingMode = LeanData.IsCommonLeanDataType(configType)
+                            ? GetDataMappingModeOrDefault(symbol, requestedDataMappingMode)
+                            : unmappedDataMappingMode;
 
                         return new SubscriptionDataConfig(
                             configType,
@@ -1355,7 +1376,7 @@ private IEnumerable<SubscriptionDataConfig> GetMatchingSubscriptions(Symbol symb
                             x.Item2,
                             true,
                             dataNormalizationMode,
-                            dataMappingMode,
+                            configDataMappingMode,
                             contractDepthOffset);
                     })
                     // lets make sure to respect the order of the data types, if used on a history request will affect outcome when using pushthrough for example