From d87d78846757f22cc51737db304b78ebc871b24b Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Thu, 24 Sep 2026 14:33:28 -0400 Subject: [PATCH 1/7] Fall back from open interest mapping for KRX futures and warn on unavailable mapping modes --- Algorithm/QCAlgorithm.History.cs | 5 +- Algorithm/QCAlgorithm.cs | 33 ++++++++++- Common/Extensions.cs | 13 ++++- Tests/Algorithm/AlgorithmAddSecurityTests.cs | 60 ++++++++++++++++++++ Tests/Common/Util/ExtensionsTests.cs | 15 +++++ 5 files changed, 121 insertions(+), 5 deletions(-) diff --git a/Algorithm/QCAlgorithm.History.cs b/Algorithm/QCAlgorithm.History.cs index afdd00def57a..f2b671a21826 100644 --- a/Algorithm/QCAlgorithm.History.cs +++ b/Algorithm/QCAlgorithm.History.cs @@ -1104,6 +1104,7 @@ protected IEnumerable CreateDateRangeHistoryRequests(IEnumerable { return symbols.Where(HistoryRequestValid).SelectMany(x => { + WarnIfDataMappingModeUnavailable(x, dataMappingMode); var requests = new List(); foreach (var config in GetMatchingSubscriptions(x, requestedType, resolution)) @@ -1148,6 +1149,8 @@ private IEnumerable CreateBarCountHistoryRequests(IEnumerable { + WarnIfDataMappingModeUnavailable(symbol, dataMappingMode); + // Match or create configs for the symbol var configs = GetMatchingSubscriptions(symbol, requestedType, resolution, useAllSubscriptions).ToList(); if (configs.Count == 0) @@ -1293,7 +1296,7 @@ private IEnumerable GetMatchingSubscriptions(Symbol symb // Inherit values from existing subscriptions or use defaults var extendedMarketHours = userConfigIfAny?.ExtendedMarketHours ?? UniverseSettings.ExtendedMarketHours; var dataNormalizationMode = userConfigIfAny?.DataNormalizationMode ?? UniverseSettings.GetUniverseNormalizationModeOrDefault(symbol.SecurityType); - var dataMappingMode = userConfigIfAny?.DataMappingMode ?? UniverseSettings.GetUniverseMappingModeOrDefault(symbol.SecurityType, symbol.ID.Market); + var dataMappingMode = userConfigIfAny?.DataMappingMode ?? GetUniverseMappingModeOrDefault(symbol); var contractDepthOffset = userConfigIfAny?.ContractDepthOffset ?? (uint)Math.Abs(UniverseSettings.ContractDepthOffset); // If type was specified and not a lean data type and also not abstract, we create a new subscription diff --git a/Algorithm/QCAlgorithm.cs b/Algorithm/QCAlgorithm.cs index fc79bf006ed8..19ec00236944 100644 --- a/Algorithm/QCAlgorithm.cs +++ b/Algorithm/QCAlgorithm.cs @@ -153,6 +153,8 @@ public partial class QCAlgorithm : MarshalByRefObject, IAlgorithm private int? _warmupBarCount; private Dictionary _parameters = new Dictionary(); private bool _deploymentDetailsSet; + private bool _dataMappingModeFallbackWarningSent; + private bool _unavailableDataMappingModeWarningSent; private SecurityDefinitionSymbolResolver _securityDefinitionSymbolResolver; private SecurityDefinitionSymbolResolver SecurityDefinitionSymbolResolver @@ -2095,6 +2097,8 @@ public Security AddSecurity(Symbol symbol, Resolution? resolution = null, bool? } else { + WarnIfDataMappingModeUnavailable(symbol, dataMappingMode); + // add the expected configurations of the canonical symbol right away, will allow it to warmup and indicators register to them var dataTypes = SubscriptionManager.LookupSubscriptionConfigDataTypes(SecurityType.Future, GetResolution(symbol, resolution, null), isCanonical: false); @@ -2102,7 +2106,7 @@ public Security AddSecurity(Symbol symbol, Resolution? resolution = null, bool? { ExtendedMarketHours = extendedMarketHours.Value, FillForward = fillForward.Value, - DataMappingMode = dataMappingMode ?? UniverseSettings.GetUniverseMappingModeOrDefault(symbol.SecurityType, symbol.ID.Market), + DataMappingMode = dataMappingMode ?? GetUniverseMappingModeOrDefault(symbol), DataNormalizationMode = dataNormalizationMode ?? UniverseSettings.GetUniverseNormalizationModeOrDefault(symbol.SecurityType), ContractDepthOffset = (int)contractOffset, SubscriptionDataTypes = dataTypes, @@ -2133,6 +2137,33 @@ public Security AddSecurity(Symbol symbol, Resolution? resolution = null, bool? return AddToUserDefinedUniverse(security, configs); } + /// + /// Gets the default data mapping mode for the given symbol, warning once if the universe settings mode is not available for its market + /// + private DataMappingMode GetUniverseMappingModeOrDefault(Symbol symbol) + { + var dataMappingMode = UniverseSettings.GetUniverseMappingModeOrDefault(symbol.SecurityType, symbol.ID.Market); + if (dataMappingMode != UniverseSettings.DataMappingMode && !_dataMappingModeFallbackWarningSent) + { + _dataMappingModeFallbackWarningSent = true; + Debug($"Warning: {UniverseSettings.DataMappingMode} data mapping mode is not available for {symbol.ID.Market.ToUpperInvariant()} futures, using {dataMappingMode} instead."); + } + return dataMappingMode; + } + + /// + /// Warns once if an explicitly requested data mapping mode has no mapping data for the future's market + /// + private void WarnIfDataMappingModeUnavailable(Symbol symbol, DataMappingMode? dataMappingMode) + { + if (dataMappingMode.HasValue && symbol.SecurityType == SecurityType.Future && !_unavailableDataMappingModeWarningSent + && !dataMappingMode.Value.IsAvailableForFutureMarket(symbol.ID.Market)) + { + _unavailableDataMappingModeWarningSent = true; + Debug($"Warning: {dataMappingMode} data mapping mode is not available for {symbol.ID.Market.ToUpperInvariant()} futures, no contract will be mapped. Use {DataMappingMode.LastTradingDay} instead."); + } + } + /// /// Creates and adds a new security to the algorithm /// diff --git a/Common/Extensions.cs b/Common/Extensions.cs index be982b6c8f16..c937c1ae3db8 100644 --- a/Common/Extensions.cs +++ b/Common/Extensions.cs @@ -3947,9 +3947,7 @@ public static DataMappingMode GetUniverseMappingModeOrDefault(this UniverseSetti switch (securityType) { case SecurityType.Future: - if ((universeSettings.DataMappingMode == DataMappingMode.OpenInterest - || universeSettings.DataMappingMode == DataMappingMode.OpenInterestAnnual) - && (market == Market.HKFE || market == Market.EUREX || market == Market.ICE)) + if (!universeSettings.DataMappingMode.IsAvailableForFutureMarket(market)) { // circle around default OI for currently no OI available data return DataMappingMode.LastTradingDay; @@ -3960,6 +3958,15 @@ public static DataMappingMode GetUniverseMappingModeOrDefault(this UniverseSetti } } + /// + /// Determines whether there is mapping data for the given data mapping mode in the given future market + /// + public static bool IsAvailableForFutureMarket(this DataMappingMode dataMappingMode, string market) + { + return dataMappingMode != DataMappingMode.OpenInterest && dataMappingMode != DataMappingMode.OpenInterestAnnual + || market != Market.HKFE && market != Market.EUREX && market != Market.ICE && market != Market.KRX; + } + /// /// Helper method to determine the right data normalization mode to use by default /// diff --git a/Tests/Algorithm/AlgorithmAddSecurityTests.cs b/Tests/Algorithm/AlgorithmAddSecurityTests.cs index 28cedbefcfc8..0e3775dbb19c 100644 --- a/Tests/Algorithm/AlgorithmAddSecurityTests.cs +++ b/Tests/Algorithm/AlgorithmAddSecurityTests.cs @@ -37,6 +37,7 @@ using QuantConnect.Securities.IndexOption; using QuantConnect.Securities.Option; using QuantConnect.Tests.Engine.DataFeeds; +using QuantConnect.Tests.Engine.HistoricalData; using System; using System.Collections.Generic; using System.IO; @@ -147,6 +148,65 @@ public void ProperlyAddsFutureWithExtendedMarketHours( Has.All.EqualTo(extendedMarketHours)); } + [TestCase("KM", Market.KRX, DataMappingMode.LastTradingDay, true)] + [TestCase("FESX", Market.EUREX, DataMappingMode.LastTradingDay, true)] + [TestCase("ES", Market.CME, DataMappingMode.OpenInterest, false)] + public void AddFutureDefaultsToAvailableDataMappingMode(string ticker, string market, DataMappingMode expectedMode, bool expectWarning) + { + var future = _algo.AddFuture(ticker, Resolution.Daily, market); + // a second fallback does not repeat the warning + _algo.AddFuture("HSI", Resolution.Daily, Market.HKFE); + + var continuousConfigs = _algo.SubscriptionManager.SubscriptionDataConfigService + .GetSubscriptionDataConfigs(future.Symbol, includeInternalConfigs: true) + .Where(x => x.Type != typeof(FutureUniverse)) + .ToList(); + Assert.Greater(continuousConfigs.Count, 0); + Assert.That(continuousConfigs.Select(x => x.DataMappingMode), Has.All.EqualTo(expectedMode)); + + var warnings = _algo.DebugMessages.Where(x => x.Contains("data mapping mode is not available")).ToList(); + Assert.AreEqual(1, warnings.Count); + var expectedMarket = expectWarning ? market : Market.HKFE; + Assert.That(warnings[0], Does.EndWith($"Warning: OpenInterest data mapping mode is not available for {expectedMarket.ToUpperInvariant()} futures, using LastTradingDay instead.")); + } + + [TestCase("KM", Market.KRX, DataMappingMode.OpenInterest, true)] + [TestCase("FESX", Market.EUREX, DataMappingMode.OpenInterestAnnual, true)] + [TestCase("FESX", Market.EUREX, DataMappingMode.LastTradingDay, false)] + [TestCase("ES", Market.CME, DataMappingMode.OpenInterest, false)] + public void AddFutureWithExplicitUnavailableDataMappingModeWarnsOnce(string ticker, string market, DataMappingMode dataMappingMode, bool expectWarning) + { + var future = _algo.AddFuture(ticker, Resolution.Daily, market, dataMappingMode: dataMappingMode); + _algo.AddFuture("HSI", Resolution.Daily, Market.HKFE, dataMappingMode: DataMappingMode.OpenInterest); + + var continuousConfigs = _algo.SubscriptionManager.SubscriptionDataConfigService + .GetSubscriptionDataConfigs(future.Symbol, includeInternalConfigs: true) + .Where(x => x.Type != typeof(FutureUniverse)) + .ToList(); + // the explicit mode is respected, not replaced by the fallback + Assert.That(continuousConfigs.Select(x => x.DataMappingMode), Has.All.EqualTo(dataMappingMode)); + Assert.IsFalse(_algo.DebugMessages.Any(x => x.Contains("using LastTradingDay instead."))); + + var warnings = _algo.DebugMessages.Where(x => x.Contains("no contract will be mapped")).ToList(); + Assert.AreEqual(1, warnings.Count); + var expected = expectWarning + ? $"Warning: {dataMappingMode} data mapping mode is not available for {market.ToUpperInvariant()} futures, no contract will be mapped. Use LastTradingDay instead." + : "Warning: OpenInterest data mapping mode is not available for HKFE futures, no contract will be mapped. Use LastTradingDay instead."; + Assert.That(warnings[0], Does.EndWith(expected)); + } + + [Test] + public void HistoryWithExplicitUnavailableDataMappingModeWarnsOnce() + { + var future = _algo.AddFuture("FESX", Resolution.Daily, Market.EUREX); + _algo.HistoryProvider = new TestHistoryProvider(); + + _algo.History(future.Symbol, 5, Resolution.Daily, dataMappingMode: DataMappingMode.OpenInterest).ToList(); + _algo.History(future.Symbol, 5, Resolution.Daily, dataMappingMode: DataMappingMode.OpenInterest).ToList(); + + Assert.AreEqual(1, _algo.DebugMessages.Count(x => x.Contains("no contract will be mapped"))); + } + [TestCaseSource(nameof(FuturesTestCases))] public void AddFutureWithExtendedMarketHours(Func getFuture) { diff --git a/Tests/Common/Util/ExtensionsTests.cs b/Tests/Common/Util/ExtensionsTests.cs index 696511e1bf21..19bad6fcfbe1 100644 --- a/Tests/Common/Util/ExtensionsTests.cs +++ b/Tests/Common/Util/ExtensionsTests.cs @@ -62,6 +62,21 @@ public void GetEnumValuesInValue(string saleCondition, params TradeConditionFlag Assert.AreEqual(expected, enums); } + [TestCase(Market.CME, DataMappingMode.OpenInterest, DataMappingMode.OpenInterest)] + [TestCase(Market.CME, DataMappingMode.OpenInterestAnnual, DataMappingMode.OpenInterestAnnual)] + [TestCase(Market.EUREX, DataMappingMode.OpenInterest, DataMappingMode.LastTradingDay)] + [TestCase(Market.HKFE, DataMappingMode.OpenInterest, DataMappingMode.LastTradingDay)] + [TestCase(Market.ICE, DataMappingMode.OpenInterest, DataMappingMode.LastTradingDay)] + [TestCase(Market.KRX, DataMappingMode.OpenInterest, DataMappingMode.LastTradingDay)] + [TestCase(Market.KRX, DataMappingMode.OpenInterestAnnual, DataMappingMode.LastTradingDay)] + [TestCase(Market.KRX, DataMappingMode.FirstDayMonth, DataMappingMode.FirstDayMonth)] + public void GetUniverseMappingModeOrDefault(string market, DataMappingMode settingsMode, DataMappingMode expected) + { + var settings = new UniverseSettings(Resolution.Daily, 1, true, false, TimeSpan.Zero) { DataMappingMode = settingsMode }; + + Assert.AreEqual(expected, settings.GetUniverseMappingModeOrDefault(SecurityType.Future, market)); + } + [TestCase("tt", "", "tt")] [TestCase("tt", "t", "t")] [TestCase("tt", "tt", "")] From 3448be9acb281671248098b835b95b0c3de136ea Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Thu, 24 Sep 2026 14:44:54 -0400 Subject: [PATCH 2/7] Warn when a future is re-added with different continuous contract settings --- Algorithm/QCAlgorithm.cs | 38 ++++++++++++++++++ Tests/Algorithm/AlgorithmAddSecurityTests.cs | 41 ++++++++++++++++++++ 2 files changed, 79 insertions(+) diff --git a/Algorithm/QCAlgorithm.cs b/Algorithm/QCAlgorithm.cs index 19ec00236944..96b719fcd2b9 100644 --- a/Algorithm/QCAlgorithm.cs +++ b/Algorithm/QCAlgorithm.cs @@ -155,6 +155,7 @@ public partial class QCAlgorithm : MarshalByRefObject, IAlgorithm private bool _deploymentDetailsSet; private bool _dataMappingModeFallbackWarningSent; private bool _unavailableDataMappingModeWarningSent; + private bool _ignoredContinuousFutureSettingsWarningSent; private SecurityDefinitionSymbolResolver _securityDefinitionSymbolResolver; private SecurityDefinitionSymbolResolver SecurityDefinitionSymbolResolver @@ -2131,6 +2132,11 @@ public Security AddSecurity(Symbol symbol, Resolution? resolution = null, bool? AddUniverse(universe); } + else if (symbol.SecurityType == SecurityType.Future + && UniverseManager.TryGetValue(ContinuousContractUniverse.CreateSymbol(symbol), out var continuousUniverse)) + { + WarnIfContinuousFutureSettingsIgnored(symbol, continuousUniverse.UniverseSettings, dataMappingMode, dataNormalizationMode, (int)contractOffset); + } return security; } @@ -2164,6 +2170,38 @@ private void WarnIfDataMappingModeUnavailable(Symbol symbol, DataMappingMode? da } } + /// + /// Warns once if a future is added again with continuous contract settings that differ from the existing ones, which are kept + /// + private void WarnIfContinuousFutureSettingsIgnored(Symbol symbol, UniverseSettings existingSettings, DataMappingMode? dataMappingMode, + DataNormalizationMode? dataNormalizationMode, int contractDepthOffset) + { + if (_ignoredContinuousFutureSettingsWarningSent) + { + return; + } + + var ignoredSettings = new List(); + if (dataMappingMode.HasValue && dataMappingMode != existingSettings.DataMappingMode) + { + ignoredSettings.Add($"data mapping mode {dataMappingMode} (keeping {existingSettings.DataMappingMode})"); + } + if (dataNormalizationMode.HasValue && dataNormalizationMode != existingSettings.DataNormalizationMode) + { + ignoredSettings.Add($"data normalization mode {dataNormalizationMode} (keeping {existingSettings.DataNormalizationMode})"); + } + if (contractDepthOffset != existingSettings.ContractDepthOffset) + { + ignoredSettings.Add($"contract depth offset {contractDepthOffset} (keeping {existingSettings.ContractDepthOffset})"); + } + + if (ignoredSettings.Count > 0) + { + _ignoredContinuousFutureSettingsWarningSent = true; + Debug($"Warning: {symbol} was already added, ignoring the requested {string.Join(", ", ignoredSettings)}."); + } + } + /// /// Creates and adds a new security to the algorithm /// diff --git a/Tests/Algorithm/AlgorithmAddSecurityTests.cs b/Tests/Algorithm/AlgorithmAddSecurityTests.cs index 0e3775dbb19c..ef6f5a04cd69 100644 --- a/Tests/Algorithm/AlgorithmAddSecurityTests.cs +++ b/Tests/Algorithm/AlgorithmAddSecurityTests.cs @@ -195,6 +195,47 @@ public void AddFutureWithExplicitUnavailableDataMappingModeWarnsOnce(string tick Assert.That(warnings[0], Does.EndWith(expected)); } + [Test] + public void AddFutureAgainWithDifferentSettingsWarnsOnce() + { + var future = _algo.AddFuture(Futures.Indices.SP500EMini, Resolution.Daily, dataMappingMode: DataMappingMode.OpenInterest); + _algo.AddFuture(Futures.Indices.SP500EMini, Resolution.Daily, dataMappingMode: DataMappingMode.LastTradingDay); + _algo.AddFuture(Futures.Indices.SP500EMini, Resolution.Daily, dataMappingMode: DataMappingMode.FirstDayMonth); + + var continuousConfigs = _algo.SubscriptionManager.SubscriptionDataConfigService + .GetSubscriptionDataConfigs(future.Symbol, includeInternalConfigs: true) + .Where(x => x.Type != typeof(FutureUniverse)) + .ToList(); + Assert.That(continuousConfigs.Select(x => x.DataMappingMode), Has.All.EqualTo(DataMappingMode.OpenInterest)); + + var warnings = _algo.DebugMessages.Where(x => x.Contains("was already added")).ToList(); + Assert.AreEqual(1, warnings.Count); + Assert.That(warnings[0], Does.EndWith("Warning: /ES was already added, ignoring the requested data mapping mode LastTradingDay (keeping OpenInterest).")); + } + + [Test] + public void AddFutureAgainWithAllSettingsDifferentListsThemInOneWarning() + { + _algo.AddFuture(Futures.Indices.SP500EMini, Resolution.Daily); + _algo.AddFuture(Futures.Indices.SP500EMini, Resolution.Daily, dataMappingMode: DataMappingMode.LastTradingDay, + dataNormalizationMode: DataNormalizationMode.Raw, contractDepthOffset: 1); + + var warnings = _algo.DebugMessages.Where(x => x.Contains("was already added")).ToList(); + Assert.AreEqual(1, warnings.Count); + Assert.That(warnings[0], Does.EndWith("Warning: /ES was already added, ignoring the requested data mapping mode LastTradingDay (keeping OpenInterest), " + + "data normalization mode Raw (keeping BackwardsRatio), contract depth offset 1 (keeping 0).")); + } + + [TestCase(null)] + [TestCase(DataMappingMode.OpenInterest)] + public void AddFutureAgainWithSameSettingsDoesNotWarn(DataMappingMode? dataMappingMode) + { + _algo.AddFuture(Futures.Indices.SP500EMini, Resolution.Daily, dataMappingMode: DataMappingMode.OpenInterest); + _algo.AddFuture(Futures.Indices.SP500EMini, Resolution.Daily, dataMappingMode: dataMappingMode); + + Assert.IsFalse(_algo.DebugMessages.Any(x => x.Contains("was already added"))); + } + [Test] public void HistoryWithExplicitUnavailableDataMappingModeWarnsOnce() { From edb7e27ca731d5b5d3458b3e8cf17d95d18dc534 Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Fri, 25 Sep 2026 09:19:43 -0400 Subject: [PATCH 3/7] Fall back from explicit unavailable mapping modes per market and add re-add instructions --- Algorithm/QCAlgorithm.History.cs | 2 +- Algorithm/QCAlgorithm.cs | 34 +++--- Tests/Algorithm/AlgorithmAddSecurityTests.cs | 105 ++++++++++++++----- 3 files changed, 100 insertions(+), 41 deletions(-) diff --git a/Algorithm/QCAlgorithm.History.cs b/Algorithm/QCAlgorithm.History.cs index f2b671a21826..e036e2d927e0 100644 --- a/Algorithm/QCAlgorithm.History.cs +++ b/Algorithm/QCAlgorithm.History.cs @@ -1296,7 +1296,7 @@ private IEnumerable GetMatchingSubscriptions(Symbol symb // Inherit values from existing subscriptions or use defaults var extendedMarketHours = userConfigIfAny?.ExtendedMarketHours ?? UniverseSettings.ExtendedMarketHours; var dataNormalizationMode = userConfigIfAny?.DataNormalizationMode ?? UniverseSettings.GetUniverseNormalizationModeOrDefault(symbol.SecurityType); - var dataMappingMode = userConfigIfAny?.DataMappingMode ?? GetUniverseMappingModeOrDefault(symbol); + var dataMappingMode = userConfigIfAny?.DataMappingMode ?? GetDataMappingModeOrDefault(symbol); var contractDepthOffset = userConfigIfAny?.ContractDepthOffset ?? (uint)Math.Abs(UniverseSettings.ContractDepthOffset); // If type was specified and not a lean data type and also not abstract, we create a new subscription diff --git a/Algorithm/QCAlgorithm.cs b/Algorithm/QCAlgorithm.cs index 96b719fcd2b9..c3c83950d7ff 100644 --- a/Algorithm/QCAlgorithm.cs +++ b/Algorithm/QCAlgorithm.cs @@ -153,7 +153,7 @@ public partial class QCAlgorithm : MarshalByRefObject, IAlgorithm private int? _warmupBarCount; private Dictionary _parameters = new Dictionary(); private bool _deploymentDetailsSet; - private bool _dataMappingModeFallbackWarningSent; + private readonly HashSet _dataMappingModeFallbackWarnedMarkets = new(); private bool _unavailableDataMappingModeWarningSent; private bool _ignoredContinuousFutureSettingsWarningSent; private SecurityDefinitionSymbolResolver _securityDefinitionSymbolResolver; @@ -2098,8 +2098,6 @@ public Security AddSecurity(Symbol symbol, Resolution? resolution = null, bool? } else { - WarnIfDataMappingModeUnavailable(symbol, dataMappingMode); - // add the expected configurations of the canonical symbol right away, will allow it to warmup and indicators register to them var dataTypes = SubscriptionManager.LookupSubscriptionConfigDataTypes(SecurityType.Future, GetResolution(symbol, resolution, null), isCanonical: false); @@ -2107,7 +2105,7 @@ public Security AddSecurity(Symbol symbol, Resolution? resolution = null, bool? { ExtendedMarketHours = extendedMarketHours.Value, FillForward = fillForward.Value, - DataMappingMode = dataMappingMode ?? GetUniverseMappingModeOrDefault(symbol), + DataMappingMode = GetDataMappingModeOrDefault(symbol, dataMappingMode), DataNormalizationMode = dataNormalizationMode ?? UniverseSettings.GetUniverseNormalizationModeOrDefault(symbol.SecurityType), ContractDepthOffset = (int)contractOffset, SubscriptionDataTypes = dataTypes, @@ -2135,7 +2133,8 @@ public Security AddSecurity(Symbol symbol, Resolution? resolution = null, bool? else if (symbol.SecurityType == SecurityType.Future && UniverseManager.TryGetValue(ContinuousContractUniverse.CreateSymbol(symbol), out var continuousUniverse)) { - WarnIfContinuousFutureSettingsIgnored(symbol, continuousUniverse.UniverseSettings, dataMappingMode, dataNormalizationMode, (int)contractOffset); + var requestedDataMappingMode = dataMappingMode.HasValue ? GetDataMappingModeOrDefault(symbol, dataMappingMode) : (DataMappingMode?)null; + WarnIfContinuousFutureSettingsIgnored(symbol, continuousUniverse.UniverseSettings, requestedDataMappingMode, dataNormalizationMode, (int)contractOffset); } return security; } @@ -2144,17 +2143,23 @@ public Security AddSecurity(Symbol symbol, Resolution? resolution = null, bool? } /// - /// Gets the default data mapping mode for the given symbol, warning once if the universe settings mode is not available for its market + /// Gets the requested or default data mapping mode for the given symbol, falling back to the market default if it is not available, + /// warning once per market when it does /// - private DataMappingMode GetUniverseMappingModeOrDefault(Symbol symbol) + private DataMappingMode GetDataMappingModeOrDefault(Symbol symbol, DataMappingMode? dataMappingMode = null) { - var dataMappingMode = UniverseSettings.GetUniverseMappingModeOrDefault(symbol.SecurityType, symbol.ID.Market); - if (dataMappingMode != UniverseSettings.DataMappingMode && !_dataMappingModeFallbackWarningSent) + var requestedDataMappingMode = dataMappingMode ?? UniverseSettings.DataMappingMode; + if (symbol.SecurityType != SecurityType.Future || requestedDataMappingMode.IsAvailableForFutureMarket(symbol.ID.Market)) + { + return requestedDataMappingMode; + } + + var fallbackDataMappingMode = UniverseSettings.GetUniverseMappingModeOrDefault(symbol.SecurityType, symbol.ID.Market); + if (_dataMappingModeFallbackWarnedMarkets.Add(symbol.ID.Market)) { - _dataMappingModeFallbackWarningSent = true; - Debug($"Warning: {UniverseSettings.DataMappingMode} data mapping mode is not available for {symbol.ID.Market.ToUpperInvariant()} futures, using {dataMappingMode} instead."); + Debug($"Warning: {requestedDataMappingMode} data mapping mode is not available for {symbol.ID.Market.ToUpperInvariant()} futures, using {fallbackDataMappingMode} instead."); } - return dataMappingMode; + return fallbackDataMappingMode; } /// @@ -2198,7 +2203,10 @@ private void WarnIfContinuousFutureSettingsIgnored(Symbol symbol, UniverseSettin if (ignoredSettings.Count > 0) { _ignoredContinuousFutureSettingsWarningSent = true; - Debug($"Warning: {symbol} was already added, ignoring the requested {string.Join(", ", ignoredSettings)}."); + var instructions = _locked + ? "To change these settings, remove it with RemoveSecurity() and add it again." + : "Add it only once in Initialize, or, to change these settings after Initialize, remove it with RemoveSecurity() and add it again."; + Debug($"Warning: {symbol} was already added, ignoring the requested {string.Join(", ", ignoredSettings)}. {instructions}"); } } diff --git a/Tests/Algorithm/AlgorithmAddSecurityTests.cs b/Tests/Algorithm/AlgorithmAddSecurityTests.cs index ef6f5a04cd69..91c4ffa7e0d3 100644 --- a/Tests/Algorithm/AlgorithmAddSecurityTests.cs +++ b/Tests/Algorithm/AlgorithmAddSecurityTests.cs @@ -148,14 +148,17 @@ public void ProperlyAddsFutureWithExtendedMarketHours( Has.All.EqualTo(extendedMarketHours)); } - [TestCase("KM", Market.KRX, DataMappingMode.LastTradingDay, true)] - [TestCase("FESX", Market.EUREX, DataMappingMode.LastTradingDay, true)] - [TestCase("ES", Market.CME, DataMappingMode.OpenInterest, false)] - public void AddFutureDefaultsToAvailableDataMappingMode(string ticker, string market, DataMappingMode expectedMode, bool expectWarning) + [TestCase("KM", Market.KRX, null, DataMappingMode.LastTradingDay, true)] + [TestCase("FESX", Market.EUREX, null, DataMappingMode.LastTradingDay, true)] + [TestCase("ES", Market.CME, null, DataMappingMode.OpenInterest, false)] + [TestCase("KM", Market.KRX, DataMappingMode.OpenInterest, DataMappingMode.LastTradingDay, true)] + [TestCase("FESX", Market.EUREX, DataMappingMode.OpenInterestAnnual, DataMappingMode.LastTradingDay, true)] + [TestCase("FESX", Market.EUREX, DataMappingMode.FirstDayMonth, DataMappingMode.FirstDayMonth, false)] + [TestCase("ES", Market.CME, DataMappingMode.OpenInterest, DataMappingMode.OpenInterest, false)] + public void AddFutureFallsBackFromUnavailableDataMappingMode(string ticker, string market, DataMappingMode? dataMappingMode, + DataMappingMode expectedMode, bool expectWarning) { - var future = _algo.AddFuture(ticker, Resolution.Daily, market); - // a second fallback does not repeat the warning - _algo.AddFuture("HSI", Resolution.Daily, Market.HKFE); + var future = _algo.AddFuture(ticker, Resolution.Daily, market, dataMappingMode: dataMappingMode); var continuousConfigs = _algo.SubscriptionManager.SubscriptionDataConfigService .GetSubscriptionDataConfigs(future.Symbol, includeInternalConfigs: true) @@ -165,34 +168,58 @@ public void AddFutureDefaultsToAvailableDataMappingMode(string ticker, string ma Assert.That(continuousConfigs.Select(x => x.DataMappingMode), Has.All.EqualTo(expectedMode)); var warnings = _algo.DebugMessages.Where(x => x.Contains("data mapping mode is not available")).ToList(); - Assert.AreEqual(1, warnings.Count); - var expectedMarket = expectWarning ? market : Market.HKFE; - Assert.That(warnings[0], Does.EndWith($"Warning: OpenInterest data mapping mode is not available for {expectedMarket.ToUpperInvariant()} futures, using LastTradingDay instead.")); + if (expectWarning) + { + Assert.AreEqual(1, warnings.Count); + var requestedMode = dataMappingMode ?? DataMappingMode.OpenInterest; + Assert.That(warnings[0], Does.EndWith($"Warning: {requestedMode} data mapping mode is not available for {market.ToUpperInvariant()} futures, using LastTradingDay instead.")); + } + else + { + Assert.IsEmpty(warnings); + } } - [TestCase("KM", Market.KRX, DataMappingMode.OpenInterest, true)] - [TestCase("FESX", Market.EUREX, DataMappingMode.OpenInterestAnnual, true)] - [TestCase("FESX", Market.EUREX, DataMappingMode.LastTradingDay, false)] - [TestCase("ES", Market.CME, DataMappingMode.OpenInterest, false)] - public void AddFutureWithExplicitUnavailableDataMappingModeWarnsOnce(string ticker, string market, DataMappingMode dataMappingMode, bool expectWarning) + [Test] + public void AddFutureDataMappingModeFallbackWarnsOncePerMarket() { - var future = _algo.AddFuture(ticker, Resolution.Daily, market, dataMappingMode: dataMappingMode); + _algo.AddFuture("FESX", Resolution.Daily, Market.EUREX, dataMappingMode: DataMappingMode.OpenInterest); + _algo.AddFuture("FDAX", Resolution.Daily, Market.EUREX); _algo.AddFuture("HSI", Resolution.Daily, Market.HKFE, dataMappingMode: DataMappingMode.OpenInterest); + _algo.AddFuture("KM", Resolution.Daily, Market.KRX); + + var warnings = _algo.DebugMessages.Where(x => x.Contains("data mapping mode is not available")).ToList(); + Assert.AreEqual(3, warnings.Count); + Assert.That(warnings[0], Does.EndWith("Warning: OpenInterest data mapping mode is not available for EUREX futures, using LastTradingDay instead.")); + Assert.That(warnings[1], Does.EndWith("Warning: OpenInterest data mapping mode is not available for HKFE futures, using LastTradingDay instead.")); + Assert.That(warnings[2], Does.EndWith("Warning: OpenInterest data mapping mode is not available for KRX futures, using LastTradingDay instead.")); + } + + [Test] + public void AddFutureFallsBackToUniverseSettingsDataMappingMode() + { + _algo.UniverseSettings.DataMappingMode = DataMappingMode.FirstDayMonth; + var future = _algo.AddFuture("FESX", Resolution.Daily, Market.EUREX, dataMappingMode: DataMappingMode.OpenInterest); var continuousConfigs = _algo.SubscriptionManager.SubscriptionDataConfigService .GetSubscriptionDataConfigs(future.Symbol, includeInternalConfigs: true) .Where(x => x.Type != typeof(FutureUniverse)) .ToList(); - // the explicit mode is respected, not replaced by the fallback - Assert.That(continuousConfigs.Select(x => x.DataMappingMode), Has.All.EqualTo(dataMappingMode)); - Assert.IsFalse(_algo.DebugMessages.Any(x => x.Contains("using LastTradingDay instead."))); + Assert.That(continuousConfigs.Select(x => x.DataMappingMode), Has.All.EqualTo(DataMappingMode.FirstDayMonth)); + Assert.That(_algo.DebugMessages.Single(x => x.Contains("data mapping mode is not available")), + Does.EndWith("Warning: OpenInterest data mapping mode is not available for EUREX futures, using FirstDayMonth instead.")); + } - var warnings = _algo.DebugMessages.Where(x => x.Contains("no contract will be mapped")).ToList(); - Assert.AreEqual(1, warnings.Count); - var expected = expectWarning - ? $"Warning: {dataMappingMode} data mapping mode is not available for {market.ToUpperInvariant()} futures, no contract will be mapped. Use LastTradingDay instead." - : "Warning: OpenInterest data mapping mode is not available for HKFE futures, no contract will be mapped. Use LastTradingDay instead."; - Assert.That(warnings[0], Does.EndWith(expected)); + [Test] + public void AddFutureAfterRemovingItAppliesNewSettings() + { + var future = _algo.AddFuture(Futures.Indices.SP500EMini, Resolution.Daily, dataMappingMode: DataMappingMode.OpenInterest); + _algo.RemoveSecurity(future.Symbol); + future = _algo.AddFuture(Futures.Indices.SP500EMini, Resolution.Daily, dataMappingMode: DataMappingMode.LastTradingDay); + + var continuousUniverse = _algo.UniverseManager[ContinuousContractUniverse.CreateSymbol(future.Symbol)]; + Assert.AreEqual(DataMappingMode.LastTradingDay, continuousUniverse.UniverseSettings.DataMappingMode); + Assert.IsFalse(_algo.DebugMessages.Any(x => x.Contains("was already added"))); } [Test] @@ -210,7 +237,8 @@ public void AddFutureAgainWithDifferentSettingsWarnsOnce() var warnings = _algo.DebugMessages.Where(x => x.Contains("was already added")).ToList(); Assert.AreEqual(1, warnings.Count); - Assert.That(warnings[0], Does.EndWith("Warning: /ES was already added, ignoring the requested data mapping mode LastTradingDay (keeping OpenInterest).")); + Assert.That(warnings[0], Does.EndWith("Warning: /ES was already added, ignoring the requested data mapping mode LastTradingDay (keeping OpenInterest). " + + "Add it only once in Initialize, or, to change these settings after Initialize, remove it with RemoveSecurity() and add it again.")); } [Test] @@ -223,7 +251,20 @@ public void AddFutureAgainWithAllSettingsDifferentListsThemInOneWarning() var warnings = _algo.DebugMessages.Where(x => x.Contains("was already added")).ToList(); Assert.AreEqual(1, warnings.Count); Assert.That(warnings[0], Does.EndWith("Warning: /ES was already added, ignoring the requested data mapping mode LastTradingDay (keeping OpenInterest), " + - "data normalization mode Raw (keeping BackwardsRatio), contract depth offset 1 (keeping 0).")); + "data normalization mode Raw (keeping BackwardsRatio), contract depth offset 1 (keeping 0). " + + "Add it only once in Initialize, or, to change these settings after Initialize, remove it with RemoveSecurity() and add it again.")); + } + + [Test] + public void AddFutureAgainAfterInitializeWarnsToRemoveItFirst() + { + _algo.AddFuture(Futures.Indices.SP500EMini, Resolution.Daily, dataMappingMode: DataMappingMode.OpenInterest); + _algo.SetLocked(); + _algo.AddFuture(Futures.Indices.SP500EMini, Resolution.Daily, dataMappingMode: DataMappingMode.LastTradingDay); + + Assert.That(_algo.DebugMessages.Single(x => x.Contains("was already added")), + Does.EndWith("Warning: /ES was already added, ignoring the requested data mapping mode LastTradingDay (keeping OpenInterest). " + + "To change these settings, remove it with RemoveSecurity() and add it again.")); } [TestCase(null)] @@ -236,6 +277,16 @@ public void AddFutureAgainWithSameSettingsDoesNotWarn(DataMappingMode? dataMappi Assert.IsFalse(_algo.DebugMessages.Any(x => x.Contains("was already added"))); } + [Test] + public void AddFutureAgainWithUnavailableDataMappingModeResolvingToSameModeDoesNotWarn() + { + _algo.AddFuture("FESX", Resolution.Daily, Market.EUREX); + _algo.AddFuture("FESX", Resolution.Daily, Market.EUREX, dataMappingMode: DataMappingMode.OpenInterest); + + Assert.IsFalse(_algo.DebugMessages.Any(x => x.Contains("was already added"))); + Assert.AreEqual(1, _algo.DebugMessages.Count(x => x.Contains("data mapping mode is not available"))); + } + [Test] public void HistoryWithExplicitUnavailableDataMappingModeWarnsOnce() { From 322920992dc5da9c23bdfd54f45ec26a74039829 Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Fri, 25 Sep 2026 09:47:44 -0400 Subject: [PATCH 4/7] Add data mapping mode fallback regression algorithm and tighten tests --- ...aMappingModeFallbackRegressionAlgorithm.cs | 163 ++++++++++++++++++ ...aMappingModeFallbackRegressionAlgorithm.py | 65 +++++++ Tests/Algorithm/AlgorithmAddSecurityTests.cs | 14 +- Tests/Algorithm/AlgorithmHistoryTests.cs | 16 ++ 4 files changed, 245 insertions(+), 13 deletions(-) create mode 100644 Algorithm.CSharp/FutureDataMappingModeFallbackRegressionAlgorithm.cs create mode 100644 Algorithm.Python/FutureDataMappingModeFallbackRegressionAlgorithm.py diff --git a/Algorithm.CSharp/FutureDataMappingModeFallbackRegressionAlgorithm.cs b/Algorithm.CSharp/FutureDataMappingModeFallbackRegressionAlgorithm.cs new file mode 100644 index 000000000000..4f86cbab6c19 --- /dev/null +++ b/Algorithm.CSharp/FutureDataMappingModeFallbackRegressionAlgorithm.cs @@ -0,0 +1,163 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ + +using System.Collections.Generic; +using System.Linq; +using QuantConnect.Data; +using QuantConnect.Interfaces; +using QuantConnect.Securities; +using QuantConnect.Securities.Future; + +namespace QuantConnect.Algorithm.CSharp +{ + /// + /// Regression algorithm asserting that a future added with a data mapping mode its market has no data for (open interest on EUREX) + /// falls back to the market default, and that the related warnings are sent. + /// + public class FutureDataMappingModeFallbackRegressionAlgorithm : QCAlgorithm, IRegressionAlgorithmDefinition + { + private Future _future; + private bool _checkedAfterInitialize; + + public override void Initialize() + { + SetStartDate(2024, 6, 3); + SetEndDate(2024, 6, 4); + SetAccountCurrency(Currencies.EUR); + + _future = AddFuture(Futures.Indices.EuroStoxx50, Resolution.Minute, dataMappingMode: DataMappingMode.OpenInterest); + AssertWarning("Warning: OpenInterest data mapping mode is not available for EUREX futures, using LastTradingDay instead."); + + // Open interest resolves to the mode already in use, so this is not a conflicting re-add + AddFuture(Futures.Indices.EuroStoxx50, Resolution.Minute, dataMappingMode: DataMappingMode.LastTradingDay); + if (DebugMessages.Any(message => message.Contains("was already added"))) + { + throw new RegressionTestException("Unexpected re-add warning for a future added again with the same data mapping mode"); + } + } + + public override void OnData(Slice slice) + { + if (_checkedAfterInitialize || _future.Mapped == null) + { + return; + } + _checkedAfterInitialize = true; + + // Last trading day maps to the June contract, first day of the month would map to September + if (_future.Mapped.ID.Date.Month != 6) + { + throw new RegressionTestException($"Unexpected mapped contract {_future.Mapped}, expected the June contract"); + } + + if (History(_future.Symbol, 10, Resolution.Minute).Count() == 0) + { + throw new RegressionTestException("Expected history for the continuous future using the fallback data mapping mode"); + } + + var openInterestHistory = History(_future.Symbol, 10, Resolution.Minute, dataMappingMode: DataMappingMode.OpenInterest); + if (openInterestHistory.Any()) + { + throw new RegressionTestException("Expected no history for the explicitly requested open interest data mapping mode"); + } + AssertWarning("Warning: OpenInterest data mapping mode is not available for EUREX futures, no contract will be mapped. Use LastTradingDay instead."); + + AddFuture(Futures.Indices.EuroStoxx50, Resolution.Minute, dataMappingMode: DataMappingMode.FirstDayMonth); + AssertWarning("Warning: /FESX was already added, ignoring the requested data mapping mode FirstDayMonth (keeping LastTradingDay). " + + "To change these settings, remove it with RemoveSecurity() and add it again."); + } + + public override void OnEndOfAlgorithm() + { + if (!_checkedAfterInitialize) + { + throw new RegressionTestException("The continuous future was never mapped"); + } + if (_future.Mapped.ID.Date.Month != 6) + { + throw new RegressionTestException($"Unexpected mapped contract {_future.Mapped} after the ignored re-add, expected the June contract"); + } + } + + private void AssertWarning(string warning) + { + if (DebugMessages.Count(message => message.EndsWith(warning)) != 1) + { + throw new RegressionTestException($"Expected the warning '{warning}' to be sent once"); + } + } + + /// + /// This is used by the regression test system to indicate if the open source Lean repository has the required data to run this algorithm. + /// + public bool CanRunLocally { get; } = true; + + /// + /// This is used by the regression test system to indicate which languages this algorithm is written in. + /// + public List Languages { get; } = new() { Language.CSharp, Language.Python }; + + /// + /// Data Points count of all timeslices of algorithm + /// + public long DataPoints => 5010; + + /// + /// Data Points count of the algorithm history + /// + public int AlgorithmHistoryDataPoints => 10; + + /// + /// Final status of the algorithm + /// + public AlgorithmStatus AlgorithmStatus => AlgorithmStatus.Completed; + + /// + /// This is used by the regression test system to indicate what the expected statistics are from running the algorithm + /// + public Dictionary ExpectedStatistics => new Dictionary + { + {"Total Orders", "0"}, + {"Average Win", "0%"}, + {"Average Loss", "0%"}, + {"Compounding Annual Return", "0%"}, + {"Drawdown", "0%"}, + {"Expectancy", "0"}, + {"Start Equity", "100000"}, + {"End Equity", "100000"}, + {"Net Profit", "0%"}, + {"Sharpe Ratio", "0"}, + {"Sortino Ratio", "0"}, + {"Probabilistic Sharpe Ratio", "0%"}, + {"Loss Rate", "0%"}, + {"Win Rate", "0%"}, + {"Profit-Loss Ratio", "0"}, + {"Alpha", "0"}, + {"Beta", "0"}, + {"Annual Standard Deviation", "0"}, + {"Annual Variance", "0"}, + {"Information Ratio", "0"}, + {"Tracking Error", "0"}, + {"Treynor Ratio", "0"}, + {"Total Fees", "€0.00"}, + {"Estimated Strategy Capacity", "€0"}, + {"Lowest Capacity Asset", ""}, + {"Portfolio Turnover", "0%"}, + {"Drawdown Recovery", "0"}, + {"OrderListHash", "d41d8cd98f00b204e9800998ecf8427e"}, + }; + } +} diff --git a/Algorithm.Python/FutureDataMappingModeFallbackRegressionAlgorithm.py b/Algorithm.Python/FutureDataMappingModeFallbackRegressionAlgorithm.py new file mode 100644 index 000000000000..0ce351849341 --- /dev/null +++ b/Algorithm.Python/FutureDataMappingModeFallbackRegressionAlgorithm.py @@ -0,0 +1,65 @@ +# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. +# Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from AlgorithmImports import * + +### +### Regression algorithm asserting that a future added with a data mapping mode its market has no data for (open interest on EUREX) +### falls back to the market default, and that the related warnings are sent. +### +class FutureDataMappingModeFallbackRegressionAlgorithm(QCAlgorithm): + def initialize(self): + self.set_start_date(2024, 6, 3) + self.set_end_date(2024, 6, 4) + self.set_account_currency(Currencies.EUR) + + self._future = self.add_future(Futures.Indices.EURO_STOXX_50, Resolution.MINUTE, data_mapping_mode=DataMappingMode.OPEN_INTEREST) + self._assert_warning("Warning: OpenInterest data mapping mode is not available for EUREX futures, using LastTradingDay instead.") + + # Open interest resolves to the mode already in use, so this is not a conflicting re-add + self.add_future(Futures.Indices.EURO_STOXX_50, Resolution.MINUTE, data_mapping_mode=DataMappingMode.LAST_TRADING_DAY) + if any("was already added" in message for message in self.debug_messages): + raise AssertionError("Unexpected re-add warning for a future added again with the same data mapping mode") + + self._checked_after_initialize = False + + def on_data(self, slice): + if self._checked_after_initialize or self._future.mapped is None: + return + self._checked_after_initialize = True + + # Last trading day maps to the June contract, first day of the month would map to September + if self._future.mapped.id.date.month != 6: + raise AssertionError(f"Unexpected mapped contract {self._future.mapped}, expected the June contract") + + if self.history(self._future.symbol, 10, Resolution.MINUTE).empty: + raise AssertionError("Expected history for the continuous future using the fallback data mapping mode") + + open_interest_history = self.history(self._future.symbol, 10, Resolution.MINUTE, data_mapping_mode=DataMappingMode.OPEN_INTEREST) + if not open_interest_history.empty: + raise AssertionError("Expected no history for the explicitly requested open interest data mapping mode") + self._assert_warning("Warning: OpenInterest data mapping mode is not available for EUREX futures, no contract will be mapped. Use LastTradingDay instead.") + + self.add_future(Futures.Indices.EURO_STOXX_50, Resolution.MINUTE, data_mapping_mode=DataMappingMode.FIRST_DAY_MONTH) + self._assert_warning("Warning: /FESX was already added, ignoring the requested data mapping mode FirstDayMonth (keeping LastTradingDay). " + "To change these settings, remove it with RemoveSecurity() and add it again.") + + def on_end_of_algorithm(self): + if not self._checked_after_initialize: + raise AssertionError("The continuous future was never mapped") + if self._future.mapped.id.date.month != 6: + raise AssertionError(f"Unexpected mapped contract {self._future.mapped} after the ignored re-add, expected the June contract") + + def _assert_warning(self, warning): + if sum(1 for message in self.debug_messages if message.endswith(warning)) != 1: + raise AssertionError(f"Expected the warning '{warning}' to be sent once") diff --git a/Tests/Algorithm/AlgorithmAddSecurityTests.cs b/Tests/Algorithm/AlgorithmAddSecurityTests.cs index 91c4ffa7e0d3..c91f42e186b6 100644 --- a/Tests/Algorithm/AlgorithmAddSecurityTests.cs +++ b/Tests/Algorithm/AlgorithmAddSecurityTests.cs @@ -37,7 +37,6 @@ using QuantConnect.Securities.IndexOption; using QuantConnect.Securities.Option; using QuantConnect.Tests.Engine.DataFeeds; -using QuantConnect.Tests.Engine.HistoricalData; using System; using System.Collections.Generic; using System.IO; @@ -152,6 +151,7 @@ public void ProperlyAddsFutureWithExtendedMarketHours( [TestCase("FESX", Market.EUREX, null, DataMappingMode.LastTradingDay, true)] [TestCase("ES", Market.CME, null, DataMappingMode.OpenInterest, false)] [TestCase("KM", Market.KRX, DataMappingMode.OpenInterest, DataMappingMode.LastTradingDay, true)] + [TestCase("FESX", Market.EUREX, DataMappingMode.OpenInterest, DataMappingMode.LastTradingDay, true)] [TestCase("FESX", Market.EUREX, DataMappingMode.OpenInterestAnnual, DataMappingMode.LastTradingDay, true)] [TestCase("FESX", Market.EUREX, DataMappingMode.FirstDayMonth, DataMappingMode.FirstDayMonth, false)] [TestCase("ES", Market.CME, DataMappingMode.OpenInterest, DataMappingMode.OpenInterest, false)] @@ -287,18 +287,6 @@ public void AddFutureAgainWithUnavailableDataMappingModeResolvingToSameModeDoesN Assert.AreEqual(1, _algo.DebugMessages.Count(x => x.Contains("data mapping mode is not available"))); } - [Test] - public void HistoryWithExplicitUnavailableDataMappingModeWarnsOnce() - { - var future = _algo.AddFuture("FESX", Resolution.Daily, Market.EUREX); - _algo.HistoryProvider = new TestHistoryProvider(); - - _algo.History(future.Symbol, 5, Resolution.Daily, dataMappingMode: DataMappingMode.OpenInterest).ToList(); - _algo.History(future.Symbol, 5, Resolution.Daily, dataMappingMode: DataMappingMode.OpenInterest).ToList(); - - Assert.AreEqual(1, _algo.DebugMessages.Count(x => x.Contains("no contract will be mapped"))); - } - [TestCaseSource(nameof(FuturesTestCases))] public void AddFutureWithExtendedMarketHours(Func getFuture) { diff --git a/Tests/Algorithm/AlgorithmHistoryTests.cs b/Tests/Algorithm/AlgorithmHistoryTests.cs index 2a2991fb1bb0..34561297506c 100644 --- a/Tests/Algorithm/AlgorithmHistoryTests.cs +++ b/Tests/Algorithm/AlgorithmHistoryTests.cs @@ -70,6 +70,22 @@ public void OneTimeSetUp() FundamentalService.Initialize(_dataProvider, new NullFundamentalDataProvider(), false); } + [Test] + public void HistoryWithExplicitUnavailableDataMappingModeKeepsItAndWarnsOnce() + { + var future = _algorithm.AddFuture("FESX", Resolution.Daily, Market.EUREX); + + _algorithm.History(future.Symbol, 5, Resolution.Daily, dataMappingMode: DataMappingMode.OpenInterest).ToList(); + _algorithm.History(future.Symbol, 5, Resolution.Daily, dataMappingMode: DataMappingMode.OpenInterest).ToList(); + + Assert.Greater(_testHistoryProvider.HistryRequests.Count, 0); + Assert.That(_testHistoryProvider.HistryRequests.Select(x => x.DataMappingMode), Has.All.EqualTo(DataMappingMode.OpenInterest)); + var warnings = _algorithm.DebugMessages.Where(x => x.Contains("no contract will be mapped")).ToList(); + Assert.AreEqual(1, warnings.Count); + Assert.That(warnings[0], Does.EndWith("Warning: OpenInterest data mapping mode is not available for EUREX futures, " + + "no contract will be mapped. Use LastTradingDay instead.")); + } + [TestCase(Language.Python)] [TestCase(Language.CSharp)] public void FundamentalHistory(Language language) From a688042bd3151e8ce4ff0342a90cf61d4f2a13e4 Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Fri, 25 Sep 2026 09:59:14 -0400 Subject: [PATCH 5/7] Move history data mapping mode warning to QCAlgorithm.History --- Algorithm/QCAlgorithm.History.cs | 14 ++++++++++++++ Algorithm/QCAlgorithm.cs | 14 -------------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/Algorithm/QCAlgorithm.History.cs b/Algorithm/QCAlgorithm.History.cs index e036e2d927e0..a987d754a8bd 100644 --- a/Algorithm/QCAlgorithm.History.cs +++ b/Algorithm/QCAlgorithm.History.cs @@ -37,6 +37,7 @@ public partial class QCAlgorithm private static readonly int SeedRetryDailyLookbackPeriod = Config.GetInt("seed-retry-daily-lookback-period", 10); private bool _dataDictionaryTickWarningSent; + private bool _unavailableDataMappingModeWarningSent; private readonly LargeHistoryRequestDiagnostics _largeHistoryRequestDiagnostics = new(); @@ -1174,6 +1175,19 @@ private IEnumerable CreateBarCountHistoryRequests(IEnumerable + /// Warns once if an explicitly requested data mapping mode has no mapping data for the future's market + /// + private void WarnIfDataMappingModeUnavailable(Symbol symbol, DataMappingMode? dataMappingMode) + { + if (dataMappingMode.HasValue && symbol.SecurityType == SecurityType.Future && !_unavailableDataMappingModeWarningSent + && !dataMappingMode.Value.IsAvailableForFutureMarket(symbol.ID.Market)) + { + _unavailableDataMappingModeWarningSent = true; + Debug($"Warning: {dataMappingMode} data mapping mode is not available for {symbol.ID.Market.ToUpperInvariant()} futures, no contract will be mapped. Use {DataMappingMode.LastTradingDay} instead."); + } + } + private int GetTickTypeOrder(SecurityType securityType, TickType tickType) { return SubscriptionManager.AvailableDataTypes[securityType].IndexOf(tickType); diff --git a/Algorithm/QCAlgorithm.cs b/Algorithm/QCAlgorithm.cs index c3c83950d7ff..a16d4c0eb191 100644 --- a/Algorithm/QCAlgorithm.cs +++ b/Algorithm/QCAlgorithm.cs @@ -154,7 +154,6 @@ public partial class QCAlgorithm : MarshalByRefObject, IAlgorithm private Dictionary _parameters = new Dictionary(); private bool _deploymentDetailsSet; private readonly HashSet _dataMappingModeFallbackWarnedMarkets = new(); - private bool _unavailableDataMappingModeWarningSent; private bool _ignoredContinuousFutureSettingsWarningSent; private SecurityDefinitionSymbolResolver _securityDefinitionSymbolResolver; @@ -2162,19 +2161,6 @@ private DataMappingMode GetDataMappingModeOrDefault(Symbol symbol, DataMappingMo return fallbackDataMappingMode; } - /// - /// Warns once if an explicitly requested data mapping mode has no mapping data for the future's market - /// - private void WarnIfDataMappingModeUnavailable(Symbol symbol, DataMappingMode? dataMappingMode) - { - if (dataMappingMode.HasValue && symbol.SecurityType == SecurityType.Future && !_unavailableDataMappingModeWarningSent - && !dataMappingMode.Value.IsAvailableForFutureMarket(symbol.ID.Market)) - { - _unavailableDataMappingModeWarningSent = true; - Debug($"Warning: {dataMappingMode} data mapping mode is not available for {symbol.ID.Market.ToUpperInvariant()} futures, no contract will be mapped. Use {DataMappingMode.LastTradingDay} instead."); - } - } - /// /// Warns once if a future is added again with continuous contract settings that differ from the existing ones, which are kept /// From 3c321d3b9eb2994fb639c228d317117e3dc20883 Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Fri, 25 Sep 2026 17:11:02 -0400 Subject: [PATCH 6/7] Shorten re-add warnings and limit history mapping mode warning to continuous futures --- ...aMappingModeFallbackRegressionAlgorithm.cs | 5 ++-- ...aMappingModeFallbackRegressionAlgorithm.py | 5 ++-- Algorithm/QCAlgorithm.History.cs | 17 ++++++++------ Algorithm/QCAlgorithm.cs | 12 +++++----- Tests/Algorithm/AlgorithmAddSecurityTests.cs | 23 ++++++++----------- Tests/Algorithm/AlgorithmHistoryTests.cs | 14 +++++++++++ 6 files changed, 44 insertions(+), 32 deletions(-) diff --git a/Algorithm.CSharp/FutureDataMappingModeFallbackRegressionAlgorithm.cs b/Algorithm.CSharp/FutureDataMappingModeFallbackRegressionAlgorithm.cs index 4f86cbab6c19..24131890def7 100644 --- a/Algorithm.CSharp/FutureDataMappingModeFallbackRegressionAlgorithm.cs +++ b/Algorithm.CSharp/FutureDataMappingModeFallbackRegressionAlgorithm.cs @@ -43,7 +43,7 @@ public override void Initialize() // Open interest resolves to the mode already in use, so this is not a conflicting re-add AddFuture(Futures.Indices.EuroStoxx50, Resolution.Minute, dataMappingMode: DataMappingMode.LastTradingDay); - if (DebugMessages.Any(message => message.Contains("was already added"))) + if (DebugMessages.Any(message => message.Contains("already added"))) { throw new RegressionTestException("Unexpected re-add warning for a future added again with the same data mapping mode"); } @@ -76,8 +76,7 @@ public override void OnData(Slice slice) AssertWarning("Warning: OpenInterest data mapping mode is not available for EUREX futures, no contract will be mapped. Use LastTradingDay instead."); AddFuture(Futures.Indices.EuroStoxx50, Resolution.Minute, dataMappingMode: DataMappingMode.FirstDayMonth); - AssertWarning("Warning: /FESX was already added, ignoring the requested data mapping mode FirstDayMonth (keeping LastTradingDay). " + - "To change these settings, remove it with RemoveSecurity() and add it again."); + AssertWarning("Warning: /FESX already added, ignoring data mapping mode FirstDayMonth. Remove it first to change its settings."); } public override void OnEndOfAlgorithm() diff --git a/Algorithm.Python/FutureDataMappingModeFallbackRegressionAlgorithm.py b/Algorithm.Python/FutureDataMappingModeFallbackRegressionAlgorithm.py index 0ce351849341..1c5ee2977898 100644 --- a/Algorithm.Python/FutureDataMappingModeFallbackRegressionAlgorithm.py +++ b/Algorithm.Python/FutureDataMappingModeFallbackRegressionAlgorithm.py @@ -28,7 +28,7 @@ def initialize(self): # Open interest resolves to the mode already in use, so this is not a conflicting re-add self.add_future(Futures.Indices.EURO_STOXX_50, Resolution.MINUTE, data_mapping_mode=DataMappingMode.LAST_TRADING_DAY) - if any("was already added" in message for message in self.debug_messages): + if any("already added" in message for message in self.debug_messages): raise AssertionError("Unexpected re-add warning for a future added again with the same data mapping mode") self._checked_after_initialize = False @@ -51,8 +51,7 @@ def on_data(self, slice): self._assert_warning("Warning: OpenInterest data mapping mode is not available for EUREX futures, no contract will be mapped. Use LastTradingDay instead.") self.add_future(Futures.Indices.EURO_STOXX_50, Resolution.MINUTE, data_mapping_mode=DataMappingMode.FIRST_DAY_MONTH) - self._assert_warning("Warning: /FESX was already added, ignoring the requested data mapping mode FirstDayMonth (keeping LastTradingDay). " - "To change these settings, remove it with RemoveSecurity() and add it again.") + self._assert_warning("Warning: /FESX already added, ignoring data mapping mode FirstDayMonth. Remove it first to change its settings.") def on_end_of_algorithm(self): if not self._checked_after_initialize: diff --git a/Algorithm/QCAlgorithm.History.cs b/Algorithm/QCAlgorithm.History.cs index a987d754a8bd..018970a2bbe0 100644 --- a/Algorithm/QCAlgorithm.History.cs +++ b/Algorithm/QCAlgorithm.History.cs @@ -1105,11 +1105,11 @@ protected IEnumerable CreateDateRangeHistoryRequests(IEnumerable { return symbols.Where(HistoryRequestValid).SelectMany(x => { - WarnIfDataMappingModeUnavailable(x, dataMappingMode); var requests = new List(); foreach (var config in GetMatchingSubscriptions(x, requestedType, resolution)) { + WarnIfDataMappingModeUnavailable(config, dataMappingMode); var request = _historyRequestFactory.CreateHistoryRequest(config, startAlgoTz, endAlgoTz, GetExchangeHours(x, requestedType), resolution, fillForward, extendedMarketHours, dataMappingMode, dataNormalizationMode, contractDepthOffset); requests.Add(request); @@ -1150,8 +1150,6 @@ private IEnumerable CreateBarCountHistoryRequests(IEnumerable { - WarnIfDataMappingModeUnavailable(symbol, dataMappingMode); - // Match or create configs for the symbol var configs = GetMatchingSubscriptions(symbol, requestedType, resolution, useAllSubscriptions).ToList(); if (configs.Count == 0) @@ -1161,6 +1159,8 @@ private IEnumerable CreateBarCountHistoryRequests(IEnumerable { + WarnIfDataMappingModeUnavailable(config, dataMappingMode); + // If no requested type was passed, use the config type to get the resolution (if not provided) and the exchange hours var type = requestedType ?? config.Type; var res = resolution ?? config.Resolution; @@ -1176,11 +1176,14 @@ private IEnumerable CreateBarCountHistoryRequests(IEnumerable - /// Warns once if an explicitly requested data mapping mode has no mapping data for the future's market + /// Warns once if an explicitly requested data mapping mode has no mapping data for a continuous future history request /// - private void WarnIfDataMappingModeUnavailable(Symbol symbol, DataMappingMode? dataMappingMode) + private void WarnIfDataMappingModeUnavailable(SubscriptionDataConfig config, DataMappingMode? dataMappingMode) { - if (dataMappingMode.HasValue && symbol.SecurityType == SecurityType.Future && !_unavailableDataMappingModeWarningSent + var symbol = config.Symbol; + // only continuous futures data is mapped, not contracts nor chain universe data + if (dataMappingMode.HasValue && symbol.SecurityType == SecurityType.Future && symbol.IsCanonical() + && LeanData.IsCommonLeanDataType(config.Type) && !_unavailableDataMappingModeWarningSent && !dataMappingMode.Value.IsAvailableForFutureMarket(symbol.ID.Market)) { _unavailableDataMappingModeWarningSent = true; @@ -1310,7 +1313,7 @@ private IEnumerable GetMatchingSubscriptions(Symbol symb // Inherit values from existing subscriptions or use defaults var extendedMarketHours = userConfigIfAny?.ExtendedMarketHours ?? UniverseSettings.ExtendedMarketHours; var dataNormalizationMode = userConfigIfAny?.DataNormalizationMode ?? UniverseSettings.GetUniverseNormalizationModeOrDefault(symbol.SecurityType); - var dataMappingMode = userConfigIfAny?.DataMappingMode ?? GetDataMappingModeOrDefault(symbol); + var dataMappingMode = GetDataMappingModeOrDefault(symbol, userConfigIfAny?.DataMappingMode); var contractDepthOffset = userConfigIfAny?.ContractDepthOffset ?? (uint)Math.Abs(UniverseSettings.ContractDepthOffset); // If type was specified and not a lean data type and also not abstract, we create a new subscription diff --git a/Algorithm/QCAlgorithm.cs b/Algorithm/QCAlgorithm.cs index a16d4c0eb191..31acf435aa64 100644 --- a/Algorithm/QCAlgorithm.cs +++ b/Algorithm/QCAlgorithm.cs @@ -2175,24 +2175,24 @@ private void WarnIfContinuousFutureSettingsIgnored(Symbol symbol, UniverseSettin var ignoredSettings = new List(); if (dataMappingMode.HasValue && dataMappingMode != existingSettings.DataMappingMode) { - ignoredSettings.Add($"data mapping mode {dataMappingMode} (keeping {existingSettings.DataMappingMode})"); + ignoredSettings.Add($"data mapping mode {dataMappingMode}"); } if (dataNormalizationMode.HasValue && dataNormalizationMode != existingSettings.DataNormalizationMode) { - ignoredSettings.Add($"data normalization mode {dataNormalizationMode} (keeping {existingSettings.DataNormalizationMode})"); + ignoredSettings.Add($"normalization mode {dataNormalizationMode}"); } if (contractDepthOffset != existingSettings.ContractDepthOffset) { - ignoredSettings.Add($"contract depth offset {contractDepthOffset} (keeping {existingSettings.ContractDepthOffset})"); + ignoredSettings.Add($"contract depth offset {contractDepthOffset}"); } if (ignoredSettings.Count > 0) { _ignoredContinuousFutureSettingsWarningSent = true; var instructions = _locked - ? "To change these settings, remove it with RemoveSecurity() and add it again." - : "Add it only once in Initialize, or, to change these settings after Initialize, remove it with RemoveSecurity() and add it again."; - Debug($"Warning: {symbol} was already added, ignoring the requested {string.Join(", ", ignoredSettings)}. {instructions}"); + ? "Remove it first to change its settings." + : "Add it once, or remove and re-add it after Initialize."; + Debug($"Warning: {symbol} already added, ignoring {string.Join(", ", ignoredSettings)}. {instructions}"); } } diff --git a/Tests/Algorithm/AlgorithmAddSecurityTests.cs b/Tests/Algorithm/AlgorithmAddSecurityTests.cs index c91f42e186b6..e885e55d304b 100644 --- a/Tests/Algorithm/AlgorithmAddSecurityTests.cs +++ b/Tests/Algorithm/AlgorithmAddSecurityTests.cs @@ -219,7 +219,7 @@ public void AddFutureAfterRemovingItAppliesNewSettings() var continuousUniverse = _algo.UniverseManager[ContinuousContractUniverse.CreateSymbol(future.Symbol)]; Assert.AreEqual(DataMappingMode.LastTradingDay, continuousUniverse.UniverseSettings.DataMappingMode); - Assert.IsFalse(_algo.DebugMessages.Any(x => x.Contains("was already added"))); + Assert.IsFalse(_algo.DebugMessages.Any(x => x.Contains("already added"))); } [Test] @@ -235,10 +235,9 @@ public void AddFutureAgainWithDifferentSettingsWarnsOnce() .ToList(); Assert.That(continuousConfigs.Select(x => x.DataMappingMode), Has.All.EqualTo(DataMappingMode.OpenInterest)); - var warnings = _algo.DebugMessages.Where(x => x.Contains("was already added")).ToList(); + var warnings = _algo.DebugMessages.Where(x => x.Contains("already added")).ToList(); Assert.AreEqual(1, warnings.Count); - Assert.That(warnings[0], Does.EndWith("Warning: /ES was already added, ignoring the requested data mapping mode LastTradingDay (keeping OpenInterest). " + - "Add it only once in Initialize, or, to change these settings after Initialize, remove it with RemoveSecurity() and add it again.")); + Assert.That(warnings[0], Does.EndWith("Warning: /ES already added, ignoring data mapping mode LastTradingDay. Add it once, or remove and re-add it after Initialize.")); } [Test] @@ -248,11 +247,10 @@ public void AddFutureAgainWithAllSettingsDifferentListsThemInOneWarning() _algo.AddFuture(Futures.Indices.SP500EMini, Resolution.Daily, dataMappingMode: DataMappingMode.LastTradingDay, dataNormalizationMode: DataNormalizationMode.Raw, contractDepthOffset: 1); - var warnings = _algo.DebugMessages.Where(x => x.Contains("was already added")).ToList(); + var warnings = _algo.DebugMessages.Where(x => x.Contains("already added")).ToList(); Assert.AreEqual(1, warnings.Count); - Assert.That(warnings[0], Does.EndWith("Warning: /ES was already added, ignoring the requested data mapping mode LastTradingDay (keeping OpenInterest), " + - "data normalization mode Raw (keeping BackwardsRatio), contract depth offset 1 (keeping 0). " + - "Add it only once in Initialize, or, to change these settings after Initialize, remove it with RemoveSecurity() and add it again.")); + Assert.That(warnings[0], Does.EndWith("Warning: /ES already added, ignoring data mapping mode LastTradingDay, normalization mode Raw, contract depth offset 1. " + + "Add it once, or remove and re-add it after Initialize.")); } [Test] @@ -262,9 +260,8 @@ public void AddFutureAgainAfterInitializeWarnsToRemoveItFirst() _algo.SetLocked(); _algo.AddFuture(Futures.Indices.SP500EMini, Resolution.Daily, dataMappingMode: DataMappingMode.LastTradingDay); - Assert.That(_algo.DebugMessages.Single(x => x.Contains("was already added")), - Does.EndWith("Warning: /ES was already added, ignoring the requested data mapping mode LastTradingDay (keeping OpenInterest). " + - "To change these settings, remove it with RemoveSecurity() and add it again.")); + Assert.That(_algo.DebugMessages.Single(x => x.Contains("already added")), + Does.EndWith("Warning: /ES already added, ignoring data mapping mode LastTradingDay. Remove it first to change its settings.")); } [TestCase(null)] @@ -274,7 +271,7 @@ public void AddFutureAgainWithSameSettingsDoesNotWarn(DataMappingMode? dataMappi _algo.AddFuture(Futures.Indices.SP500EMini, Resolution.Daily, dataMappingMode: DataMappingMode.OpenInterest); _algo.AddFuture(Futures.Indices.SP500EMini, Resolution.Daily, dataMappingMode: dataMappingMode); - Assert.IsFalse(_algo.DebugMessages.Any(x => x.Contains("was already added"))); + Assert.IsFalse(_algo.DebugMessages.Any(x => x.Contains("already added"))); } [Test] @@ -283,7 +280,7 @@ public void AddFutureAgainWithUnavailableDataMappingModeResolvingToSameModeDoesN _algo.AddFuture("FESX", Resolution.Daily, Market.EUREX); _algo.AddFuture("FESX", Resolution.Daily, Market.EUREX, dataMappingMode: DataMappingMode.OpenInterest); - Assert.IsFalse(_algo.DebugMessages.Any(x => x.Contains("was already added"))); + Assert.IsFalse(_algo.DebugMessages.Any(x => x.Contains("already added"))); Assert.AreEqual(1, _algo.DebugMessages.Count(x => x.Contains("data mapping mode is not available"))); } diff --git a/Tests/Algorithm/AlgorithmHistoryTests.cs b/Tests/Algorithm/AlgorithmHistoryTests.cs index 34561297506c..9470ea12e162 100644 --- a/Tests/Algorithm/AlgorithmHistoryTests.cs +++ b/Tests/Algorithm/AlgorithmHistoryTests.cs @@ -86,6 +86,20 @@ public void HistoryWithExplicitUnavailableDataMappingModeKeepsItAndWarnsOnce() "no contract will be mapped. Use LastTradingDay instead.")); } + [Test] + public void HistoryWithExplicitUnavailableDataMappingModeDoesNotWarnForContractsOrChains() + { + var future = _algorithm.AddFuture("FESX", Resolution.Daily, Market.EUREX); + var contract = Symbol.CreateFuture("FESX", Market.EUREX, new DateTime(2024, 6, 21)); + + _algorithm.History(contract, 5, Resolution.Daily, dataMappingMode: DataMappingMode.OpenInterest).ToList(); + _algorithm.History(future.Symbol, 5, Resolution.Daily, dataMappingMode: DataMappingMode.OpenInterest).ToList(); + + Assert.IsTrue(_testHistoryProvider.HistryRequests.Any(x => x.Symbol == contract)); + Assert.IsTrue(_testHistoryProvider.HistryRequests.Any(x => x.DataType == typeof(FutureUniverse))); + Assert.IsFalse(_algorithm.DebugMessages.Any(x => x.Contains("no contract will be mapped"))); + } + [TestCase(Language.Python)] [TestCase(Language.CSharp)] public void FundamentalHistory(Language language) From 011566613736efae810867d829033a1412e91630 Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Fri, 25 Sep 2026 18:01:49 -0400 Subject: [PATCH 7/7] Fall back from unavailable mapping modes in continuous future history requests --- ...aMappingModeFallbackRegressionAlgorithm.cs | 14 ++-- ...aMappingModeFallbackRegressionAlgorithm.py | 10 +-- Algorithm/QCAlgorithm.History.cs | 39 ++++++----- Algorithm/QCAlgorithm.cs | 2 +- Tests/Algorithm/AlgorithmHistoryTests.cs | 69 ++++++++++++++----- 5 files changed, 87 insertions(+), 47 deletions(-) diff --git a/Algorithm.CSharp/FutureDataMappingModeFallbackRegressionAlgorithm.cs b/Algorithm.CSharp/FutureDataMappingModeFallbackRegressionAlgorithm.cs index 24131890def7..cf64f7f6def8 100644 --- a/Algorithm.CSharp/FutureDataMappingModeFallbackRegressionAlgorithm.cs +++ b/Algorithm.CSharp/FutureDataMappingModeFallbackRegressionAlgorithm.cs @@ -63,17 +63,19 @@ public override void OnData(Slice slice) throw new RegressionTestException($"Unexpected mapped contract {_future.Mapped}, expected the June contract"); } - if (History(_future.Symbol, 10, Resolution.Minute).Count() == 0) + var history = History(_future.Symbol, 10, Resolution.Minute).Select(x => x.Close).ToList(); + if (history.Count == 0) { throw new RegressionTestException("Expected history for the continuous future using the fallback data mapping mode"); } - var openInterestHistory = History(_future.Symbol, 10, Resolution.Minute, dataMappingMode: DataMappingMode.OpenInterest); - if (openInterestHistory.Any()) + // explicit open interest falls back to the same mode, the market warning was already sent by AddFuture + var openInterestHistory = History(_future.Symbol, 10, Resolution.Minute, dataMappingMode: DataMappingMode.OpenInterest).Select(x => x.Close).ToList(); + if (!openInterestHistory.SequenceEqual(history)) { - throw new RegressionTestException("Expected no history for the explicitly requested open interest data mapping mode"); + throw new RegressionTestException("Expected the explicit open interest history to match the fallback data mapping mode history"); } - AssertWarning("Warning: OpenInterest data mapping mode is not available for EUREX futures, no contract will be mapped. Use LastTradingDay instead."); + AssertWarning("Warning: OpenInterest data mapping mode is not available for EUREX futures, using LastTradingDay instead."); AddFuture(Futures.Indices.EuroStoxx50, Resolution.Minute, dataMappingMode: DataMappingMode.FirstDayMonth); AssertWarning("Warning: /FESX already added, ignoring data mapping mode FirstDayMonth. Remove it first to change its settings."); @@ -117,7 +119,7 @@ private void AssertWarning(string warning) /// /// Data Points count of the algorithm history /// - public int AlgorithmHistoryDataPoints => 10; + public int AlgorithmHistoryDataPoints => 20; /// /// Final status of the algorithm diff --git a/Algorithm.Python/FutureDataMappingModeFallbackRegressionAlgorithm.py b/Algorithm.Python/FutureDataMappingModeFallbackRegressionAlgorithm.py index 1c5ee2977898..b6f140e22741 100644 --- a/Algorithm.Python/FutureDataMappingModeFallbackRegressionAlgorithm.py +++ b/Algorithm.Python/FutureDataMappingModeFallbackRegressionAlgorithm.py @@ -42,13 +42,15 @@ def on_data(self, slice): if self._future.mapped.id.date.month != 6: raise AssertionError(f"Unexpected mapped contract {self._future.mapped}, expected the June contract") - if self.history(self._future.symbol, 10, Resolution.MINUTE).empty: + history = self.history(self._future.symbol, 10, Resolution.MINUTE) + if history.empty: raise AssertionError("Expected history for the continuous future using the fallback data mapping mode") + # explicit open interest falls back to the same mode, the market warning was already sent by add_future open_interest_history = self.history(self._future.symbol, 10, Resolution.MINUTE, data_mapping_mode=DataMappingMode.OPEN_INTEREST) - if not open_interest_history.empty: - raise AssertionError("Expected no history for the explicitly requested open interest data mapping mode") - self._assert_warning("Warning: OpenInterest data mapping mode is not available for EUREX futures, no contract will be mapped. Use LastTradingDay instead.") + if list(open_interest_history["close"]) != list(history["close"]): + raise AssertionError("Expected the explicit open interest history to match the fallback data mapping mode history") + self._assert_warning("Warning: OpenInterest data mapping mode is not available for EUREX futures, using LastTradingDay instead.") self.add_future(Futures.Indices.EURO_STOXX_50, Resolution.MINUTE, data_mapping_mode=DataMappingMode.FIRST_DAY_MONTH) self._assert_warning("Warning: /FESX already added, ignoring data mapping mode FirstDayMonth. Remove it first to change its settings.") diff --git a/Algorithm/QCAlgorithm.History.cs b/Algorithm/QCAlgorithm.History.cs index 018970a2bbe0..69d8043ddbae 100644 --- a/Algorithm/QCAlgorithm.History.cs +++ b/Algorithm/QCAlgorithm.History.cs @@ -37,7 +37,6 @@ public partial class QCAlgorithm private static readonly int SeedRetryDailyLookbackPeriod = Config.GetInt("seed-retry-daily-lookback-period", 10); private bool _dataDictionaryTickWarningSent; - private bool _unavailableDataMappingModeWarningSent; private readonly LargeHistoryRequestDiagnostics _largeHistoryRequestDiagnostics = new(); @@ -1107,11 +1106,10 @@ protected IEnumerable CreateDateRangeHistoryRequests(IEnumerable { var requests = new List(); - foreach (var config in GetMatchingSubscriptions(x, requestedType, resolution)) + foreach (var config in GetMatchingSubscriptions(x, requestedType, resolution, dataMappingMode: dataMappingMode)) { - WarnIfDataMappingModeUnavailable(config, dataMappingMode); var request = _historyRequestFactory.CreateHistoryRequest(config, startAlgoTz, endAlgoTz, GetExchangeHours(x, requestedType), resolution, - fillForward, extendedMarketHours, dataMappingMode, dataNormalizationMode, contractDepthOffset); + fillForward, extendedMarketHours, GetHistoryDataMappingMode(config, dataMappingMode), dataNormalizationMode, contractDepthOffset); requests.Add(request); } @@ -1151,7 +1149,7 @@ private IEnumerable CreateBarCountHistoryRequests(IEnumerable { // Match or create configs for the symbol - var configs = GetMatchingSubscriptions(symbol, requestedType, resolution, useAllSubscriptions).ToList(); + var configs = GetMatchingSubscriptions(symbol, requestedType, resolution, useAllSubscriptions, dataMappingMode).ToList(); if (configs.Count == 0) { return Enumerable.Empty(); @@ -1159,8 +1157,6 @@ private IEnumerable CreateBarCountHistoryRequests(IEnumerable { - WarnIfDataMappingModeUnavailable(config, dataMappingMode); - // If no requested type was passed, use the config type to get the resolution (if not provided) and the exchange hours var type = requestedType ?? config.Type; var res = resolution ?? config.Resolution; @@ -1170,25 +1166,24 @@ private IEnumerable CreateBarCountHistoryRequests(IEnumerable - /// Warns once if an explicitly requested data mapping mode has no mapping data for a continuous future history request + /// Gets the data mapping mode for a history request, falling back to the market default if the requested one is not available /// - private void WarnIfDataMappingModeUnavailable(SubscriptionDataConfig config, DataMappingMode? dataMappingMode) + private DataMappingMode? GetHistoryDataMappingMode(SubscriptionDataConfig config, DataMappingMode? dataMappingMode) { var symbol = config.Symbol; // only continuous futures data is mapped, not contracts nor chain universe data - if (dataMappingMode.HasValue && symbol.SecurityType == SecurityType.Future && symbol.IsCanonical() - && LeanData.IsCommonLeanDataType(config.Type) && !_unavailableDataMappingModeWarningSent - && !dataMappingMode.Value.IsAvailableForFutureMarket(symbol.ID.Market)) + if (!dataMappingMode.HasValue || symbol.SecurityType != SecurityType.Future || !symbol.IsCanonical() + || !LeanData.IsCommonLeanDataType(config.Type)) { - _unavailableDataMappingModeWarningSent = true; - Debug($"Warning: {dataMappingMode} data mapping mode is not available for {symbol.ID.Market.ToUpperInvariant()} futures, no contract will be mapped. Use {DataMappingMode.LastTradingDay} instead."); + return dataMappingMode; } + return GetDataMappingModeOrDefault(symbol, dataMappingMode); } private int GetTickTypeOrder(SecurityType securityType, TickType tickType) @@ -1196,7 +1191,8 @@ private int GetTickTypeOrder(SecurityType securityType, TickType tickType) return SubscriptionManager.AvailableDataTypes[securityType].IndexOf(tickType); } - private IEnumerable GetMatchingSubscriptions(Symbol symbol, Type type, Resolution? resolution = null, bool useAllSubscriptions = false) + private IEnumerable GetMatchingSubscriptions(Symbol symbol, Type type, Resolution? resolution = null, bool useAllSubscriptions = false, + DataMappingMode? dataMappingMode = null) { var subscriptions = SubscriptionManager.SubscriptionDataConfigService // we add internal subscription so that history requests are covered, this allows us to warm them up too @@ -1313,7 +1309,9 @@ private IEnumerable GetMatchingSubscriptions(Symbol symb // Inherit values from existing subscriptions or use defaults var extendedMarketHours = userConfigIfAny?.ExtendedMarketHours ?? UniverseSettings.ExtendedMarketHours; var dataNormalizationMode = userConfigIfAny?.DataNormalizationMode ?? UniverseSettings.GetUniverseNormalizationModeOrDefault(symbol.SecurityType); - var dataMappingMode = GetDataMappingModeOrDefault(symbol, userConfigIfAny?.DataMappingMode); + var requestedDataMappingMode = dataMappingMode ?? userConfigIfAny?.DataMappingMode; + // only continuous futures data falls back from an unavailable mapping mode, other data types are not mapped + var unmappedDataMappingMode = requestedDataMappingMode ?? UniverseSettings.GetUniverseMappingModeOrDefault(symbol.SecurityType, symbol.ID.Market); var contractDepthOffset = userConfigIfAny?.ContractDepthOffset ?? (uint)Math.Abs(UniverseSettings.ContractDepthOffset); // If type was specified and not a lean data type and also not abstract, we create a new subscription @@ -1345,7 +1343,7 @@ private IEnumerable GetMatchingSubscriptions(Symbol symb LeanData.GetCommonTickTypeForCommonDataTypes(dataType, symbol.SecurityType), true, dataNormalizationMode, - dataMappingMode, + unmappedDataMappingMode, contractDepthOffset)}; } @@ -1361,6 +1359,9 @@ private IEnumerable GetMatchingSubscriptions(Symbol symb // Use the config type to get an accurate mhdb entry var entry = MarketHoursDatabase.GetEntry(symbol, new[] { configType }); var res = GetResolution(symbol, resolution, configType); + var configDataMappingMode = LeanData.IsCommonLeanDataType(configType) + ? GetDataMappingModeOrDefault(symbol, requestedDataMappingMode) + : unmappedDataMappingMode; return new SubscriptionDataConfig( configType, @@ -1375,7 +1376,7 @@ private IEnumerable GetMatchingSubscriptions(Symbol symb x.Item2, true, dataNormalizationMode, - dataMappingMode, + configDataMappingMode, contractDepthOffset); }) // lets make sure to respect the order of the data types, if used on a history request will affect outcome when using pushthrough for example diff --git a/Algorithm/QCAlgorithm.cs b/Algorithm/QCAlgorithm.cs index 31acf435aa64..ed31d96b7be5 100644 --- a/Algorithm/QCAlgorithm.cs +++ b/Algorithm/QCAlgorithm.cs @@ -2148,7 +2148,7 @@ public Security AddSecurity(Symbol symbol, Resolution? resolution = null, bool? private DataMappingMode GetDataMappingModeOrDefault(Symbol symbol, DataMappingMode? dataMappingMode = null) { var requestedDataMappingMode = dataMappingMode ?? UniverseSettings.DataMappingMode; - if (symbol.SecurityType != SecurityType.Future || requestedDataMappingMode.IsAvailableForFutureMarket(symbol.ID.Market)) + if (symbol.SecurityType != SecurityType.Future || !symbol.IsCanonical() || requestedDataMappingMode.IsAvailableForFutureMarket(symbol.ID.Market)) { return requestedDataMappingMode; } diff --git a/Tests/Algorithm/AlgorithmHistoryTests.cs b/Tests/Algorithm/AlgorithmHistoryTests.cs index 9470ea12e162..270905a95a00 100644 --- a/Tests/Algorithm/AlgorithmHistoryTests.cs +++ b/Tests/Algorithm/AlgorithmHistoryTests.cs @@ -70,34 +70,69 @@ public void OneTimeSetUp() FundamentalService.Initialize(_dataProvider, new NullFundamentalDataProvider(), false); } + [TestCase(null)] + [TestCase(DataMappingMode.FirstDayMonth)] + public void HistoryWithExplicitUnavailableDataMappingModeFallsBackToMarketDefault(DataMappingMode? addedFutureDataMappingMode) + { + var symbol = Symbol.Create("FESX", SecurityType.Future, Market.EUREX); + if (addedFutureDataMappingMode.HasValue) + { + _algorithm.AddFuture("FESX", Resolution.Daily, Market.EUREX, dataMappingMode: addedFutureDataMappingMode); + } + + _algorithm.History(symbol, 5, Resolution.Daily, dataMappingMode: DataMappingMode.OpenInterest).ToList(); + _algorithm.History(symbol, _algorithm.Time.AddDays(-5), _algorithm.Time, Resolution.Daily, dataMappingMode: DataMappingMode.OpenInterest).ToList(); + + var requests = _testHistoryProvider.HistryRequests.Where(x => x.Symbol == symbol).ToList(); + Assert.Greater(requests.Count, 0); + // the market default is used, not the added future's mode + Assert.That(requests.Select(x => x.DataMappingMode), Has.All.EqualTo(DataMappingMode.LastTradingDay)); + Assert.That(_algorithm.DebugMessages.Single(x => x.Contains("data mapping mode is not available")), + Does.EndWith("Warning: OpenInterest data mapping mode is not available for EUREX futures, using LastTradingDay instead.")); + } + [Test] - public void HistoryWithExplicitUnavailableDataMappingModeKeepsItAndWarnsOnce() + public void HistoryWithoutDataMappingModeFallsBackForFutureNotAdded() { - var future = _algorithm.AddFuture("FESX", Resolution.Daily, Market.EUREX); + var symbol = Symbol.Create("FESX", SecurityType.Future, Market.EUREX); - _algorithm.History(future.Symbol, 5, Resolution.Daily, dataMappingMode: DataMappingMode.OpenInterest).ToList(); - _algorithm.History(future.Symbol, 5, Resolution.Daily, dataMappingMode: DataMappingMode.OpenInterest).ToList(); + _algorithm.History(symbol, 5, Resolution.Daily).ToList(); - Assert.Greater(_testHistoryProvider.HistryRequests.Count, 0); - Assert.That(_testHistoryProvider.HistryRequests.Select(x => x.DataMappingMode), Has.All.EqualTo(DataMappingMode.OpenInterest)); - var warnings = _algorithm.DebugMessages.Where(x => x.Contains("no contract will be mapped")).ToList(); - Assert.AreEqual(1, warnings.Count); - Assert.That(warnings[0], Does.EndWith("Warning: OpenInterest data mapping mode is not available for EUREX futures, " + - "no contract will be mapped. Use LastTradingDay instead.")); + var requests = _testHistoryProvider.HistryRequests.Where(x => x.Symbol == symbol).ToList(); + Assert.Greater(requests.Count, 0); + Assert.That(requests.Select(x => x.DataMappingMode), Has.All.EqualTo(DataMappingMode.LastTradingDay)); + Assert.That(_algorithm.DebugMessages.Single(x => x.Contains("data mapping mode is not available")), + Does.EndWith("Warning: OpenInterest data mapping mode is not available for EUREX futures, using LastTradingDay instead.")); } [Test] - public void HistoryWithExplicitUnavailableDataMappingModeDoesNotWarnForContractsOrChains() + public void HistoryWithExplicitAvailableDataMappingModeKeepsIt() { - var future = _algorithm.AddFuture("FESX", Resolution.Daily, Market.EUREX); + var symbol = Symbol.Create("FESX", SecurityType.Future, Market.EUREX); + + _algorithm.History(symbol, 5, Resolution.Daily, dataMappingMode: DataMappingMode.FirstDayMonth).ToList(); + + var requests = _testHistoryProvider.HistryRequests.Where(x => x.Symbol == symbol).ToList(); + Assert.Greater(requests.Count, 0); + Assert.That(requests.Select(x => x.DataMappingMode), Has.All.EqualTo(DataMappingMode.FirstDayMonth)); + Assert.IsFalse(_algorithm.DebugMessages.Any(x => x.Contains("data mapping mode is not available"))); + } + + [Test] + public void HistoryWithExplicitUnavailableDataMappingModeDoesNotFallBackForContractsOrChains() + { + var symbol = Symbol.Create("FESX", SecurityType.Future, Market.EUREX); var contract = Symbol.CreateFuture("FESX", Market.EUREX, new DateTime(2024, 6, 21)); _algorithm.History(contract, 5, Resolution.Daily, dataMappingMode: DataMappingMode.OpenInterest).ToList(); - _algorithm.History(future.Symbol, 5, Resolution.Daily, dataMappingMode: DataMappingMode.OpenInterest).ToList(); - - Assert.IsTrue(_testHistoryProvider.HistryRequests.Any(x => x.Symbol == contract)); - Assert.IsTrue(_testHistoryProvider.HistryRequests.Any(x => x.DataType == typeof(FutureUniverse))); - Assert.IsFalse(_algorithm.DebugMessages.Any(x => x.Contains("no contract will be mapped"))); + _algorithm.History(symbol, 5, Resolution.Daily, dataMappingMode: DataMappingMode.OpenInterest).ToList(); + + var contractRequests = _testHistoryProvider.HistryRequests.Where(x => x.Symbol == contract).ToList(); + var chainRequests = _testHistoryProvider.HistryRequests.Where(x => x.DataType == typeof(FutureUniverse)).ToList(); + Assert.Greater(contractRequests.Count, 0); + Assert.Greater(chainRequests.Count, 0); + Assert.That(contractRequests.Concat(chainRequests).Select(x => x.DataMappingMode), Has.All.EqualTo(DataMappingMode.OpenInterest)); + Assert.IsFalse(_algorithm.DebugMessages.Any(x => x.Contains("data mapping mode is not available"))); } [TestCase(Language.Python)]