diff --git a/Algorithm.CSharp/FutureDataMappingModeFallbackRegressionAlgorithm.cs b/Algorithm.CSharp/FutureDataMappingModeFallbackRegressionAlgorithm.cs new file mode 100644 index 000000000000..cf64f7f6def8 --- /dev/null +++ b/Algorithm.CSharp/FutureDataMappingModeFallbackRegressionAlgorithm.cs @@ -0,0 +1,164 @@ +/* + * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. + * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * +*/ + +using System.Collections.Generic; +using System.Linq; +using QuantConnect.Data; +using QuantConnect.Interfaces; +using QuantConnect.Securities; +using QuantConnect.Securities.Future; + +namespace QuantConnect.Algorithm.CSharp +{ + /// + /// Regression algorithm asserting that a future added with a data mapping mode its market has no data for (open interest on EUREX) + /// falls back to the market default, and that the related warnings are sent. + /// + public class FutureDataMappingModeFallbackRegressionAlgorithm : QCAlgorithm, IRegressionAlgorithmDefinition + { + private Future _future; + private bool _checkedAfterInitialize; + + public override void Initialize() + { + SetStartDate(2024, 6, 3); + SetEndDate(2024, 6, 4); + SetAccountCurrency(Currencies.EUR); + + _future = AddFuture(Futures.Indices.EuroStoxx50, Resolution.Minute, dataMappingMode: DataMappingMode.OpenInterest); + AssertWarning("Warning: OpenInterest data mapping mode is not available for EUREX futures, using LastTradingDay instead."); + + // Open interest resolves to the mode already in use, so this is not a conflicting re-add + AddFuture(Futures.Indices.EuroStoxx50, Resolution.Minute, dataMappingMode: DataMappingMode.LastTradingDay); + if (DebugMessages.Any(message => message.Contains("already added"))) + { + throw new RegressionTestException("Unexpected re-add warning for a future added again with the same data mapping mode"); + } + } + + public override void OnData(Slice slice) + { + if (_checkedAfterInitialize || _future.Mapped == null) + { + return; + } + _checkedAfterInitialize = true; + + // Last trading day maps to the June contract, first day of the month would map to September + if (_future.Mapped.ID.Date.Month != 6) + { + throw new RegressionTestException($"Unexpected mapped contract {_future.Mapped}, expected the June contract"); + } + + var history = History(_future.Symbol, 10, Resolution.Minute).Select(x => x.Close).ToList(); + if (history.Count == 0) + { + throw new RegressionTestException("Expected history for the continuous future using the fallback data mapping mode"); + } + + // explicit open interest falls back to the same mode, the market warning was already sent by AddFuture + var openInterestHistory = History(_future.Symbol, 10, Resolution.Minute, dataMappingMode: DataMappingMode.OpenInterest).Select(x => x.Close).ToList(); + if (!openInterestHistory.SequenceEqual(history)) + { + throw new RegressionTestException("Expected the explicit open interest history to match the fallback data mapping mode history"); + } + AssertWarning("Warning: OpenInterest data mapping mode is not available for EUREX futures, using LastTradingDay instead."); + + AddFuture(Futures.Indices.EuroStoxx50, Resolution.Minute, dataMappingMode: DataMappingMode.FirstDayMonth); + AssertWarning("Warning: /FESX already added, ignoring data mapping mode FirstDayMonth. Remove it first to change its settings."); + } + + public override void OnEndOfAlgorithm() + { + if (!_checkedAfterInitialize) + { + throw new RegressionTestException("The continuous future was never mapped"); + } + if (_future.Mapped.ID.Date.Month != 6) + { + throw new RegressionTestException($"Unexpected mapped contract {_future.Mapped} after the ignored re-add, expected the June contract"); + } + } + + private void AssertWarning(string warning) + { + if (DebugMessages.Count(message => message.EndsWith(warning)) != 1) + { + throw new RegressionTestException($"Expected the warning '{warning}' to be sent once"); + } + } + + /// + /// This is used by the regression test system to indicate if the open source Lean repository has the required data to run this algorithm. + /// + public bool CanRunLocally { get; } = true; + + /// + /// This is used by the regression test system to indicate which languages this algorithm is written in. + /// + public List Languages { get; } = new() { Language.CSharp, Language.Python }; + + /// + /// Data Points count of all timeslices of algorithm + /// + public long DataPoints => 5010; + + /// + /// Data Points count of the algorithm history + /// + public int AlgorithmHistoryDataPoints => 20; + + /// + /// Final status of the algorithm + /// + public AlgorithmStatus AlgorithmStatus => AlgorithmStatus.Completed; + + /// + /// This is used by the regression test system to indicate what the expected statistics are from running the algorithm + /// + public Dictionary ExpectedStatistics => new Dictionary + { + {"Total Orders", "0"}, + {"Average Win", "0%"}, + {"Average Loss", "0%"}, + {"Compounding Annual Return", "0%"}, + {"Drawdown", "0%"}, + {"Expectancy", "0"}, + {"Start Equity", "100000"}, + {"End Equity", "100000"}, + {"Net Profit", "0%"}, + {"Sharpe Ratio", "0"}, + {"Sortino Ratio", "0"}, + {"Probabilistic Sharpe Ratio", "0%"}, + {"Loss Rate", "0%"}, + {"Win Rate", "0%"}, + {"Profit-Loss Ratio", "0"}, + {"Alpha", "0"}, + {"Beta", "0"}, + {"Annual Standard Deviation", "0"}, + {"Annual Variance", "0"}, + {"Information Ratio", "0"}, + {"Tracking Error", "0"}, + {"Treynor Ratio", "0"}, + {"Total Fees", "€0.00"}, + {"Estimated Strategy Capacity", "€0"}, + {"Lowest Capacity Asset", ""}, + {"Portfolio Turnover", "0%"}, + {"Drawdown Recovery", "0"}, + {"OrderListHash", "d41d8cd98f00b204e9800998ecf8427e"}, + }; + } +} diff --git a/Algorithm.Python/FutureDataMappingModeFallbackRegressionAlgorithm.py b/Algorithm.Python/FutureDataMappingModeFallbackRegressionAlgorithm.py new file mode 100644 index 000000000000..b6f140e22741 --- /dev/null +++ b/Algorithm.Python/FutureDataMappingModeFallbackRegressionAlgorithm.py @@ -0,0 +1,66 @@ +# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. +# Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from AlgorithmImports import * + +### +### Regression algorithm asserting that a future added with a data mapping mode its market has no data for (open interest on EUREX) +### falls back to the market default, and that the related warnings are sent. +### +class FutureDataMappingModeFallbackRegressionAlgorithm(QCAlgorithm): + def initialize(self): + self.set_start_date(2024, 6, 3) + self.set_end_date(2024, 6, 4) + self.set_account_currency(Currencies.EUR) + + self._future = self.add_future(Futures.Indices.EURO_STOXX_50, Resolution.MINUTE, data_mapping_mode=DataMappingMode.OPEN_INTEREST) + self._assert_warning("Warning: OpenInterest data mapping mode is not available for EUREX futures, using LastTradingDay instead.") + + # Open interest resolves to the mode already in use, so this is not a conflicting re-add + self.add_future(Futures.Indices.EURO_STOXX_50, Resolution.MINUTE, data_mapping_mode=DataMappingMode.LAST_TRADING_DAY) + if any("already added" in message for message in self.debug_messages): + raise AssertionError("Unexpected re-add warning for a future added again with the same data mapping mode") + + self._checked_after_initialize = False + + def on_data(self, slice): + if self._checked_after_initialize or self._future.mapped is None: + return + self._checked_after_initialize = True + + # Last trading day maps to the June contract, first day of the month would map to September + if self._future.mapped.id.date.month != 6: + raise AssertionError(f"Unexpected mapped contract {self._future.mapped}, expected the June contract") + + history = self.history(self._future.symbol, 10, Resolution.MINUTE) + if history.empty: + raise AssertionError("Expected history for the continuous future using the fallback data mapping mode") + + # explicit open interest falls back to the same mode, the market warning was already sent by add_future + open_interest_history = self.history(self._future.symbol, 10, Resolution.MINUTE, data_mapping_mode=DataMappingMode.OPEN_INTEREST) + if list(open_interest_history["close"]) != list(history["close"]): + raise AssertionError("Expected the explicit open interest history to match the fallback data mapping mode history") + self._assert_warning("Warning: OpenInterest data mapping mode is not available for EUREX futures, using LastTradingDay instead.") + + self.add_future(Futures.Indices.EURO_STOXX_50, Resolution.MINUTE, data_mapping_mode=DataMappingMode.FIRST_DAY_MONTH) + self._assert_warning("Warning: /FESX already added, ignoring data mapping mode FirstDayMonth. Remove it first to change its settings.") + + def on_end_of_algorithm(self): + if not self._checked_after_initialize: + raise AssertionError("The continuous future was never mapped") + if self._future.mapped.id.date.month != 6: + raise AssertionError(f"Unexpected mapped contract {self._future.mapped} after the ignored re-add, expected the June contract") + + def _assert_warning(self, warning): + if sum(1 for message in self.debug_messages if message.endswith(warning)) != 1: + raise AssertionError(f"Expected the warning '{warning}' to be sent once") diff --git a/Algorithm/QCAlgorithm.History.cs b/Algorithm/QCAlgorithm.History.cs index afdd00def57a..69d8043ddbae 100644 --- a/Algorithm/QCAlgorithm.History.cs +++ b/Algorithm/QCAlgorithm.History.cs @@ -1106,10 +1106,10 @@ protected IEnumerable CreateDateRangeHistoryRequests(IEnumerable { var requests = new List(); - foreach (var config in GetMatchingSubscriptions(x, requestedType, resolution)) + foreach (var config in GetMatchingSubscriptions(x, requestedType, resolution, dataMappingMode: dataMappingMode)) { var request = _historyRequestFactory.CreateHistoryRequest(config, startAlgoTz, endAlgoTz, GetExchangeHours(x, requestedType), resolution, - fillForward, extendedMarketHours, dataMappingMode, dataNormalizationMode, contractDepthOffset); + fillForward, extendedMarketHours, GetHistoryDataMappingMode(config, dataMappingMode), dataNormalizationMode, contractDepthOffset); requests.Add(request); } @@ -1149,7 +1149,7 @@ private IEnumerable CreateBarCountHistoryRequests(IEnumerable { // Match or create configs for the symbol - var configs = GetMatchingSubscriptions(symbol, requestedType, resolution, useAllSubscriptions).ToList(); + var configs = GetMatchingSubscriptions(symbol, requestedType, resolution, useAllSubscriptions, dataMappingMode).ToList(); if (configs.Count == 0) { return Enumerable.Empty(); @@ -1166,17 +1166,33 @@ private IEnumerable CreateBarCountHistoryRequests(IEnumerable + /// Gets the data mapping mode for a history request, falling back to the market default if the requested one is not available + /// + private DataMappingMode? GetHistoryDataMappingMode(SubscriptionDataConfig config, DataMappingMode? dataMappingMode) + { + var symbol = config.Symbol; + // only continuous futures data is mapped, not contracts nor chain universe data + if (!dataMappingMode.HasValue || symbol.SecurityType != SecurityType.Future || !symbol.IsCanonical() + || !LeanData.IsCommonLeanDataType(config.Type)) + { + return dataMappingMode; + } + return GetDataMappingModeOrDefault(symbol, dataMappingMode); + } + private int GetTickTypeOrder(SecurityType securityType, TickType tickType) { return SubscriptionManager.AvailableDataTypes[securityType].IndexOf(tickType); } - private IEnumerable GetMatchingSubscriptions(Symbol symbol, Type type, Resolution? resolution = null, bool useAllSubscriptions = false) + private IEnumerable GetMatchingSubscriptions(Symbol symbol, Type type, Resolution? resolution = null, bool useAllSubscriptions = false, + DataMappingMode? dataMappingMode = null) { var subscriptions = SubscriptionManager.SubscriptionDataConfigService // we add internal subscription so that history requests are covered, this allows us to warm them up too @@ -1293,7 +1309,9 @@ private IEnumerable GetMatchingSubscriptions(Symbol symb // Inherit values from existing subscriptions or use defaults var extendedMarketHours = userConfigIfAny?.ExtendedMarketHours ?? UniverseSettings.ExtendedMarketHours; var dataNormalizationMode = userConfigIfAny?.DataNormalizationMode ?? UniverseSettings.GetUniverseNormalizationModeOrDefault(symbol.SecurityType); - var dataMappingMode = userConfigIfAny?.DataMappingMode ?? UniverseSettings.GetUniverseMappingModeOrDefault(symbol.SecurityType, symbol.ID.Market); + var requestedDataMappingMode = dataMappingMode ?? userConfigIfAny?.DataMappingMode; + // only continuous futures data falls back from an unavailable mapping mode, other data types are not mapped + var unmappedDataMappingMode = requestedDataMappingMode ?? UniverseSettings.GetUniverseMappingModeOrDefault(symbol.SecurityType, symbol.ID.Market); var contractDepthOffset = userConfigIfAny?.ContractDepthOffset ?? (uint)Math.Abs(UniverseSettings.ContractDepthOffset); // If type was specified and not a lean data type and also not abstract, we create a new subscription @@ -1325,7 +1343,7 @@ private IEnumerable GetMatchingSubscriptions(Symbol symb LeanData.GetCommonTickTypeForCommonDataTypes(dataType, symbol.SecurityType), true, dataNormalizationMode, - dataMappingMode, + unmappedDataMappingMode, contractDepthOffset)}; } @@ -1341,6 +1359,9 @@ private IEnumerable GetMatchingSubscriptions(Symbol symb // Use the config type to get an accurate mhdb entry var entry = MarketHoursDatabase.GetEntry(symbol, new[] { configType }); var res = GetResolution(symbol, resolution, configType); + var configDataMappingMode = LeanData.IsCommonLeanDataType(configType) + ? GetDataMappingModeOrDefault(symbol, requestedDataMappingMode) + : unmappedDataMappingMode; return new SubscriptionDataConfig( configType, @@ -1355,7 +1376,7 @@ private IEnumerable GetMatchingSubscriptions(Symbol symb x.Item2, true, dataNormalizationMode, - dataMappingMode, + configDataMappingMode, contractDepthOffset); }) // lets make sure to respect the order of the data types, if used on a history request will affect outcome when using pushthrough for example diff --git a/Algorithm/QCAlgorithm.cs b/Algorithm/QCAlgorithm.cs index fc79bf006ed8..ed31d96b7be5 100644 --- a/Algorithm/QCAlgorithm.cs +++ b/Algorithm/QCAlgorithm.cs @@ -153,6 +153,8 @@ public partial class QCAlgorithm : MarshalByRefObject, IAlgorithm private int? _warmupBarCount; private Dictionary _parameters = new Dictionary(); private bool _deploymentDetailsSet; + private readonly HashSet _dataMappingModeFallbackWarnedMarkets = new(); + private bool _ignoredContinuousFutureSettingsWarningSent; private SecurityDefinitionSymbolResolver _securityDefinitionSymbolResolver; private SecurityDefinitionSymbolResolver SecurityDefinitionSymbolResolver @@ -2102,7 +2104,7 @@ public Security AddSecurity(Symbol symbol, Resolution? resolution = null, bool? { ExtendedMarketHours = extendedMarketHours.Value, FillForward = fillForward.Value, - DataMappingMode = dataMappingMode ?? UniverseSettings.GetUniverseMappingModeOrDefault(symbol.SecurityType, symbol.ID.Market), + DataMappingMode = GetDataMappingModeOrDefault(symbol, dataMappingMode), DataNormalizationMode = dataNormalizationMode ?? UniverseSettings.GetUniverseNormalizationModeOrDefault(symbol.SecurityType), ContractDepthOffset = (int)contractOffset, SubscriptionDataTypes = dataTypes, @@ -2127,12 +2129,73 @@ public Security AddSecurity(Symbol symbol, Resolution? resolution = null, bool? AddUniverse(universe); } + else if (symbol.SecurityType == SecurityType.Future + && UniverseManager.TryGetValue(ContinuousContractUniverse.CreateSymbol(symbol), out var continuousUniverse)) + { + var requestedDataMappingMode = dataMappingMode.HasValue ? GetDataMappingModeOrDefault(symbol, dataMappingMode) : (DataMappingMode?)null; + WarnIfContinuousFutureSettingsIgnored(symbol, continuousUniverse.UniverseSettings, requestedDataMappingMode, dataNormalizationMode, (int)contractOffset); + } return security; } return AddToUserDefinedUniverse(security, configs); } + /// + /// Gets the requested or default data mapping mode for the given symbol, falling back to the market default if it is not available, + /// warning once per market when it does + /// + private DataMappingMode GetDataMappingModeOrDefault(Symbol symbol, DataMappingMode? dataMappingMode = null) + { + var requestedDataMappingMode = dataMappingMode ?? UniverseSettings.DataMappingMode; + if (symbol.SecurityType != SecurityType.Future || !symbol.IsCanonical() || requestedDataMappingMode.IsAvailableForFutureMarket(symbol.ID.Market)) + { + return requestedDataMappingMode; + } + + var fallbackDataMappingMode = UniverseSettings.GetUniverseMappingModeOrDefault(symbol.SecurityType, symbol.ID.Market); + if (_dataMappingModeFallbackWarnedMarkets.Add(symbol.ID.Market)) + { + Debug($"Warning: {requestedDataMappingMode} data mapping mode is not available for {symbol.ID.Market.ToUpperInvariant()} futures, using {fallbackDataMappingMode} instead."); + } + return fallbackDataMappingMode; + } + + /// + /// Warns once if a future is added again with continuous contract settings that differ from the existing ones, which are kept + /// + private void WarnIfContinuousFutureSettingsIgnored(Symbol symbol, UniverseSettings existingSettings, DataMappingMode? dataMappingMode, + DataNormalizationMode? dataNormalizationMode, int contractDepthOffset) + { + if (_ignoredContinuousFutureSettingsWarningSent) + { + return; + } + + var ignoredSettings = new List(); + if (dataMappingMode.HasValue && dataMappingMode != existingSettings.DataMappingMode) + { + ignoredSettings.Add($"data mapping mode {dataMappingMode}"); + } + if (dataNormalizationMode.HasValue && dataNormalizationMode != existingSettings.DataNormalizationMode) + { + ignoredSettings.Add($"normalization mode {dataNormalizationMode}"); + } + if (contractDepthOffset != existingSettings.ContractDepthOffset) + { + ignoredSettings.Add($"contract depth offset {contractDepthOffset}"); + } + + if (ignoredSettings.Count > 0) + { + _ignoredContinuousFutureSettingsWarningSent = true; + var instructions = _locked + ? "Remove it first to change its settings." + : "Add it once, or remove and re-add it after Initialize."; + Debug($"Warning: {symbol} already added, ignoring {string.Join(", ", ignoredSettings)}. {instructions}"); + } + } + /// /// Creates and adds a new security to the algorithm /// diff --git a/Common/Extensions.cs b/Common/Extensions.cs index be982b6c8f16..c937c1ae3db8 100644 --- a/Common/Extensions.cs +++ b/Common/Extensions.cs @@ -3947,9 +3947,7 @@ public static DataMappingMode GetUniverseMappingModeOrDefault(this UniverseSetti switch (securityType) { case SecurityType.Future: - if ((universeSettings.DataMappingMode == DataMappingMode.OpenInterest - || universeSettings.DataMappingMode == DataMappingMode.OpenInterestAnnual) - && (market == Market.HKFE || market == Market.EUREX || market == Market.ICE)) + if (!universeSettings.DataMappingMode.IsAvailableForFutureMarket(market)) { // circle around default OI for currently no OI available data return DataMappingMode.LastTradingDay; @@ -3960,6 +3958,15 @@ public static DataMappingMode GetUniverseMappingModeOrDefault(this UniverseSetti } } + /// + /// Determines whether there is mapping data for the given data mapping mode in the given future market + /// + public static bool IsAvailableForFutureMarket(this DataMappingMode dataMappingMode, string market) + { + return dataMappingMode != DataMappingMode.OpenInterest && dataMappingMode != DataMappingMode.OpenInterestAnnual + || market != Market.HKFE && market != Market.EUREX && market != Market.ICE && market != Market.KRX; + } + /// /// Helper method to determine the right data normalization mode to use by default /// diff --git a/Tests/Algorithm/AlgorithmAddSecurityTests.cs b/Tests/Algorithm/AlgorithmAddSecurityTests.cs index 28cedbefcfc8..e885e55d304b 100644 --- a/Tests/Algorithm/AlgorithmAddSecurityTests.cs +++ b/Tests/Algorithm/AlgorithmAddSecurityTests.cs @@ -147,6 +147,143 @@ public void ProperlyAddsFutureWithExtendedMarketHours( Has.All.EqualTo(extendedMarketHours)); } + [TestCase("KM", Market.KRX, null, DataMappingMode.LastTradingDay, true)] + [TestCase("FESX", Market.EUREX, null, DataMappingMode.LastTradingDay, true)] + [TestCase("ES", Market.CME, null, DataMappingMode.OpenInterest, false)] + [TestCase("KM", Market.KRX, DataMappingMode.OpenInterest, DataMappingMode.LastTradingDay, true)] + [TestCase("FESX", Market.EUREX, DataMappingMode.OpenInterest, DataMappingMode.LastTradingDay, true)] + [TestCase("FESX", Market.EUREX, DataMappingMode.OpenInterestAnnual, DataMappingMode.LastTradingDay, true)] + [TestCase("FESX", Market.EUREX, DataMappingMode.FirstDayMonth, DataMappingMode.FirstDayMonth, false)] + [TestCase("ES", Market.CME, DataMappingMode.OpenInterest, DataMappingMode.OpenInterest, false)] + public void AddFutureFallsBackFromUnavailableDataMappingMode(string ticker, string market, DataMappingMode? dataMappingMode, + DataMappingMode expectedMode, bool expectWarning) + { + var future = _algo.AddFuture(ticker, Resolution.Daily, market, dataMappingMode: dataMappingMode); + + var continuousConfigs = _algo.SubscriptionManager.SubscriptionDataConfigService + .GetSubscriptionDataConfigs(future.Symbol, includeInternalConfigs: true) + .Where(x => x.Type != typeof(FutureUniverse)) + .ToList(); + Assert.Greater(continuousConfigs.Count, 0); + Assert.That(continuousConfigs.Select(x => x.DataMappingMode), Has.All.EqualTo(expectedMode)); + + var warnings = _algo.DebugMessages.Where(x => x.Contains("data mapping mode is not available")).ToList(); + if (expectWarning) + { + Assert.AreEqual(1, warnings.Count); + var requestedMode = dataMappingMode ?? DataMappingMode.OpenInterest; + Assert.That(warnings[0], Does.EndWith($"Warning: {requestedMode} data mapping mode is not available for {market.ToUpperInvariant()} futures, using LastTradingDay instead.")); + } + else + { + Assert.IsEmpty(warnings); + } + } + + [Test] + public void AddFutureDataMappingModeFallbackWarnsOncePerMarket() + { + _algo.AddFuture("FESX", Resolution.Daily, Market.EUREX, dataMappingMode: DataMappingMode.OpenInterest); + _algo.AddFuture("FDAX", Resolution.Daily, Market.EUREX); + _algo.AddFuture("HSI", Resolution.Daily, Market.HKFE, dataMappingMode: DataMappingMode.OpenInterest); + _algo.AddFuture("KM", Resolution.Daily, Market.KRX); + + var warnings = _algo.DebugMessages.Where(x => x.Contains("data mapping mode is not available")).ToList(); + Assert.AreEqual(3, warnings.Count); + Assert.That(warnings[0], Does.EndWith("Warning: OpenInterest data mapping mode is not available for EUREX futures, using LastTradingDay instead.")); + Assert.That(warnings[1], Does.EndWith("Warning: OpenInterest data mapping mode is not available for HKFE futures, using LastTradingDay instead.")); + Assert.That(warnings[2], Does.EndWith("Warning: OpenInterest data mapping mode is not available for KRX futures, using LastTradingDay instead.")); + } + + [Test] + public void AddFutureFallsBackToUniverseSettingsDataMappingMode() + { + _algo.UniverseSettings.DataMappingMode = DataMappingMode.FirstDayMonth; + var future = _algo.AddFuture("FESX", Resolution.Daily, Market.EUREX, dataMappingMode: DataMappingMode.OpenInterest); + + var continuousConfigs = _algo.SubscriptionManager.SubscriptionDataConfigService + .GetSubscriptionDataConfigs(future.Symbol, includeInternalConfigs: true) + .Where(x => x.Type != typeof(FutureUniverse)) + .ToList(); + Assert.That(continuousConfigs.Select(x => x.DataMappingMode), Has.All.EqualTo(DataMappingMode.FirstDayMonth)); + Assert.That(_algo.DebugMessages.Single(x => x.Contains("data mapping mode is not available")), + Does.EndWith("Warning: OpenInterest data mapping mode is not available for EUREX futures, using FirstDayMonth instead.")); + } + + [Test] + public void AddFutureAfterRemovingItAppliesNewSettings() + { + var future = _algo.AddFuture(Futures.Indices.SP500EMini, Resolution.Daily, dataMappingMode: DataMappingMode.OpenInterest); + _algo.RemoveSecurity(future.Symbol); + future = _algo.AddFuture(Futures.Indices.SP500EMini, Resolution.Daily, dataMappingMode: DataMappingMode.LastTradingDay); + + var continuousUniverse = _algo.UniverseManager[ContinuousContractUniverse.CreateSymbol(future.Symbol)]; + Assert.AreEqual(DataMappingMode.LastTradingDay, continuousUniverse.UniverseSettings.DataMappingMode); + Assert.IsFalse(_algo.DebugMessages.Any(x => x.Contains("already added"))); + } + + [Test] + public void AddFutureAgainWithDifferentSettingsWarnsOnce() + { + var future = _algo.AddFuture(Futures.Indices.SP500EMini, Resolution.Daily, dataMappingMode: DataMappingMode.OpenInterest); + _algo.AddFuture(Futures.Indices.SP500EMini, Resolution.Daily, dataMappingMode: DataMappingMode.LastTradingDay); + _algo.AddFuture(Futures.Indices.SP500EMini, Resolution.Daily, dataMappingMode: DataMappingMode.FirstDayMonth); + + var continuousConfigs = _algo.SubscriptionManager.SubscriptionDataConfigService + .GetSubscriptionDataConfigs(future.Symbol, includeInternalConfigs: true) + .Where(x => x.Type != typeof(FutureUniverse)) + .ToList(); + Assert.That(continuousConfigs.Select(x => x.DataMappingMode), Has.All.EqualTo(DataMappingMode.OpenInterest)); + + var warnings = _algo.DebugMessages.Where(x => x.Contains("already added")).ToList(); + Assert.AreEqual(1, warnings.Count); + Assert.That(warnings[0], Does.EndWith("Warning: /ES already added, ignoring data mapping mode LastTradingDay. Add it once, or remove and re-add it after Initialize.")); + } + + [Test] + public void AddFutureAgainWithAllSettingsDifferentListsThemInOneWarning() + { + _algo.AddFuture(Futures.Indices.SP500EMini, Resolution.Daily); + _algo.AddFuture(Futures.Indices.SP500EMini, Resolution.Daily, dataMappingMode: DataMappingMode.LastTradingDay, + dataNormalizationMode: DataNormalizationMode.Raw, contractDepthOffset: 1); + + var warnings = _algo.DebugMessages.Where(x => x.Contains("already added")).ToList(); + Assert.AreEqual(1, warnings.Count); + Assert.That(warnings[0], Does.EndWith("Warning: /ES already added, ignoring data mapping mode LastTradingDay, normalization mode Raw, contract depth offset 1. " + + "Add it once, or remove and re-add it after Initialize.")); + } + + [Test] + public void AddFutureAgainAfterInitializeWarnsToRemoveItFirst() + { + _algo.AddFuture(Futures.Indices.SP500EMini, Resolution.Daily, dataMappingMode: DataMappingMode.OpenInterest); + _algo.SetLocked(); + _algo.AddFuture(Futures.Indices.SP500EMini, Resolution.Daily, dataMappingMode: DataMappingMode.LastTradingDay); + + Assert.That(_algo.DebugMessages.Single(x => x.Contains("already added")), + Does.EndWith("Warning: /ES already added, ignoring data mapping mode LastTradingDay. Remove it first to change its settings.")); + } + + [TestCase(null)] + [TestCase(DataMappingMode.OpenInterest)] + public void AddFutureAgainWithSameSettingsDoesNotWarn(DataMappingMode? dataMappingMode) + { + _algo.AddFuture(Futures.Indices.SP500EMini, Resolution.Daily, dataMappingMode: DataMappingMode.OpenInterest); + _algo.AddFuture(Futures.Indices.SP500EMini, Resolution.Daily, dataMappingMode: dataMappingMode); + + Assert.IsFalse(_algo.DebugMessages.Any(x => x.Contains("already added"))); + } + + [Test] + public void AddFutureAgainWithUnavailableDataMappingModeResolvingToSameModeDoesNotWarn() + { + _algo.AddFuture("FESX", Resolution.Daily, Market.EUREX); + _algo.AddFuture("FESX", Resolution.Daily, Market.EUREX, dataMappingMode: DataMappingMode.OpenInterest); + + Assert.IsFalse(_algo.DebugMessages.Any(x => x.Contains("already added"))); + Assert.AreEqual(1, _algo.DebugMessages.Count(x => x.Contains("data mapping mode is not available"))); + } + [TestCaseSource(nameof(FuturesTestCases))] public void AddFutureWithExtendedMarketHours(Func getFuture) { diff --git a/Tests/Algorithm/AlgorithmHistoryTests.cs b/Tests/Algorithm/AlgorithmHistoryTests.cs index 2a2991fb1bb0..270905a95a00 100644 --- a/Tests/Algorithm/AlgorithmHistoryTests.cs +++ b/Tests/Algorithm/AlgorithmHistoryTests.cs @@ -70,6 +70,71 @@ public void OneTimeSetUp() FundamentalService.Initialize(_dataProvider, new NullFundamentalDataProvider(), false); } + [TestCase(null)] + [TestCase(DataMappingMode.FirstDayMonth)] + public void HistoryWithExplicitUnavailableDataMappingModeFallsBackToMarketDefault(DataMappingMode? addedFutureDataMappingMode) + { + var symbol = Symbol.Create("FESX", SecurityType.Future, Market.EUREX); + if (addedFutureDataMappingMode.HasValue) + { + _algorithm.AddFuture("FESX", Resolution.Daily, Market.EUREX, dataMappingMode: addedFutureDataMappingMode); + } + + _algorithm.History(symbol, 5, Resolution.Daily, dataMappingMode: DataMappingMode.OpenInterest).ToList(); + _algorithm.History(symbol, _algorithm.Time.AddDays(-5), _algorithm.Time, Resolution.Daily, dataMappingMode: DataMappingMode.OpenInterest).ToList(); + + var requests = _testHistoryProvider.HistryRequests.Where(x => x.Symbol == symbol).ToList(); + Assert.Greater(requests.Count, 0); + // the market default is used, not the added future's mode + Assert.That(requests.Select(x => x.DataMappingMode), Has.All.EqualTo(DataMappingMode.LastTradingDay)); + Assert.That(_algorithm.DebugMessages.Single(x => x.Contains("data mapping mode is not available")), + Does.EndWith("Warning: OpenInterest data mapping mode is not available for EUREX futures, using LastTradingDay instead.")); + } + + [Test] + public void HistoryWithoutDataMappingModeFallsBackForFutureNotAdded() + { + var symbol = Symbol.Create("FESX", SecurityType.Future, Market.EUREX); + + _algorithm.History(symbol, 5, Resolution.Daily).ToList(); + + var requests = _testHistoryProvider.HistryRequests.Where(x => x.Symbol == symbol).ToList(); + Assert.Greater(requests.Count, 0); + Assert.That(requests.Select(x => x.DataMappingMode), Has.All.EqualTo(DataMappingMode.LastTradingDay)); + Assert.That(_algorithm.DebugMessages.Single(x => x.Contains("data mapping mode is not available")), + Does.EndWith("Warning: OpenInterest data mapping mode is not available for EUREX futures, using LastTradingDay instead.")); + } + + [Test] + public void HistoryWithExplicitAvailableDataMappingModeKeepsIt() + { + var symbol = Symbol.Create("FESX", SecurityType.Future, Market.EUREX); + + _algorithm.History(symbol, 5, Resolution.Daily, dataMappingMode: DataMappingMode.FirstDayMonth).ToList(); + + var requests = _testHistoryProvider.HistryRequests.Where(x => x.Symbol == symbol).ToList(); + Assert.Greater(requests.Count, 0); + Assert.That(requests.Select(x => x.DataMappingMode), Has.All.EqualTo(DataMappingMode.FirstDayMonth)); + Assert.IsFalse(_algorithm.DebugMessages.Any(x => x.Contains("data mapping mode is not available"))); + } + + [Test] + public void HistoryWithExplicitUnavailableDataMappingModeDoesNotFallBackForContractsOrChains() + { + var symbol = Symbol.Create("FESX", SecurityType.Future, Market.EUREX); + var contract = Symbol.CreateFuture("FESX", Market.EUREX, new DateTime(2024, 6, 21)); + + _algorithm.History(contract, 5, Resolution.Daily, dataMappingMode: DataMappingMode.OpenInterest).ToList(); + _algorithm.History(symbol, 5, Resolution.Daily, dataMappingMode: DataMappingMode.OpenInterest).ToList(); + + var contractRequests = _testHistoryProvider.HistryRequests.Where(x => x.Symbol == contract).ToList(); + var chainRequests = _testHistoryProvider.HistryRequests.Where(x => x.DataType == typeof(FutureUniverse)).ToList(); + Assert.Greater(contractRequests.Count, 0); + Assert.Greater(chainRequests.Count, 0); + Assert.That(contractRequests.Concat(chainRequests).Select(x => x.DataMappingMode), Has.All.EqualTo(DataMappingMode.OpenInterest)); + Assert.IsFalse(_algorithm.DebugMessages.Any(x => x.Contains("data mapping mode is not available"))); + } + [TestCase(Language.Python)] [TestCase(Language.CSharp)] public void FundamentalHistory(Language language) diff --git a/Tests/Common/Util/ExtensionsTests.cs b/Tests/Common/Util/ExtensionsTests.cs index 696511e1bf21..19bad6fcfbe1 100644 --- a/Tests/Common/Util/ExtensionsTests.cs +++ b/Tests/Common/Util/ExtensionsTests.cs @@ -62,6 +62,21 @@ public void GetEnumValuesInValue(string saleCondition, params TradeConditionFlag Assert.AreEqual(expected, enums); } + [TestCase(Market.CME, DataMappingMode.OpenInterest, DataMappingMode.OpenInterest)] + [TestCase(Market.CME, DataMappingMode.OpenInterestAnnual, DataMappingMode.OpenInterestAnnual)] + [TestCase(Market.EUREX, DataMappingMode.OpenInterest, DataMappingMode.LastTradingDay)] + [TestCase(Market.HKFE, DataMappingMode.OpenInterest, DataMappingMode.LastTradingDay)] + [TestCase(Market.ICE, DataMappingMode.OpenInterest, DataMappingMode.LastTradingDay)] + [TestCase(Market.KRX, DataMappingMode.OpenInterest, DataMappingMode.LastTradingDay)] + [TestCase(Market.KRX, DataMappingMode.OpenInterestAnnual, DataMappingMode.LastTradingDay)] + [TestCase(Market.KRX, DataMappingMode.FirstDayMonth, DataMappingMode.FirstDayMonth)] + public void GetUniverseMappingModeOrDefault(string market, DataMappingMode settingsMode, DataMappingMode expected) + { + var settings = new UniverseSettings(Resolution.Daily, 1, true, false, TimeSpan.Zero) { DataMappingMode = settingsMode }; + + Assert.AreEqual(expected, settings.GetUniverseMappingModeOrDefault(SecurityType.Future, market)); + } + [TestCase("tt", "", "tt")] [TestCase("tt", "t", "t")] [TestCase("tt", "tt", "")]