From f5c61d775a52700f6976ddd3a938a452ff5ed6f5 Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Thu, 22 Feb 2024 13:21:07 -0400 Subject: [PATCH 1/3] Return null on unsupported history requests --- .../BinanceBrokerageDataQueueHandlerTests.cs | 8 +- .../BinanceBrokerageHistoryProviderTests.cs | 169 +++++++----------- ...oinFuturesBrokerageHistoryProviderTests.cs | 35 ++-- ...nceFuturesBrokerageHistoryProviderTests.cs | 33 ++-- ...BinanceUSBrokerageDataQueueHandlerTests.cs | 8 +- .../BinanceUSBrokerageHistoryProviderTests.cs | 36 ++-- .../BaseDataDownloader.cs | 2 +- .../BinanceBrokerage.cs | 51 ++++-- 8 files changed, 169 insertions(+), 173 deletions(-) diff --git a/QuantConnect.BinanceBrokerage.Tests/BinanceBrokerageDataQueueHandlerTests.cs b/QuantConnect.BinanceBrokerage.Tests/BinanceBrokerageDataQueueHandlerTests.cs index a9f3c97..b67ef5e 100644 --- a/QuantConnect.BinanceBrokerage.Tests/BinanceBrokerageDataQueueHandlerTests.cs +++ b/QuantConnect.BinanceBrokerage.Tests/BinanceBrokerageDataQueueHandlerTests.cs @@ -1,11 +1,11 @@ /* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); + * + * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -18,6 +18,7 @@ using QuantConnect.Data.Market; using QuantConnect.Interfaces; using QuantConnect.Logging; +using QuantConnect.Tests; using System; using System.Threading; @@ -30,6 +31,7 @@ private static TestCaseData[] TestParameters { get { + TestGlobals.Initialize(); var FTM_USDT = Symbol.Create("FTMUSDT", SecurityType.Crypto, Market.Binance); return new[] diff --git a/QuantConnect.BinanceBrokerage.Tests/BinanceBrokerageHistoryProviderTests.cs b/QuantConnect.BinanceBrokerage.Tests/BinanceBrokerageHistoryProviderTests.cs index f1b9db2..c8a933b 100644 --- a/QuantConnect.BinanceBrokerage.Tests/BinanceBrokerageHistoryProviderTests.cs +++ b/QuantConnect.BinanceBrokerage.Tests/BinanceBrokerageHistoryProviderTests.cs @@ -20,14 +20,11 @@ using QuantConnect.Data; using QuantConnect.Data.Market; using QuantConnect.Lean.Engine.DataFeeds; -using QuantConnect.Lean.Engine.HistoricalData; using QuantConnect.Logging; using QuantConnect.Securities; using QuantConnect.Tests; using System; using System.Linq; -using QuantConnect.Lean.Engine.Storage; -using QuantConnect.Storage; namespace QuantConnect.BinanceBrokerage.Tests { @@ -39,6 +36,7 @@ public class BinanceBrokerageHistoryProviderTests [OneTimeSetUp] public void Setup() { + TestGlobals.Initialize(); _brokerage = CreateBrokerage(); } @@ -52,9 +50,9 @@ public void TearDown() [Test] [TestCaseSource(nameof(ValidHistory))] [TestCaseSource(nameof(InvalidHistory))] - public virtual void GetsHistory(Symbol symbol, Resolution resolution, TimeSpan period, bool throwsException) + public virtual void GetsHistory(Symbol symbol, Resolution resolution, TimeSpan period, TickType tickType, bool unsupported) { - BaseHistoryTest(symbol, resolution, period, throwsException, _brokerage); + BaseHistoryTest(symbol, resolution, period, tickType, unsupported, _brokerage); } [Test] @@ -66,95 +64,61 @@ public virtual void GetEmptyHistory(Symbol symbol, Resolution resolution, TimeSp public static void BaseEmptyHistoryTest(Symbol symbol, Resolution resolution, TimeSpan period, TickType tickType, Brokerage brokerage) { - TestDelegate test = () => - { - var historyProvider = new BrokerageHistoryProvider(); - historyProvider.SetBrokerage(brokerage); - historyProvider.Initialize(new HistoryProviderInitializeParameters(null, null, null, null, null, null, null, false, new DataPermissionManager(), new LocalObjectStore())); - - var now = DateTime.UtcNow; - - var requests = new[] - { - new HistoryRequest(now.Add(-period), - now, - typeof(TradeBar), - symbol, - resolution, - SecurityExchangeHours.AlwaysOpen(TimeZones.Utc), - DateTimeZone.Utc, - Resolution.Minute, - false, - false, - DataNormalizationMode.Adjusted, - tickType) - }; - - var history = historyProvider.GetHistory(requests, TimeZones.Utc).ToList(); - - Log.Trace("Data points retrieved: " + historyProvider.DataPointCount); - Assert.AreEqual(0, historyProvider.DataPointCount); - }; - - Assert.DoesNotThrow(test); + var now = DateTime.UtcNow; + var request = new HistoryRequest(now.Add(-period), + now, + typeof(TradeBar), + symbol, + resolution, + SecurityExchangeHours.AlwaysOpen(TimeZones.Utc), + DateTimeZone.Utc, + Resolution.Minute, + false, + false, + DataNormalizationMode.Adjusted, + tickType); + + var history = brokerage.GetHistory(request)?.ToList(); + + Assert.IsNotNull(history); + + Log.Debug("Data points retrieved: " + history.Count); + Assert.AreEqual(0, history.Count); } - public static void BaseHistoryTest(Symbol symbol, Resolution resolution, TimeSpan period, bool throwsException, Brokerage brokerage) + public static void BaseHistoryTest(Symbol symbol, Resolution resolution, TimeSpan period, TickType tickType, bool unsupported, Brokerage brokerage) { - TestDelegate test = () => + var now = DateTime.UtcNow; + var request = new HistoryRequest(now.Add(-period), + now, + typeof(TradeBar), + symbol, + resolution, + SecurityExchangeHours.AlwaysOpen(TimeZones.Utc), + DateTimeZone.Utc, + Resolution.Minute, + false, + false, + DataNormalizationMode.Adjusted, + tickType); + + var history = brokerage.GetHistory(request)?.ToList(); + + if (unsupported) { - var historyProvider = new BrokerageHistoryProvider(); - historyProvider.SetBrokerage(brokerage); - historyProvider.Initialize(new HistoryProviderInitializeParameters(null, null, null, null, null, null, null, false, new DataPermissionManager(), new LocalObjectStore())); - - var now = DateTime.UtcNow; - - var requests = new[] - { - new HistoryRequest(now.Add(-period), - now, - typeof(TradeBar), - symbol, - resolution, - SecurityExchangeHours.AlwaysOpen(TimeZones.Utc), - DateTimeZone.Utc, - Resolution.Minute, - false, - false, - DataNormalizationMode.Adjusted, - TickType.Trade) - }; - - var history = historyProvider.GetHistory(requests, TimeZones.Utc); + Assert.IsNull(history); + } + else + { + Assert.IsNotNull(history); - foreach (var slice in history) + foreach (var bar in history.Cast()) { - if (resolution == Resolution.Tick) - { - foreach (var tick in slice.Ticks[symbol]) - { - Log.Trace("{0}: {1} - {2} / {3}", tick.Time.ToStringInvariant("yyyy-MM-dd HH:mm:ss.fff"), tick.Symbol, tick.BidPrice, tick.AskPrice); - } - } - else - { - var bar = slice.Bars[symbol]; - - Log.Trace("{0}: {1} - O={2}, H={3}, L={4}, C={5}", bar.Time, bar.Symbol, bar.Open, bar.High, bar.Low, bar.Close); - } + Log.Trace("{0}: {1} - O={2}, H={3}, L={4}, C={5}", bar.Time, bar.Symbol, bar.Open, bar.High, bar.Low, bar.Close); } - Assert.Greater(historyProvider.DataPointCount, 0); - Log.Debug("Data points retrieved: " + historyProvider.DataPointCount); - }; - - if (throwsException) - { - Assert.Throws(test); - } - else - { - Assert.DoesNotThrow(test); + Assert.Greater(history.Count, 0); + Log.Debug("Data points retrieved: " + history.Count); } } @@ -164,10 +128,9 @@ private static TestCaseData[] ValidHistory { return new[] { - // valid - new TestCaseData(Symbol.Create("ETHUSDT", SecurityType.Crypto, Market.Binance), Resolution.Minute, Time.OneHour, false), - new TestCaseData(Symbol.Create("ETHUSDT", SecurityType.Crypto, Market.Binance), Resolution.Hour, Time.OneDay, false), - new TestCaseData(Symbol.Create("ETHUSDT", SecurityType.Crypto, Market.Binance), Resolution.Daily, TimeSpan.FromDays(15), false), + new TestCaseData(Symbol.Create("ETHUSDT", SecurityType.Crypto, Market.Binance), Resolution.Minute, Time.OneHour, TickType.Trade, false), + new TestCaseData(Symbol.Create("ETHUSDT", SecurityType.Crypto, Market.Binance), Resolution.Hour, Time.OneDay, TickType.Trade, false), + new TestCaseData(Symbol.Create("ETHUSDT", SecurityType.Crypto, Market.Binance), Resolution.Daily, TimeSpan.FromDays(15), TickType.Trade, false), }; } } @@ -178,9 +141,8 @@ private static TestCaseData[] NoHistory { return new[] { - new TestCaseData(Symbol.Create("ETHUSDT", SecurityType.Crypto, Market.Binance), Resolution.Tick, TimeSpan.FromSeconds(15), TickType.Trade), - new TestCaseData(Symbol.Create("ETHUSDT", SecurityType.Crypto, Market.Binance), Resolution.Second, Time.OneMinute, TickType.Trade), - new TestCaseData(Symbol.Create("ETHUSDT", SecurityType.Crypto, Market.Binance), Resolution.Minute, Time.OneHour, TickType.Quote), + // invalid period + new TestCaseData(Symbol.Create("ETHUSDT", SecurityType.Crypto, Market.Binance), Resolution.Daily, TimeSpan.FromDays(-15), TickType.Trade), }; } } @@ -191,14 +153,19 @@ private static TestCaseData[] InvalidHistory { return new[] { - // invalid period, no error, empty result - new TestCaseData(Symbols.EURUSD, Resolution.Daily, TimeSpan.FromDays(-15), false), - - // invalid symbol, throws "System.ArgumentException : Unknown symbol: XYZ" - new TestCaseData(Symbol.Create("XYZ", SecurityType.Crypto, Market.Binance), Resolution.Daily, TimeSpan.FromDays(15), true), - - // invalid security type, throws "System.ArgumentException : Invalid security type: Equity" - new TestCaseData(Symbols.AAPL, Resolution.Daily, TimeSpan.FromDays(15), false), + // invalid symbol + new TestCaseData(Symbol.Create("XYZ", SecurityType.Crypto, Market.Binance), Resolution.Daily, TimeSpan.FromDays(15), TickType.Trade, true), + // invalid security type + new TestCaseData(Symbols.AAPL, Resolution.Daily, TimeSpan.FromDays(15), TickType.Trade, true), + new TestCaseData(Symbols.USDJPY, Resolution.Daily, TimeSpan.FromDays(15), TickType.Trade, true), + // invalid market + new TestCaseData(Symbol.Create("ETHUSDT", SecurityType.Crypto, Market.BinanceUS), Resolution.Daily, TimeSpan.FromDays(15), TickType.Trade, true), + // invalid resolution + new TestCaseData(Symbol.Create("ETHUSDT", SecurityType.Crypto, Market.Binance), Resolution.Tick, TimeSpan.FromSeconds(15), TickType.Trade, true), + new TestCaseData(Symbol.Create("ETHUSDT", SecurityType.Crypto, Market.Binance), Resolution.Second, Time.OneMinute, TickType.Trade, true), + // Invalid tick type + new TestCaseData(Symbol.Create("ETHUSDT", SecurityType.Crypto, Market.Binance), Resolution.Minute, Time.OneHour, TickType.Quote, true), + new TestCaseData(Symbol.Create("ETHUSDT", SecurityType.Crypto, Market.Binance), Resolution.Minute, Time.OneHour, TickType.OpenInterest, true), }; } } diff --git a/QuantConnect.BinanceBrokerage.Tests/BinanceCoinFuturesBrokerageHistoryProviderTests.cs b/QuantConnect.BinanceBrokerage.Tests/BinanceCoinFuturesBrokerageHistoryProviderTests.cs index e4c9b0f..d0a09d6 100644 --- a/QuantConnect.BinanceBrokerage.Tests/BinanceCoinFuturesBrokerageHistoryProviderTests.cs +++ b/QuantConnect.BinanceBrokerage.Tests/BinanceCoinFuturesBrokerageHistoryProviderTests.cs @@ -43,9 +43,9 @@ public void TearDown() [Test] [TestCaseSource(nameof(ValidHistory))] [TestCaseSource(nameof(InvalidHistory))] - public virtual void GetsHistory(Symbol symbol, Resolution resolution, TimeSpan period, bool throwsException) + public virtual void GetsHistory(Symbol symbol, Resolution resolution, TimeSpan period, TickType tickType, bool unsupported) { - BinanceBrokerageHistoryProviderTests.BaseHistoryTest(symbol, resolution, period, throwsException, _brokerage); + BinanceBrokerageHistoryProviderTests.BaseHistoryTest(symbol, resolution, period, tickType, unsupported, _brokerage); } [Test] @@ -62,9 +62,9 @@ private static TestCaseData[] ValidHistory return new[] { // valid - new TestCaseData(Symbol.Create("ETHUSD", SecurityType.CryptoFuture, Market.Binance), Resolution.Minute, Time.OneHour, false), - new TestCaseData(Symbol.Create("ETHUSD", SecurityType.CryptoFuture, Market.Binance), Resolution.Hour, Time.OneDay, false), - new TestCaseData(Symbol.Create("ETHUSD", SecurityType.CryptoFuture, Market.Binance), Resolution.Daily, TimeSpan.FromDays(15), false), + new TestCaseData(Symbol.Create("ETHUSD", SecurityType.CryptoFuture, Market.Binance), Resolution.Minute, Time.OneHour, TickType.Trade, false), + new TestCaseData(Symbol.Create("ETHUSD", SecurityType.CryptoFuture, Market.Binance), Resolution.Hour, Time.OneDay, TickType.Trade, false), + new TestCaseData(Symbol.Create("ETHUSD", SecurityType.CryptoFuture, Market.Binance), Resolution.Daily, TimeSpan.FromDays(15), TickType.Trade, false), }; } } @@ -75,9 +75,8 @@ private static TestCaseData[] NoHistory { return new[] { - new TestCaseData(Symbol.Create("ETHUSD", SecurityType.CryptoFuture, Market.Binance), Resolution.Tick, TimeSpan.FromSeconds(15), TickType.Trade), - new TestCaseData(Symbol.Create("ETHUSD", SecurityType.CryptoFuture, Market.Binance), Resolution.Second, Time.OneMinute, TickType.Trade), - new TestCaseData(Symbol.Create("ETHUSD", SecurityType.CryptoFuture, Market.Binance), Resolution.Minute, Time.OneHour, TickType.Quote), + // invalid period, no error, empty result + new TestCaseData(Symbol.Create("ETHUSD", SecurityType.CryptoFuture, Market.Binance), Resolution.Minute, TimeSpan.FromDays(-15), TickType.Trade), }; } } @@ -88,14 +87,18 @@ private static TestCaseData[] InvalidHistory { return new[] { - // invalid period, no error, empty result - new TestCaseData(Symbols.EURUSD, Resolution.Daily, TimeSpan.FromDays(-15), false), - - // invalid symbol, throws "System.ArgumentException : Unknown symbol: XYZ" - new TestCaseData(Symbol.Create("XYZ", SecurityType.CryptoFuture, Market.Binance), Resolution.Daily, TimeSpan.FromDays(15), true), - - // invalid security type, throws "System.ArgumentException : Invalid security type: Equity" - new TestCaseData(Symbols.AAPL, Resolution.Daily, TimeSpan.FromDays(15), false), + // Invalid type of future + new TestCaseData(Symbol.Create("ETHUSDT", SecurityType.CryptoFuture, Market.Binance), Resolution.Minute, TimeSpan.FromSeconds(15), TickType.Trade, true), + // invalid resolution + new TestCaseData(Symbol.Create("ETHUSD", SecurityType.CryptoFuture, Market.Binance), Resolution.Tick, TimeSpan.FromSeconds(15), TickType.Trade, true), + new TestCaseData(Symbol.Create("ETHUSD", SecurityType.CryptoFuture, Market.Binance), Resolution.Second, Time.OneMinute, TickType.Trade, true), + // invalid tick type + new TestCaseData(Symbol.Create("ETHUSD", SecurityType.CryptoFuture, Market.Binance), Resolution.Minute, Time.OneHour, TickType.Quote, true), + new TestCaseData(Symbol.Create("ETHUSD", SecurityType.CryptoFuture, Market.Binance), Resolution.Minute, Time.OneHour, TickType.OpenInterest, true), + // invalid symbol + new TestCaseData(Symbol.Create("XYZ", SecurityType.CryptoFuture, Market.Binance), Resolution.Daily, TimeSpan.FromDays(15), TickType.Trade, true), + // invalid security type + new TestCaseData(Symbols.AAPL, Resolution.Daily, TimeSpan.FromDays(15), TickType.Trade, true), }; } } diff --git a/QuantConnect.BinanceBrokerage.Tests/BinanceFuturesBrokerageHistoryProviderTests.cs b/QuantConnect.BinanceBrokerage.Tests/BinanceFuturesBrokerageHistoryProviderTests.cs index a73e1ee..402cefa 100644 --- a/QuantConnect.BinanceBrokerage.Tests/BinanceFuturesBrokerageHistoryProviderTests.cs +++ b/QuantConnect.BinanceBrokerage.Tests/BinanceFuturesBrokerageHistoryProviderTests.cs @@ -43,9 +43,9 @@ public void TearDown() [Test] [TestCaseSource(nameof(ValidHistory))] [TestCaseSource(nameof(InvalidHistory))] - public virtual void GetsHistory(Symbol symbol, Resolution resolution, TimeSpan period, bool throwsException) + public virtual void GetsHistory(Symbol symbol, Resolution resolution, TimeSpan period, TickType tickType, bool unsupported) { - BinanceBrokerageHistoryProviderTests.BaseHistoryTest(symbol, resolution, period, throwsException, _brokerage); + BinanceBrokerageHistoryProviderTests.BaseHistoryTest(symbol, resolution, period, tickType, unsupported, _brokerage); } [Test] @@ -62,9 +62,9 @@ private static TestCaseData[] ValidHistory return new[] { // valid - new TestCaseData(Symbol.Create("ETHUSDT", SecurityType.CryptoFuture, Market.Binance), Resolution.Minute, Time.OneHour, false), - new TestCaseData(Symbol.Create("ETHUSDT", SecurityType.CryptoFuture, Market.Binance), Resolution.Hour, Time.OneDay, false), - new TestCaseData(Symbol.Create("ETHUSDT", SecurityType.CryptoFuture, Market.Binance), Resolution.Daily, TimeSpan.FromDays(15), false), + new TestCaseData(Symbol.Create("ETHUSDT", SecurityType.CryptoFuture, Market.Binance), Resolution.Minute, Time.OneHour, TickType.Trade, false), + new TestCaseData(Symbol.Create("ETHUSDT", SecurityType.CryptoFuture, Market.Binance), Resolution.Hour, Time.OneDay, TickType.Trade, false), + new TestCaseData(Symbol.Create("ETHUSDT", SecurityType.CryptoFuture, Market.Binance), Resolution.Daily, TimeSpan.FromDays(15), TickType.Trade, false), }; } } @@ -75,9 +75,8 @@ private static TestCaseData[] NoHistory { return new[] { - new TestCaseData(Symbol.Create("ETHUSDT", SecurityType.CryptoFuture, Market.Binance), Resolution.Tick, TimeSpan.FromSeconds(15), TickType.Trade), - new TestCaseData(Symbol.Create("ETHUSDT", SecurityType.CryptoFuture, Market.Binance), Resolution.Second, Time.OneMinute, TickType.Trade), - new TestCaseData(Symbol.Create("ETHUSDT", SecurityType.CryptoFuture, Market.Binance), Resolution.Minute, Time.OneHour, TickType.Quote), + // invalid period, no error, empty result + new TestCaseData(Symbol.Create("ETHUSDT", SecurityType.CryptoFuture, Market.Binance), Resolution.Daily, TimeSpan.FromDays(-15), TickType.Trade), }; } } @@ -88,14 +87,16 @@ private static TestCaseData[] InvalidHistory { return new[] { - // invalid period, no error, empty result - new TestCaseData(Symbols.EURUSD, Resolution.Daily, TimeSpan.FromDays(-15), false), - - // invalid symbol, throws "System.ArgumentException : Unknown symbol: XYZ" - new TestCaseData(Symbol.Create("XYZ", SecurityType.CryptoFuture, Market.Binance), Resolution.Daily, TimeSpan.FromDays(15), true), - - // invalid security type, throws "System.ArgumentException : Invalid security type: Equity" - new TestCaseData(Symbols.AAPL, Resolution.Daily, TimeSpan.FromDays(15), false), + // invalid resolution + new TestCaseData(Symbol.Create("ETHUSDT", SecurityType.CryptoFuture, Market.Binance), Resolution.Tick, TimeSpan.FromSeconds(15), TickType.Trade, true), + new TestCaseData(Symbol.Create("ETHUSDT", SecurityType.CryptoFuture, Market.Binance), Resolution.Second, Time.OneMinute, TickType.Trade, true), + //invalid tick type + new TestCaseData(Symbol.Create("ETHUSDT", SecurityType.CryptoFuture, Market.Binance), Resolution.Minute, Time.OneHour, TickType.Quote, true), + new TestCaseData(Symbol.Create("ETHUSDT", SecurityType.CryptoFuture, Market.Binance), Resolution.Minute, Time.OneHour, TickType.OpenInterest, true), + // invalid symbol + new TestCaseData(Symbol.Create("XYZ", SecurityType.CryptoFuture, Market.Binance), Resolution.Daily, TimeSpan.FromDays(15), TickType.Trade, true), + // invalid security type + new TestCaseData(Symbols.AAPL, Resolution.Daily, TimeSpan.FromDays(15), TickType.Trade, true), }; } } diff --git a/QuantConnect.BinanceBrokerage.Tests/BinanceUSBrokerageDataQueueHandlerTests.cs b/QuantConnect.BinanceBrokerage.Tests/BinanceUSBrokerageDataQueueHandlerTests.cs index f551123..cf321b9 100644 --- a/QuantConnect.BinanceBrokerage.Tests/BinanceUSBrokerageDataQueueHandlerTests.cs +++ b/QuantConnect.BinanceBrokerage.Tests/BinanceUSBrokerageDataQueueHandlerTests.cs @@ -1,11 +1,11 @@ /* * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); + * + * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -14,6 +14,7 @@ */ using NUnit.Framework; +using QuantConnect.Tests; namespace QuantConnect.BinanceBrokerage.Tests { @@ -24,6 +25,7 @@ private static TestCaseData[] TestParameters { get { + TestGlobals.Initialize(); var symbol = Symbol.Create("FTMUSDT", SecurityType.Crypto, Market.BinanceUS); return new[] diff --git a/QuantConnect.BinanceBrokerage.Tests/BinanceUSBrokerageHistoryProviderTests.cs b/QuantConnect.BinanceBrokerage.Tests/BinanceUSBrokerageHistoryProviderTests.cs index 0c81d9b..bcee401 100644 --- a/QuantConnect.BinanceBrokerage.Tests/BinanceUSBrokerageHistoryProviderTests.cs +++ b/QuantConnect.BinanceBrokerage.Tests/BinanceUSBrokerageHistoryProviderTests.cs @@ -13,19 +13,12 @@ * limitations under the License. */ -using NodaTime; using NUnit.Framework; using QuantConnect.Brokerages; using QuantConnect.Configuration; -using QuantConnect.Data; -using QuantConnect.Data.Market; using QuantConnect.Lean.Engine.DataFeeds; -using QuantConnect.Lean.Engine.HistoricalData; -using QuantConnect.Logging; -using QuantConnect.Securities; using QuantConnect.Tests; using System; -using System.Linq; namespace QuantConnect.BinanceBrokerage.Tests { @@ -35,9 +28,9 @@ public class BinanceUSBrokerageHistoryProviderTests : BinanceBrokerageHistoryPro [Test] [TestCaseSource(nameof(ValidHistory))] [TestCaseSource(nameof(InvalidHistory))] - public override void GetsHistory(Symbol symbol, Resolution resolution, TimeSpan period, bool throwsException) + public override void GetsHistory(Symbol symbol, Resolution resolution, TimeSpan period, TickType tickType, bool unsupported) { - base.GetsHistory(symbol, resolution, period, throwsException); + base.GetsHistory(symbol, resolution, period, tickType, unsupported); } [Test] @@ -67,9 +60,8 @@ private static TestCaseData[] NoHistory { return new[] { - new TestCaseData(Symbol.Create("ETHUSDT", SecurityType.Crypto, Market.BinanceUS), Resolution.Tick, TimeSpan.FromSeconds(15), TickType.Trade), - new TestCaseData(Symbol.Create("ETHUSDT", SecurityType.Crypto, Market.BinanceUS), Resolution.Second, Time.OneMinute, TickType.Trade), - new TestCaseData(Symbol.Create("ETHUSDT", SecurityType.Crypto, Market.BinanceUS), Resolution.Minute, Time.OneHour, TickType.Quote), + // invalid period, no error, empty result + new TestCaseData(Symbol.Create("ETHUSDT", SecurityType.Crypto, Market.BinanceUS), Resolution.Daily, TimeSpan.FromDays(-15), TickType.Trade), }; } } @@ -80,14 +72,18 @@ private static TestCaseData[] InvalidHistory { return new[] { - // invalid period, no error, empty result - new TestCaseData(Symbols.EURUSD, Resolution.Daily, TimeSpan.FromDays(-15), true), - - // invalid symbol, throws "System.ArgumentException : Unknown symbol: XYZ" - new TestCaseData(Symbol.Create("XYZ", SecurityType.Crypto, Market.BinanceUS), Resolution.Daily, TimeSpan.FromDays(15), true), - - // invalid security type, throws "System.ArgumentException : Invalid security type: Equity" - new TestCaseData(Symbols.AAPL, Resolution.Daily, TimeSpan.FromDays(15), true), + // invalid market + new TestCaseData(Symbol.Create("ETHUSDT", SecurityType.Crypto, Market.Binance), Resolution.Minute, TimeSpan.FromSeconds(15), TickType.Trade, true), + // invalid resolution + new TestCaseData(Symbol.Create("ETHUSDT", SecurityType.Crypto, Market.BinanceUS), Resolution.Tick, TimeSpan.FromSeconds(15), TickType.Trade, true), + new TestCaseData(Symbol.Create("ETHUSDT", SecurityType.Crypto, Market.BinanceUS), Resolution.Second, Time.OneMinute, TickType.Trade, true), + // invalid tick type + new TestCaseData(Symbol.Create("ETHUSDT", SecurityType.Crypto, Market.BinanceUS), Resolution.Minute, Time.OneHour, TickType.Quote, true), + new TestCaseData(Symbol.Create("ETHUSDT", SecurityType.Crypto, Market.BinanceUS), Resolution.Minute, Time.OneHour, TickType.OpenInterest, true), + // invalid symbol + new TestCaseData(Symbol.Create("XYZ", SecurityType.Crypto, Market.BinanceUS), Resolution.Daily, TimeSpan.FromDays(15), TickType.Trade, true), + // invalid security type + new TestCaseData(Symbols.AAPL, Resolution.Daily, TimeSpan.FromDays(15), TickType.Trade, true), }; } } diff --git a/QuantConnect.BinanceBrokerage.ToolBox/BaseDataDownloader.cs b/QuantConnect.BinanceBrokerage.ToolBox/BaseDataDownloader.cs index d776ed3..af857cd 100644 --- a/QuantConnect.BinanceBrokerage.ToolBox/BaseDataDownloader.cs +++ b/QuantConnect.BinanceBrokerage.ToolBox/BaseDataDownloader.cs @@ -81,7 +81,7 @@ public IEnumerable Get(DataDownloaderGetParameters dataDownloaderGetPa DataNormalizationMode.Raw, TickType.Trade); - return Brokerage.GetHistory(historyRequest); + return Brokerage.GetHistory(historyRequest) ?? Enumerable.Empty(); } /// diff --git a/QuantConnect.BinanceBrokerage/BinanceBrokerage.cs b/QuantConnect.BinanceBrokerage/BinanceBrokerage.cs index 84894ce..5e723f9 100644 --- a/QuantConnect.BinanceBrokerage/BinanceBrokerage.cs +++ b/QuantConnect.BinanceBrokerage/BinanceBrokerage.cs @@ -64,6 +64,10 @@ public partial class BinanceBrokerage : BaseWebsocketsBrokerage, IDataQueueHandl private BrokerageConcurrentMessageHandler _messageHandler; + private bool _unsupportedAssetHistoryLogged; + private bool _unsupportedResolutionHistoryLogged; + private bool _unsupportedTickTypeHistoryLogged; + private const int MaximumSymbolsPerConnection = 512; protected BinanceBaseRestApiClient ApiClient => _apiClientLazy?.Value; @@ -310,27 +314,47 @@ public override bool CancelOrder(Order order) /// An enumerable of bars covering the span specified in the request public override IEnumerable GetHistory(Data.HistoryRequest request) { - if (request.Resolution == Resolution.Tick || request.Resolution == Resolution.Second) + if (!CanSubscribe(request.Symbol)) { - OnMessage(new BrokerageMessageEvent(BrokerageMessageType.Warning, "InvalidResolution", - $"{request.Resolution} resolution is not supported, no history returned")); - yield break; + if (!_unsupportedAssetHistoryLogged) + { + _unsupportedAssetHistoryLogged = true; + OnMessage(new BrokerageMessageEvent(BrokerageMessageType.Warning, "InvalidSymbol", + $"{request.Symbol} is not supported, no history returned")); + } + + return null; } - if (request.TickType != TickType.Trade) + if (request.Resolution == Resolution.Tick || request.Resolution == Resolution.Second) { - OnMessage(new BrokerageMessageEvent(BrokerageMessageType.Warning, "InvalidTickType", - $"{request.TickType} tick type not supported, no history returned")); - yield break; + if (!_unsupportedResolutionHistoryLogged) + { + _unsupportedResolutionHistoryLogged = true; + OnMessage(new BrokerageMessageEvent(BrokerageMessageType.Warning, "InvalidResolution", + $"{request.Resolution} resolution is not supported, no history returned")); + } + + return null; } - if (request.Symbol.SecurityType != GetSupportedSecurityType()) + if (request.TickType != TickType.Trade) { - OnMessage(new BrokerageMessageEvent(BrokerageMessageType.Warning, "InvalidSecurityType", - $"{request.Symbol.SecurityType} security type not supported, no history returned")); - yield break; + if (!_unsupportedTickTypeHistoryLogged) + { + _unsupportedTickTypeHistoryLogged = true; + OnMessage(new BrokerageMessageEvent(BrokerageMessageType.Warning, "InvalidTickType", + $"{request.TickType} tick type not supported, no history returned")); + } + + return null; } + return GetHistoryImpl(request); + } + + private IEnumerable GetHistoryImpl(Data.HistoryRequest request) + { var period = request.Resolution.ToTimeSpan(); var restApiClient = _apiClientLazy?.IsValueCreated == true ? ApiClient @@ -436,7 +460,8 @@ protected virtual bool CanSubscribe(Symbol symbol) { return !symbol.Value.Contains("UNIVERSE") && symbol.SecurityType == GetSupportedSecurityType() && - symbol.ID.Market == MarketName; + symbol.ID.Market == MarketName && + _symbolMapper.IsKnownLeanSymbol(symbol); } /// From c4b6b5abe8a02c56b9f7733498faf972f2d8f734 Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Fri, 23 Feb 2024 09:34:39 -0400 Subject: [PATCH 2/3] Return null on unsupported data download requests --- QuantConnect.BinanceBrokerage.ToolBox/BaseDataDownloader.cs | 2 +- QuantConnect.BinanceBrokerage.ToolBox/Program.cs | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/QuantConnect.BinanceBrokerage.ToolBox/BaseDataDownloader.cs b/QuantConnect.BinanceBrokerage.ToolBox/BaseDataDownloader.cs index af857cd..d776ed3 100644 --- a/QuantConnect.BinanceBrokerage.ToolBox/BaseDataDownloader.cs +++ b/QuantConnect.BinanceBrokerage.ToolBox/BaseDataDownloader.cs @@ -81,7 +81,7 @@ public IEnumerable Get(DataDownloaderGetParameters dataDownloaderGetPa DataNormalizationMode.Raw, TickType.Trade); - return Brokerage.GetHistory(historyRequest) ?? Enumerable.Empty(); + return Brokerage.GetHistory(historyRequest); } /// diff --git a/QuantConnect.BinanceBrokerage.ToolBox/Program.cs b/QuantConnect.BinanceBrokerage.ToolBox/Program.cs index 035ee96..ef7e576 100644 --- a/QuantConnect.BinanceBrokerage.ToolBox/Program.cs +++ b/QuantConnect.BinanceBrokerage.ToolBox/Program.cs @@ -112,6 +112,11 @@ public static void DownloadData(BaseDataDownloader downloader, IList tic // Download the data var symbol = downloader.GetSymbol(ticker); var data = downloader.Get(new DataDownloaderGetParameters(symbol, castResolution, fromDate, toDate)); + if (data == null) + { + continue; + } + var bars = data.Cast().ToList(); // Save the data (single resolution) From fa79be126822ed684744c89d664d142fc24019a4 Mon Sep 17 00:00:00 2001 From: Jhonathan Abreu Date: Fri, 23 Feb 2024 15:29:33 -0400 Subject: [PATCH 3/3] Minor improvements --- .../BinanceBrokerageHistoryProviderTests.cs | 45 +------------------ ...oinFuturesBrokerageHistoryProviderTests.cs | 21 +-------- ...nceFuturesBrokerageHistoryProviderTests.cs | 21 +-------- .../BinanceUSBrokerageHistoryProviderTests.cs | 21 +-------- .../BaseDataDownloader.cs | 21 --------- .../BinanceBrokerage.cs | 12 +++++ 6 files changed, 20 insertions(+), 121 deletions(-) diff --git a/QuantConnect.BinanceBrokerage.Tests/BinanceBrokerageHistoryProviderTests.cs b/QuantConnect.BinanceBrokerage.Tests/BinanceBrokerageHistoryProviderTests.cs index c8a933b..f7a4f92 100644 --- a/QuantConnect.BinanceBrokerage.Tests/BinanceBrokerageHistoryProviderTests.cs +++ b/QuantConnect.BinanceBrokerage.Tests/BinanceBrokerageHistoryProviderTests.cs @@ -55,37 +55,6 @@ public virtual void GetsHistory(Symbol symbol, Resolution resolution, TimeSpan p BaseHistoryTest(symbol, resolution, period, tickType, unsupported, _brokerage); } - [Test] - [TestCaseSource(nameof(NoHistory))] - public virtual void GetEmptyHistory(Symbol symbol, Resolution resolution, TimeSpan period, TickType tickType) - { - BaseEmptyHistoryTest(symbol, resolution, period, tickType, _brokerage); - } - - public static void BaseEmptyHistoryTest(Symbol symbol, Resolution resolution, TimeSpan period, TickType tickType, Brokerage brokerage) - { - var now = DateTime.UtcNow; - var request = new HistoryRequest(now.Add(-period), - now, - typeof(TradeBar), - symbol, - resolution, - SecurityExchangeHours.AlwaysOpen(TimeZones.Utc), - DateTimeZone.Utc, - Resolution.Minute, - false, - false, - DataNormalizationMode.Adjusted, - tickType); - - var history = brokerage.GetHistory(request)?.ToList(); - - Assert.IsNotNull(history); - - Log.Debug("Data points retrieved: " + history.Count); - Assert.AreEqual(0, history.Count); - } - public static void BaseHistoryTest(Symbol symbol, Resolution resolution, TimeSpan period, TickType tickType, bool unsupported, Brokerage brokerage) { var now = DateTime.UtcNow; @@ -135,24 +104,14 @@ private static TestCaseData[] ValidHistory } } - private static TestCaseData[] NoHistory - { - get - { - return new[] - { - // invalid period - new TestCaseData(Symbol.Create("ETHUSDT", SecurityType.Crypto, Market.Binance), Resolution.Daily, TimeSpan.FromDays(-15), TickType.Trade), - }; - } - } - private static TestCaseData[] InvalidHistory { get { return new[] { + // invalid period + new TestCaseData(Symbol.Create("ETHUSDT", SecurityType.Crypto, Market.Binance), Resolution.Daily, TimeSpan.FromDays(-15), TickType.Trade, true), // invalid symbol new TestCaseData(Symbol.Create("XYZ", SecurityType.Crypto, Market.Binance), Resolution.Daily, TimeSpan.FromDays(15), TickType.Trade, true), // invalid security type diff --git a/QuantConnect.BinanceBrokerage.Tests/BinanceCoinFuturesBrokerageHistoryProviderTests.cs b/QuantConnect.BinanceBrokerage.Tests/BinanceCoinFuturesBrokerageHistoryProviderTests.cs index d0a09d6..82db279 100644 --- a/QuantConnect.BinanceBrokerage.Tests/BinanceCoinFuturesBrokerageHistoryProviderTests.cs +++ b/QuantConnect.BinanceBrokerage.Tests/BinanceCoinFuturesBrokerageHistoryProviderTests.cs @@ -48,13 +48,6 @@ public virtual void GetsHistory(Symbol symbol, Resolution resolution, TimeSpan p BinanceBrokerageHistoryProviderTests.BaseHistoryTest(symbol, resolution, period, tickType, unsupported, _brokerage); } - [Test] - [TestCaseSource(nameof(NoHistory))] - public virtual void GetEmptyHistory(Symbol symbol, Resolution resolution, TimeSpan period, TickType tickType) - { - BinanceBrokerageHistoryProviderTests.BaseEmptyHistoryTest(symbol, resolution, period, tickType, _brokerage); - } - private static TestCaseData[] ValidHistory { get @@ -69,24 +62,14 @@ private static TestCaseData[] ValidHistory } } - private static TestCaseData[] NoHistory - { - get - { - return new[] - { - // invalid period, no error, empty result - new TestCaseData(Symbol.Create("ETHUSD", SecurityType.CryptoFuture, Market.Binance), Resolution.Minute, TimeSpan.FromDays(-15), TickType.Trade), - }; - } - } - private static TestCaseData[] InvalidHistory { get { return new[] { + // invalid period + new TestCaseData(Symbol.Create("ETHUSD", SecurityType.CryptoFuture, Market.Binance), Resolution.Minute, TimeSpan.FromDays(-15), TickType.Trade, true), // Invalid type of future new TestCaseData(Symbol.Create("ETHUSDT", SecurityType.CryptoFuture, Market.Binance), Resolution.Minute, TimeSpan.FromSeconds(15), TickType.Trade, true), // invalid resolution diff --git a/QuantConnect.BinanceBrokerage.Tests/BinanceFuturesBrokerageHistoryProviderTests.cs b/QuantConnect.BinanceBrokerage.Tests/BinanceFuturesBrokerageHistoryProviderTests.cs index 402cefa..3d99e36 100644 --- a/QuantConnect.BinanceBrokerage.Tests/BinanceFuturesBrokerageHistoryProviderTests.cs +++ b/QuantConnect.BinanceBrokerage.Tests/BinanceFuturesBrokerageHistoryProviderTests.cs @@ -48,13 +48,6 @@ public virtual void GetsHistory(Symbol symbol, Resolution resolution, TimeSpan p BinanceBrokerageHistoryProviderTests.BaseHistoryTest(symbol, resolution, period, tickType, unsupported, _brokerage); } - [Test] - [TestCaseSource(nameof(NoHistory))] - public virtual void GetEmptyHistory(Symbol symbol, Resolution resolution, TimeSpan period, TickType tickType) - { - BinanceBrokerageHistoryProviderTests.BaseEmptyHistoryTest(symbol, resolution, period, tickType, _brokerage); - } - private static TestCaseData[] ValidHistory { get @@ -69,24 +62,14 @@ private static TestCaseData[] ValidHistory } } - private static TestCaseData[] NoHistory - { - get - { - return new[] - { - // invalid period, no error, empty result - new TestCaseData(Symbol.Create("ETHUSDT", SecurityType.CryptoFuture, Market.Binance), Resolution.Daily, TimeSpan.FromDays(-15), TickType.Trade), - }; - } - } - private static TestCaseData[] InvalidHistory { get { return new[] { + // invalid period + new TestCaseData(Symbol.Create("ETHUSDT", SecurityType.CryptoFuture, Market.Binance), Resolution.Daily, TimeSpan.FromDays(-15), TickType.Trade, true), // invalid resolution new TestCaseData(Symbol.Create("ETHUSDT", SecurityType.CryptoFuture, Market.Binance), Resolution.Tick, TimeSpan.FromSeconds(15), TickType.Trade, true), new TestCaseData(Symbol.Create("ETHUSDT", SecurityType.CryptoFuture, Market.Binance), Resolution.Second, Time.OneMinute, TickType.Trade, true), diff --git a/QuantConnect.BinanceBrokerage.Tests/BinanceUSBrokerageHistoryProviderTests.cs b/QuantConnect.BinanceBrokerage.Tests/BinanceUSBrokerageHistoryProviderTests.cs index bcee401..27aec9e 100644 --- a/QuantConnect.BinanceBrokerage.Tests/BinanceUSBrokerageHistoryProviderTests.cs +++ b/QuantConnect.BinanceBrokerage.Tests/BinanceUSBrokerageHistoryProviderTests.cs @@ -33,13 +33,6 @@ public override void GetsHistory(Symbol symbol, Resolution resolution, TimeSpan base.GetsHistory(symbol, resolution, period, tickType, unsupported); } - [Test] - [TestCaseSource(nameof(NoHistory))] - public override void GetEmptyHistory(Symbol symbol, Resolution resolution, TimeSpan period, TickType tickType) - { - base.GetEmptyHistory(symbol, resolution, period, tickType); - } - private static TestCaseData[] ValidHistory { get @@ -54,24 +47,14 @@ private static TestCaseData[] ValidHistory } } - private static TestCaseData[] NoHistory - { - get - { - return new[] - { - // invalid period, no error, empty result - new TestCaseData(Symbol.Create("ETHUSDT", SecurityType.Crypto, Market.BinanceUS), Resolution.Daily, TimeSpan.FromDays(-15), TickType.Trade), - }; - } - } - private static TestCaseData[] InvalidHistory { get { return new[] { + // invalid period + new TestCaseData(Symbol.Create("ETHUSDT", SecurityType.Crypto, Market.BinanceUS), Resolution.Daily, TimeSpan.FromDays(-15), TickType.Trade, true), // invalid market new TestCaseData(Symbol.Create("ETHUSDT", SecurityType.Crypto, Market.Binance), Resolution.Minute, TimeSpan.FromSeconds(15), TickType.Trade, true), // invalid resolution diff --git a/QuantConnect.BinanceBrokerage.ToolBox/BaseDataDownloader.cs b/QuantConnect.BinanceBrokerage.ToolBox/BaseDataDownloader.cs index d776ed3..eaa9d98 100644 --- a/QuantConnect.BinanceBrokerage.ToolBox/BaseDataDownloader.cs +++ b/QuantConnect.BinanceBrokerage.ToolBox/BaseDataDownloader.cs @@ -45,27 +45,6 @@ public IEnumerable Get(DataDownloaderGetParameters dataDownloaderGetPa var resolution = dataDownloaderGetParameters.Resolution; var startUtc = dataDownloaderGetParameters.StartUtc; var endUtc = dataDownloaderGetParameters.EndUtc; - var tickType = dataDownloaderGetParameters.TickType; - - if (tickType != TickType.Trade) - { - return Enumerable.Empty(); - } - - if (resolution == Resolution.Tick || resolution == Resolution.Second) - { - throw new ArgumentException($"Resolution not available: {resolution}"); - } - - if (!SymbolMapper.IsKnownLeanSymbol(symbol)) - { - throw new ArgumentException($"The ticker {symbol.Value} is not available."); - } - - if (endUtc < startUtc) - { - throw new ArgumentException("The end date must be greater or equal than the start date."); - } var historyRequest = new HistoryRequest( startUtc, diff --git a/QuantConnect.BinanceBrokerage/BinanceBrokerage.cs b/QuantConnect.BinanceBrokerage/BinanceBrokerage.cs index 5e723f9..844b909 100644 --- a/QuantConnect.BinanceBrokerage/BinanceBrokerage.cs +++ b/QuantConnect.BinanceBrokerage/BinanceBrokerage.cs @@ -67,6 +67,7 @@ public partial class BinanceBrokerage : BaseWebsocketsBrokerage, IDataQueueHandl private bool _unsupportedAssetHistoryLogged; private bool _unsupportedResolutionHistoryLogged; private bool _unsupportedTickTypeHistoryLogged; + private bool _invalidTimeRangeHistoryLogged; private const int MaximumSymbolsPerConnection = 512; @@ -350,6 +351,17 @@ public override IEnumerable GetHistory(Data.HistoryRequest request) return null; } + if (request.StartTimeUtc >= request.EndTimeUtc) + { + if (!_invalidTimeRangeHistoryLogged) + { + _invalidTimeRangeHistoryLogged = true; + OnMessage(new BrokerageMessageEvent(BrokerageMessageType.Warning, "InvalidDateRange", + "The history request start date must precede the end date, no history returned")); + } + return null; + } + return GetHistoryImpl(request); }