Skip to content

Log warning when downloader returns no data for requested date range #9344

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
JosueNina wants to merge 1 commit into
base: master
Choose a base branch
from
+42 −3
Open

Log warning when downloader returns no data for requested date range #9344

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 13 additions & 3 deletions DownloaderDataProvider/Program.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -118,7 +118,7 @@ public static void RunDownload(IDataDownloader dataDownloader, BaseDataDownloadC
if (downloadedData == null)
{
completeSymbolCount++;
Log.Trace($"DownloaderDataProvider.Main(): No data available for the following parameters: {downloadParameters}");
Log.Trace($"DownloaderDataProvider.RunDownload(): No data returned for {downloadParameters.Symbol}. The brokerage does not support this request (unsupported resolution, tick type, or asset type).");
continue;
}

Expand All @@ -136,9 +136,11 @@ public static void RunDownload(IDataDownloader dataDownloader, BaseDataDownloadC
downloadParameters.EndUtc);

var lastLogStatusTime = DateTime.UtcNow;
var dataWasWritten = false;

foreach (var data in groupedData)
{
dataWasWritten = true;
writer.Write(data.Select(data =>
{
var utcNow = DateTime.UtcNow;
Expand All @@ -155,8 +157,16 @@ public static void RunDownload(IDataDownloader dataDownloader, BaseDataDownloadC
var symbolPercentComplete = (double)completeSymbolCount / totalDownloadSymbols * 100;
Log.Trace($"DownloaderDataProvider.RunDownload(): {symbolPercentComplete:F2}% complete ({completeSymbolCount} out of {totalDownloadSymbols} symbols)");

Log.Trace($"DownloaderDataProvider.RunDownload(): Download completed for {downloadParameters.Symbol} at {downloadParameters.Resolution} resolution, " +
$"covering the period from {dataDownloadConfig.StartDate} to {dataDownloadConfig.EndDate}.");
if (!dataWasWritten)
{
Log.Trace($"DownloaderDataProvider.RunDownload(): No data found for {downloadParameters.Symbol} between " +
$"{dataDownloadConfig.StartDate:yyyy-MM-dd} and {dataDownloadConfig.EndDate:yyyy-MM-dd}.");
}
else
{
Log.Trace($"DownloaderDataProvider.RunDownload(): Download completed for {downloadParameters.Symbol} at {downloadParameters.Resolution} resolution, " +
$"covering the period from {dataDownloadConfig.StartDate} to {dataDownloadConfig.EndDate}.");
}
}
Log.Trace($"All downloads completed in {(DateTime.UtcNow - startDownloadUtcTime).TotalSeconds:F2} seconds.");
}
Expand Down
29 changes: 29 additions & 0 deletions Tests/DownloaderDataProvider/DownloadHelperTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
using NUnit.Framework;
using QuantConnect.Data;
using QuantConnect.Util;
using QuantConnect.Logging;
using QuantConnect.Interfaces;
using QuantConnect.Data.Market;
using System.Collections.Generic;
Expand Down Expand Up @@ -111,6 +112,34 @@ private static List<Symbol> GenerateOptionContracts(Symbol underlying, decimal s
return contracts;
}

[Test]
public void RunDownloadLogsWarningWhenDownloaderReturnsEmptyData()
{
var startDate = new DateTime(2015, 7, 20);
var endDate = new DateTime(2016, 1, 1);
var symbol = Symbols.SPY;

var downloadDataConfig = new DataDownloadConfig(TickType.Trade, SecurityType.Equity, Resolution.Daily, startDate, endDate, Market.USA, new List<Symbol> { symbol });

var logHandler = new QueueLogHandler();
var originalHandler = Log.LogHandler;
Log.LogHandler = logHandler;

try
{
var downloader = new DataDownloaderTest(Enumerable.Empty<BaseData>());
Program.RunDownload(downloader, downloadDataConfig, _dataDirectory, _cacheProvider);

var logs = logHandler.Logs.Select(x => x.Message).ToList();
Assert.IsTrue(logs.Any(msg => msg.Contains("No data found", StringComparison.OrdinalIgnoreCase)),
$"Expected a warning log when the downloader returns no data, but got: {string.Join(", ", logs)}");
}
finally
{
Log.LogHandler = originalHandler;
}
}

private class DataDownloaderTest : IDataDownloader
{
public IEnumerable<BaseData> Data { get; }
Expand Down
Loading