From 8e0d9676f6da188c84280c2853e669d678005252 Mon Sep 17 00:00:00 2001 From: Martin Molinero Date: Fri, 5 Apr 2024 11:02:12 -0300 Subject: [PATCH] Reorganize project structure --- .github/workflows/gh-actions.yml | 2 +- DataProcessing/DataProcessing.csproj | 2 +- DataProcessing/FactSetDataProcessor.cs | 3 +- DropboxDownloader.py | 119 ------------------ QuantConnect.DataSource.FactSet.sln | 4 +- QuantConnect.DataSource.csproj | 29 ----- .../FactSetDataDownloaderTests.cs | 0 .../FactSetDataProviderHistoryTests.cs | 0 .../FactSetOptionChainProviderTests.cs | 0 .../FactSetSymbolMapperTests.cs | 0 .../QuantConnect.FactSet.Tests.csproj | 2 +- .../TestSetup.cs | 0 .../config.json | 0 .../FactSetApi.cs | 0 .../FactSetDataDownloader.cs | 0 .../FactSetDataProcessingDataDownloader.cs | 11 +- .../FactSetDataProcessingDataProvider.cs | 8 +- .../FactSetDataProvider.cs | 0 .../FactSetOptionChainProvider.cs | 0 .../FactSetSymbolMapper.cs | 0 .../QuantConnect.DataSource.FactSet.csproj | 42 +++++++ 21 files changed, 58 insertions(+), 164 deletions(-) delete mode 100644 DropboxDownloader.py delete mode 100644 QuantConnect.DataSource.csproj rename {tests => QuantConnect.FactSet.Tests}/FactSetDataDownloaderTests.cs (100%) rename {tests => QuantConnect.FactSet.Tests}/FactSetDataProviderHistoryTests.cs (100%) rename {tests => QuantConnect.FactSet.Tests}/FactSetOptionChainProviderTests.cs (100%) rename {tests => QuantConnect.FactSet.Tests}/FactSetSymbolMapperTests.cs (100%) rename tests/Tests.csproj => QuantConnect.FactSet.Tests/QuantConnect.FactSet.Tests.csproj (91%) rename {tests => QuantConnect.FactSet.Tests}/TestSetup.cs (100%) rename {tests => QuantConnect.FactSet.Tests}/config.json (100%) rename FactSetApi.cs => QuantConnect.FactSet/FactSetApi.cs (100%) rename FactSetDataDownloader.cs => QuantConnect.FactSet/FactSetDataDownloader.cs (100%) rename {DataProcessing => QuantConnect.FactSet}/FactSetDataProcessingDataDownloader.cs (86%) rename {DataProcessing => QuantConnect.FactSet}/FactSetDataProcessingDataProvider.cs (83%) rename FactSetDataProvider.cs => QuantConnect.FactSet/FactSetDataProvider.cs (100%) rename FactSetOptionChainProvider.cs => QuantConnect.FactSet/FactSetOptionChainProvider.cs (100%) rename FactSetSymbolMapper.cs => QuantConnect.FactSet/FactSetSymbolMapper.cs (100%) create mode 100644 QuantConnect.FactSet/QuantConnect.DataSource.FactSet.csproj diff --git a/.github/workflows/gh-actions.yml b/.github/workflows/gh-actions.yml index 4c857ad..a3f6dd3 100644 --- a/.github/workflows/gh-actions.yml +++ b/.github/workflows/gh-actions.yml @@ -44,4 +44,4 @@ jobs: # Build dotnet build /p:Configuration=Release /v:quiet /p:WarningLevel=1 QuantConnect.DataSource.FactSet.sln # Run Tests - dotnet test ./tests/bin/Release/net6.0/QuantConnect.DataLibrary.Tests.dll \ No newline at end of file + dotnet test ./QuantConnect.FactSet.Tests/bin/Release/net6.0/QuantConnect.DataLibrary.Tests.dll \ No newline at end of file diff --git a/DataProcessing/DataProcessing.csproj b/DataProcessing/DataProcessing.csproj index 40c9338..ec9b4ea 100644 --- a/DataProcessing/DataProcessing.csproj +++ b/DataProcessing/DataProcessing.csproj @@ -6,7 +6,7 @@ true - + diff --git a/DataProcessing/FactSetDataProcessor.cs b/DataProcessing/FactSetDataProcessor.cs index dc06256..e59f2b2 100644 --- a/DataProcessing/FactSetDataProcessor.cs +++ b/DataProcessing/FactSetDataProcessor.cs @@ -16,11 +16,10 @@ using System; using System.Collections.Generic; using System.Diagnostics; -using System.IO; using System.Linq; -using System.Threading; using System.Threading.Tasks; using QuantConnect.Data; +using QuantConnect.Lean.DataSource.FactSet; using QuantConnect.Logging; using QuantConnect.Util; diff --git a/DropboxDownloader.py b/DropboxDownloader.py deleted file mode 100644 index 54b7e55..0000000 --- a/DropboxDownloader.py +++ /dev/null @@ -1,119 +0,0 @@ -# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. -# Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This script is used to download files from a given dropbox directory. -# Files to be downloaded are filtered based on given date present in file name. - -# ARGUMENTS -# DROPBOX_API_KEY: Dropbox API KEY with read access. -# DROPBOX_SOURCE_DIRECTORY: path of the dropbox directory to search files within. -# DROPBOX_OUTPUT_DIRECTORY(optional): base path of the output directory to store to downloaded files. -# cmdline args expected in order: DROPBOX_API_KEY, DROPBOX_SOURCE_DIRECTORY, QC_DATAFLEET_DEPLOYMENT_DATE, DROPBOX_OUTPUT_DIRECTORY - -import requests -import json -import sys -import time -import os -from pathlib import Path - -DROPBOX_API_KEY = os.environ.get("DROPBOX_API_KEY") -DROPBOX_SOURCE_DIRECTORY = os.environ.get("DROPBOX_SOURCE_DIRECTORY") -QC_DATAFLEET_DEPLOYMENT_DATE = os.environ.get("QC_DATAFLEET_DEPLOYMENT_DATE") -DROPBOX_OUTPUT_DIRECTORY = os.environ.get("DROPBOX_OUTPUT_DIRECTORY", "/raw") - -def DownloadZipFile(filePath): - - print(f"Starting downloading file at: {filePath}") - - # defining the api-endpoint - API_ENDPOINT_DOWNLOAD = "https://content.dropboxapi.com/2/files/download" - - # data to be sent to api - data = {"path": filePath} - - headers = {"Authorization": f"Bearer {DROPBOX_API_KEY}", - "Dropbox-API-Arg": json.dumps(data)} - - # sending post request and saving response as response object - response = requests.post(url = API_ENDPOINT_DOWNLOAD, headers=headers) - - response.raise_for_status() # ensure we notice bad responses - - fileName = filePath.split("/")[-1] - outputPath = os.path.join(DROPBOX_OUTPUT_DIRECTORY, fileName) - - with open(outputPath, "wb") as f: - f.write(response.content) - print(f"Succesfully saved file at: {outputPath}") - -def GetFilePathsFromDate(targetLocation, dateString): - # defining the api-endpoint - API_ENDPOINT_FILEPATH = "https://api.dropboxapi.com/2/files/list_folder" - - headers = {"Content-Type": "application/json", - "Authorization": f"Bearer {DROPBOX_API_KEY}"} - - # data to be sent to api - data = {"path": targetLocation, - "recursive": False, - "include_media_info": False, - "include_deleted": False, - "include_has_explicit_shared_members": False, - "include_mounted_folders": True, - "include_non_downloadable_files": True} - - # sending post request and saving response as response object - response = requests.post(url = API_ENDPOINT_FILEPATH, headers=headers, data = json.dumps(data)) - - response.raise_for_status() # ensure we notice bad responses - - target_paths = [entry["path_display"] for entry in response.json()["entries"] if dateString in entry["path_display"]] - return target_paths - -def main(): - global DROPBOX_API_KEY, DROPBOX_SOURCE_DIRECTORY, QC_DATAFLEET_DEPLOYMENT_DATE, DROPBOX_OUTPUT_DIRECTORY - inputCount = len(sys.argv) - if inputCount > 1: - DROPBOX_API_KEY = sys.argv[1] - if inputCount > 2: - DROPBOX_SOURCE_DIRECTORY = sys.argv[2] - if inputCount > 3: - QC_DATAFLEET_DEPLOYMENT_DATE = sys.argv[3] - if inputCount > 4: - DROPBOX_OUTPUT_DIRECTORY = sys.argv[4] - - # make output path if doesn't exists - Path(DROPBOX_OUTPUT_DIRECTORY).mkdir(parents=True, exist_ok=True) - - target_paths = GetFilePathsFromDate(DROPBOX_SOURCE_DIRECTORY, QC_DATAFLEET_DEPLOYMENT_DATE) - print(f"Found {len(target_paths)} files with following paths {target_paths}") - - #download files - for path in target_paths: - count = 0 - maxTries = 3 - while True: - try: - DownloadZipFile(path) - break - except Exception as e: - count +=1 - if count > maxTries: - print(f"Error for file with path {path} --error message: {e}") - break - print(f"Error, sleep for 5 sec and retry download file with --path: {path}") - time.sleep(5) - -if __name__== "__main__": - main() diff --git a/QuantConnect.DataSource.FactSet.sln b/QuantConnect.DataSource.FactSet.sln index 327c7aa..ee0e46e 100644 --- a/QuantConnect.DataSource.FactSet.sln +++ b/QuantConnect.DataSource.FactSet.sln @@ -3,11 +3,11 @@ Microsoft Visual Studio Solution File, Format Version 12.00 # Visual Studio Version 17 VisualStudioVersion = 17.0.31903.59 MinimumVisualStudioVersion = 10.0.40219.1 -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "QuantConnect.DataSource", "QuantConnect.DataSource.csproj", "{FBA4214D-0148-41A9-9C9E-42F17305578F}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "QuantConnect.FactSet", "QuantConnect.FactSet\QuantConnect.DataSource.FactSet.csproj", "{FBA4214D-0148-41A9-9C9E-42F17305578F}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DataProcessing", "DataProcessing\DataProcessing.csproj", "{0CE7E210-7ECF-426E-9751-5FB515EC91DA}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Tests", "tests\Tests.csproj", "{078083C8-B501-44C5-A708-DA93BAA31A0A}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "QuantConnect.FactSet.Tests", "QuantConnect.FactSet.Tests\QuantConnect.FactSet.Tests.csproj", "{078083C8-B501-44C5-A708-DA93BAA31A0A}" EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "QuantConnect.Tests", "..\Lean\Tests\QuantConnect.Tests.csproj", "{D6AA5B05-7897-4B6E-AE64-45A8CBF11D8C}" EndProject diff --git a/QuantConnect.DataSource.csproj b/QuantConnect.DataSource.csproj deleted file mode 100644 index e500874..0000000 --- a/QuantConnect.DataSource.csproj +++ /dev/null @@ -1,29 +0,0 @@ - - - net6.0 - QuantConnect.DataSource - QuantConnect.DataSource.FactSet - bin\$(Configuration) - $(OutputPath)\QuantConnect.DataSource.FactSet.xml - enable - - - - - - - - - - - - - - - - - - - - - diff --git a/tests/FactSetDataDownloaderTests.cs b/QuantConnect.FactSet.Tests/FactSetDataDownloaderTests.cs similarity index 100% rename from tests/FactSetDataDownloaderTests.cs rename to QuantConnect.FactSet.Tests/FactSetDataDownloaderTests.cs diff --git a/tests/FactSetDataProviderHistoryTests.cs b/QuantConnect.FactSet.Tests/FactSetDataProviderHistoryTests.cs similarity index 100% rename from tests/FactSetDataProviderHistoryTests.cs rename to QuantConnect.FactSet.Tests/FactSetDataProviderHistoryTests.cs diff --git a/tests/FactSetOptionChainProviderTests.cs b/QuantConnect.FactSet.Tests/FactSetOptionChainProviderTests.cs similarity index 100% rename from tests/FactSetOptionChainProviderTests.cs rename to QuantConnect.FactSet.Tests/FactSetOptionChainProviderTests.cs diff --git a/tests/FactSetSymbolMapperTests.cs b/QuantConnect.FactSet.Tests/FactSetSymbolMapperTests.cs similarity index 100% rename from tests/FactSetSymbolMapperTests.cs rename to QuantConnect.FactSet.Tests/FactSetSymbolMapperTests.cs diff --git a/tests/Tests.csproj b/QuantConnect.FactSet.Tests/QuantConnect.FactSet.Tests.csproj similarity index 91% rename from tests/Tests.csproj rename to QuantConnect.FactSet.Tests/QuantConnect.FactSet.Tests.csproj index 54f604b..f89924b 100644 --- a/tests/Tests.csproj +++ b/QuantConnect.FactSet.Tests/QuantConnect.FactSet.Tests.csproj @@ -15,7 +15,7 @@ - + diff --git a/tests/TestSetup.cs b/QuantConnect.FactSet.Tests/TestSetup.cs similarity index 100% rename from tests/TestSetup.cs rename to QuantConnect.FactSet.Tests/TestSetup.cs diff --git a/tests/config.json b/QuantConnect.FactSet.Tests/config.json similarity index 100% rename from tests/config.json rename to QuantConnect.FactSet.Tests/config.json diff --git a/FactSetApi.cs b/QuantConnect.FactSet/FactSetApi.cs similarity index 100% rename from FactSetApi.cs rename to QuantConnect.FactSet/FactSetApi.cs diff --git a/FactSetDataDownloader.cs b/QuantConnect.FactSet/FactSetDataDownloader.cs similarity index 100% rename from FactSetDataDownloader.cs rename to QuantConnect.FactSet/FactSetDataDownloader.cs diff --git a/DataProcessing/FactSetDataProcessingDataDownloader.cs b/QuantConnect.FactSet/FactSetDataProcessingDataDownloader.cs similarity index 86% rename from DataProcessing/FactSetDataProcessingDataDownloader.cs rename to QuantConnect.FactSet/FactSetDataProcessingDataDownloader.cs index 3d20bb2..685ab87 100644 --- a/DataProcessing/FactSetDataProcessingDataDownloader.cs +++ b/QuantConnect.FactSet/FactSetDataProcessingDataDownloader.cs @@ -13,11 +13,12 @@ * limitations under the License. */ -using QuantConnect.Lean.DataSource.FactSet; -using System.Collections.Generic; using System; +using System.Collections.Generic; + +using FactSetSDK = FactSet.SDK; -namespace QuantConnect.DataProcessing +namespace QuantConnect.Lean.DataSource.FactSet { /// /// The only purpose of this class is to provide a way to access the class and use its constructor @@ -27,12 +28,12 @@ namespace QuantConnect.DataProcessing /// without exposing its additional capabilities of storing the raw data downloaded from FactSet, which is only useful /// for the Data Processing program. /// - internal class FactSetDataProcessingDataDownloader : FactSetDataDownloader + public class FactSetDataProcessingDataDownloader : FactSetDataDownloader { /// /// Initializes a new instance of the /// - public FactSetDataProcessingDataDownloader(FactSet.SDK.Utils.Authentication.Configuration factSetAuthConfiguration, string rawDataFolder) + public FactSetDataProcessingDataDownloader(FactSetSDK.Utils.Authentication.Configuration factSetAuthConfiguration, string rawDataFolder) : base(new FactSetDataProcessingDataProvider(factSetAuthConfiguration, rawDataFolder)) { } diff --git a/DataProcessing/FactSetDataProcessingDataProvider.cs b/QuantConnect.FactSet/FactSetDataProcessingDataProvider.cs similarity index 83% rename from DataProcessing/FactSetDataProcessingDataProvider.cs rename to QuantConnect.FactSet/FactSetDataProcessingDataProvider.cs index 3bf1c4c..af24047 100644 --- a/DataProcessing/FactSetDataProcessingDataProvider.cs +++ b/QuantConnect.FactSet/FactSetDataProcessingDataProvider.cs @@ -13,9 +13,9 @@ * limitations under the License. */ -using QuantConnect.Lean.DataSource.FactSet; +using FactSetSDK = FactSet.SDK; -namespace QuantConnect.DataProcessing +namespace QuantConnect.Lean.DataSource.FactSet { /// /// The only purpose of this class is to provide a way to access the class and use its constructor @@ -25,12 +25,12 @@ namespace QuantConnect.DataProcessing /// without exposing its additional capabilities of storing the raw data downloaded from FactSet, which is only useful /// for the Data Processing program. /// - internal class FactSetDataProcessingDataProvider : FactSetDataProvider + public class FactSetDataProcessingDataProvider : FactSetDataProvider { /// /// Initializes a new instance of the /// - public FactSetDataProcessingDataProvider(FactSet.SDK.Utils.Authentication.Configuration factSetAuthConfig, string rawDataFolder) + public FactSetDataProcessingDataProvider(FactSetSDK.Utils.Authentication.Configuration factSetAuthConfig, string rawDataFolder) : base(factSetAuthConfig, rawDataFolder) { } diff --git a/FactSetDataProvider.cs b/QuantConnect.FactSet/FactSetDataProvider.cs similarity index 100% rename from FactSetDataProvider.cs rename to QuantConnect.FactSet/FactSetDataProvider.cs diff --git a/FactSetOptionChainProvider.cs b/QuantConnect.FactSet/FactSetOptionChainProvider.cs similarity index 100% rename from FactSetOptionChainProvider.cs rename to QuantConnect.FactSet/FactSetOptionChainProvider.cs diff --git a/FactSetSymbolMapper.cs b/QuantConnect.FactSet/FactSetSymbolMapper.cs similarity index 100% rename from FactSetSymbolMapper.cs rename to QuantConnect.FactSet/FactSetSymbolMapper.cs diff --git a/QuantConnect.FactSet/QuantConnect.DataSource.FactSet.csproj b/QuantConnect.FactSet/QuantConnect.DataSource.FactSet.csproj new file mode 100644 index 0000000..81e2a7a --- /dev/null +++ b/QuantConnect.FactSet/QuantConnect.DataSource.FactSet.csproj @@ -0,0 +1,42 @@ + + + Release + AnyCPU + net6.0 + QuantConnect.Lean.DataSource.FactSet + QuantConnect.Lean.DataSource.FactSet + QuantConnect.Lean.DataSource.FactSet + QuantConnect.Lean.DataSource.FactSet + Library + bin\$(Configuration)\ + false + true + false + QuantConnect LEAN FactSet Data Source: FactSet Data Source plugin for Lean + + enable + enable + + + + full + bin\Debug\ + + + + pdbonly + bin\Release\ + + + + + + + + + + + + + + \ No newline at end of file