Add --data-provider option to the optimize command

README.md

@@ -1454,6 +1454,10 @@ Options:
   --parameter <TEXT FLOAT FLOAT FLOAT>...
                                   The 'parameter min max step' pairs configuring the parameters to optimize
   --constraint TEXT               The 'statistic operator value' pairs configuring the constraints of the optimization
+  --data-provider [IQFeed|Polygon|QuantConnect|Local|Terminal Link]
+                                  Update the Lean configuration file to retrieve data from the given provider
+  --download-data                 Update the Lean configuration file to download data from the QuantConnect API, alias
+                                  for --data-provider QuantConnect
   --release                       Compile C# projects in release configuration instead of debug
   --image TEXT                    The LEAN engine image to use (defaults to quantconnect/lean:latest)
   --update                        Pull the LEAN engine image before running the optimizer
@@ -1463,6 +1467,24 @@ Options:
   --extra-docker-config TEXT      Extra docker configuration as a JSON string. For more information https://docker-
                                   py.readthedocs.io/en/stable/containers.html
   --no-update                     Use the local LEAN engine image instead of pulling the latest version
+  --iqfeed-iqconnect TEXT         The path to the IQConnect binary
+  --iqfeed-username TEXT          Your IQFeed username
+  --iqfeed-password TEXT          Your IQFeed password
+  --iqfeed-version TEXT           The product version of your IQFeed developer account
+  --iqfeed-host TEXT              The IQFeed host address
+  --polygon-api-key TEXT          Your Polygon.io API Key
+  --terminal-link-connection-type [DAPI|SAPI]
+                                  Terminal Link Connection Type [DAPI, SAPI]
+  --terminal-link-server-auth-id TEXT
+                                  The Auth ID of the TerminalLink server
+  --terminal-link-environment [Production|Beta]
+                                  The environment to run in
+  --terminal-link-server-host TEXT
+                                  The host of the TerminalLink server
+  --terminal-link-server-port INTEGER
+                                  The port of the TerminalLink server
+  --terminal-link-openfigi-api-key TEXT
+                                  The Open FIGI API key to use for mapping options
   --lean-config FILE              The Lean configuration file that should be used (defaults to the nearest lean.json)
   --verbose                       Enable debug logging
   --help                          Show this message and exit.

lean/commands/optimize.py

@@ -22,9 +22,12 @@
 from lean.constants import DEFAULT_ENGINE_IMAGE
 from lean.container import container
 from lean.models.api import QCParameter, QCBacktest
+from lean.models.click_options import options_from_json, get_configs_for_options
+from lean.models.data_providers import all_data_providers, QuantConnectDataProvider, DataProvider
 from lean.models.errors import MoreInfoError
 from lean.models.optimizer import OptimizationTarget
-from lean.components.util.json_modules_handler import build_and_configure_modules
+from lean.components.util.json_modules_handler import build_and_configure_modules, get_and_build_module
+
 
 def _get_latest_backtest_runtime(algorithm_directory: Path) -> timedelta:
     from re import findall
@@ -94,6 +97,14 @@ def get_filename_timestamp(path: Path) -> datetime:
               type=str,
               multiple=True,
               help="The 'statistic operator value' pairs configuring the constraints of the optimization")
+@option("--data-provider",
+              type=Choice([dp.get_name() for dp in all_data_providers], case_sensitive=False),
+              default="Local",
+              help="Update the Lean configuration file to retrieve data from the given provider")
+@option("--download-data",
+              is_flag=True,
+              default=False,
+              help="Update the Lean configuration file to download data from the QuantConnect API, alias for --data-provider QuantConnect")
 @option("--release",
               is_flag=True,
               default=False,
@@ -129,6 +140,7 @@ def get_filename_timestamp(path: Path) -> datetime:
               is_flag=True,
               default=False,
               help="Use the local LEAN engine image instead of pulling the latest version")
+@options_from_json(get_configs_for_options("backtest"))
 def optimize(project: Path,
              output: Optional[Path],
              detach: bool,
@@ -138,6 +150,8 @@ def optimize(project: Path,
              target_direction: Optional[str],
              parameter: List[Tuple[str, float, float, float]],
              constraint: List[str],
+             data_provider: Optional[str],
+             download_data: bool,
              release: bool,
              image: Optional[str],
              update: bool,
@@ -146,7 +160,8 @@ def optimize(project: Path,
              addon_module: Optional[List[str]],
              extra_config: Optional[Tuple[str, str]],
              extra_docker_config: Optional[str],
-             no_update: bool) -> None:
+             no_update: bool,
+             **kwargs) -> None:
     """Optimize a project's parameters locally using Docker.
 
     \b
@@ -281,6 +296,17 @@ def optimize(project: Path,
     lean_config_manager = container.lean_config_manager
     lean_config = lean_config_manager.get_complete_lean_config(environment, algorithm_file, None)
 
+    organization_id = container.organization_manager.try_get_working_organization_id()
+
+    if download_data:
+        data_provider = QuantConnectDataProvider.get_name()
+
+    if data_provider is not None:
+        data_provider_configurer: DataProvider = get_and_build_module(data_provider, all_data_providers, kwargs, logger)
+        data_provider_configurer.ensure_module_installed(organization_id)
+        data_provider_configurer.configure(lean_config, environment)
+        logger.info(lean_config)
+
     if not output.exists():
         output.mkdir(parents=True)
 
@@ -301,7 +327,7 @@ def optimize(project: Path,
     lean_config["algorithm-id"] = str(output_config_manager.get_optimization_id(output))
 
     # Configure addon modules
-    build_and_configure_modules(addon_module, container.organization_manager.try_get_working_organization_id(), lean_config, logger, environment)
+    build_and_configure_modules(addon_module, organization_id, lean_config, logger, environment)
 
     run_options = lean_runner.get_basic_docker_config(lean_config, algorithm_file, output, None, release, should_detach)
 

tests/commands/test_optimize.py

@@ -19,6 +19,7 @@
 from click.testing import CliRunner
 
 from lean.commands import lean
+from lean.components.cloud.module_manager import ModuleManager
 from lean.components.config.storage import Storage
 from lean.constants import DEFAULT_ENGINE_IMAGE, LEAN_ROOT_PATH
 from lean.container import container
@@ -786,3 +787,32 @@ def test_optimize_runs_lean_container_with_extra_docker_config() -> None:
     volumes = kwargs["volumes"]
     assert "extra/path" in volumes
     assert volumes["extra/path"] == {"bind": "/extra/path", "mode": "rw"}
+
+
+def test_optimize_used_data_downloader_specified_with_data_provider_option() -> None:
+    create_fake_lean_cli_directory()
+
+    docker_manager = mock.MagicMock()
+    docker_manager.run_image.side_effect = run_image
+    container.initialize(docker_manager=docker_manager)
+    container.optimizer_config_manager = _get_optimizer_config_manager_mock()
+
+    Storage(str(Path.cwd() / "Python Project" / "config.json")).set("parameters", {"param1": "1"})
+
+    with mock.patch.object(ModuleManager, "install_module"):
+        result = CliRunner().invoke(lean, ["optimize", "Python Project",
+                                           "--data-provider", "Polygon",
+                                           "--polygon-api-key", "my-key"])
+
+    assert result.exit_code == 0
+
+    docker_manager.run_image.assert_called_once()
+    args, kwargs = docker_manager.run_image.call_args
+    mounts = kwargs["mounts"]
+
+    lean_config_filename = next(mount["Source"] for mount in mounts if mount["Target"] == "/Lean/Launcher/bin/Debug/config.json")
+    assert lean_config_filename is not None
+
+    config = json.loads(Path(lean_config_filename).read_text(encoding="utf-8"))
+    assert "data-downloader" in config
+    assert config["data-downloader"] == "QuantConnect.Polygon.PolygonDataDownloader"