diff --git a/lean/commands/backtest.py b/lean/commands/backtest.py index 98d6fb3e..40ba8eba 100644 --- a/lean/commands/backtest.py +++ b/lean/commands/backtest.py @@ -278,6 +278,15 @@ def _migrate_csharp_csproj(project_dir: Path) -> None: default="{}", help="Extra docker configuration as a JSON string. " "For more information https://docker-py.readthedocs.io/en/stable/containers.html") +@option("--extra-docker-config-file", + type=PathParameter(exists=True, file_okay=True, dir_okay=False), + help="Path to a JSON file with extra docker configuration. " + "This is recommended over --extra-docker-config on Windows to avoid shell quote issues.") +@option("--docker-timeout", + type=int, + help="Timeout in seconds for Docker operations (default: 60). " + "Increase this for slow connections or large image pulls. " + "Can also be set via DOCKER_CLIENT_TIMEOUT environment variable.") @option("--no-update", is_flag=True, default=False, @@ -298,6 +307,8 @@ def backtest(project: Path, addon_module: Optional[List[str]], extra_config: Optional[Tuple[str, str]], extra_docker_config: Optional[str], + extra_docker_config_file: Optional[Path], + docker_timeout: Optional[int], no_update: bool, parameter: List[Tuple[str, str]], **kwargs) -> None: @@ -316,9 +327,14 @@ def backtest(project: Path, Alternatively you can set the default engine image for all commands using `lean config set engine-image `. """ from datetime import datetime - from json import loads + from lean.components.util.json_parser import load_json_from_file_or_string logger = container.logger + + # Set Docker timeout if specified + if docker_timeout is not None: + container.docker_manager.set_timeout(docker_timeout) + project_manager = container.project_manager algorithm_file = project_manager.find_algorithm_file(Path(project)) lean_config_manager = container.lean_config_manager @@ -402,6 +418,12 @@ def backtest(project: Path, # Override existing parameters if any are provided via --parameter lean_config["parameters"] = lean_config_manager.get_parameters(parameter) + # Parse extra Docker configuration from string or file + parsed_extra_docker_config = load_json_from_file_or_string( + json_string=extra_docker_config if extra_docker_config != "{}" else None, + json_file=extra_docker_config_file + ) + lean_runner = container.lean_runner lean_runner.run_lean(lean_config, environment_name, @@ -411,5 +433,5 @@ def backtest(project: Path, debugging_method, release, detach, - loads(extra_docker_config), + parsed_extra_docker_config, paths_to_mount) diff --git a/lean/commands/live/deploy.py b/lean/commands/live/deploy.py index de7576ed..788a375d 100644 --- a/lean/commands/live/deploy.py +++ b/lean/commands/live/deploy.py @@ -110,6 +110,15 @@ def _get_history_provider_name(data_provider_live_names: [str]) -> [str]: default="{}", help="Extra docker configuration as a JSON string. " "For more information https://docker-py.readthedocs.io/en/stable/containers.html") +@option("--extra-docker-config-file", + type=PathParameter(exists=True, file_okay=True, dir_okay=False), + help="Path to a JSON file with extra docker configuration. " + "This is recommended over --extra-docker-config on Windows to avoid shell quote issues.") +@option("--docker-timeout", + type=int, + help="Timeout in seconds for Docker operations (default: 60). " + "Increase this for slow connections or large image pulls. " + "Can also be set via DOCKER_CLIENT_TIMEOUT environment variable.") @option("--no-update", is_flag=True, default=False, @@ -131,6 +140,8 @@ def deploy(project: Path, addon_module: Optional[List[str]], extra_config: Optional[Tuple[str, str]], extra_docker_config: Optional[str], + extra_docker_config_file: Optional[Path], + docker_timeout: Optional[int], no_update: bool, **kwargs) -> None: """Start live trading a project locally using Docker. @@ -155,10 +166,14 @@ def deploy(project: Path, """ from copy import copy from datetime import datetime - from json import loads + from lean.components.util.json_parser import load_json_from_file_or_string logger = container.logger - + + # Set Docker timeout if specified + if docker_timeout is not None: + container.docker_manager.set_timeout(docker_timeout) + project_manager = container.project_manager algorithm_file = project_manager.find_algorithm_file(Path(project)) @@ -323,5 +338,8 @@ def deploy(project: Path, None, release, detach, - loads(extra_docker_config), + load_json_from_file_or_string( + json_string=extra_docker_config if extra_docker_config != "{}" else None, + json_file=extra_docker_config_file + ), paths_to_mount) diff --git a/lean/commands/optimize.py b/lean/commands/optimize.py index 8b686bee..bb5680bc 100644 --- a/lean/commands/optimize.py +++ b/lean/commands/optimize.py @@ -139,6 +139,15 @@ def get_filename_timestamp(path: Path) -> datetime: default="{}", help="Extra docker configuration as a JSON string. " "For more information https://docker-py.readthedocs.io/en/stable/containers.html") +@option("--extra-docker-config-file", + type=PathParameter(exists=True, file_okay=True, dir_okay=False), + help="Path to a JSON file with extra docker configuration. " + "This is recommended over --extra-docker-config on Windows to avoid shell quote issues.") +@option("--docker-timeout", + type=int, + help="Timeout in seconds for Docker operations (default: 60). " + "Increase this for slow connections or large image pulls. " + "Can also be set via DOCKER_CLIENT_TIMEOUT environment variable.") @option("--no-update", is_flag=True, default=False, @@ -164,6 +173,8 @@ def optimize(project: Path, addon_module: Optional[List[str]], extra_config: Optional[Tuple[str, str]], extra_docker_config: Optional[str], + extra_docker_config_file: Optional[Path], + docker_timeout: Optional[int], no_update: bool, **kwargs) -> None: """Optimize a project's parameters locally using Docker. @@ -207,8 +218,13 @@ def optimize(project: Path, from docker.types import Mount from re import findall, search from os import cpu_count + from lean.components.util.json_parser import load_json_from_file_or_string from math import floor + # Set Docker timeout if specified + if docker_timeout is not None: + container.docker_manager.set_timeout(docker_timeout) + should_detach = detach and not estimate environment_name = "backtesting" project_manager = container.project_manager @@ -343,7 +359,12 @@ def optimize(project: Path, ) # Add known additional run options from the extra docker config - LeanRunner.parse_extra_docker_config(run_options, loads(extra_docker_config)) + # Parse extra Docker configuration from string or file + parsed_extra_docker_config = load_json_from_file_or_string( + json_string=extra_docker_config if extra_docker_config != "{}" else None, + json_file=extra_docker_config_file + ) + LeanRunner.parse_extra_docker_config(run_options, parsed_extra_docker_config) project_manager.copy_code(algorithm_file.parent, output / "code") diff --git a/lean/commands/research.py b/lean/commands/research.py index 018c618c..9242dfc6 100644 --- a/lean/commands/research.py +++ b/lean/commands/research.py @@ -71,6 +71,15 @@ def _check_docker_output(chunk: str, port: int) -> None: default="{}", help="Extra docker configuration as a JSON string. " "For more information https://docker-py.readthedocs.io/en/stable/containers.html") +@option("--extra-docker-config-file", + type=PathParameter(exists=True, file_okay=True, dir_okay=False), + help="Path to a JSON file with extra docker configuration. " + "This is recommended over --extra-docker-config on Windows to avoid shell quote issues.") +@option("--docker-timeout", + type=int, + help="Timeout in seconds for Docker operations (default: 60). " + "Increase this for slow connections or large image pulls. " + "Can also be set via DOCKER_CLIENT_TIMEOUT environment variable.") @option("--no-update", is_flag=True, default=False, @@ -86,6 +95,8 @@ def research(project: Path, update: bool, extra_config: Optional[Tuple[str, str]], extra_docker_config: Optional[str], + extra_docker_config_file: Optional[Path], + docker_timeout: Optional[int], no_update: bool, **kwargs) -> None: """Run a Jupyter Lab environment locally using Docker. @@ -96,10 +107,14 @@ def research(project: Path, """ from docker.types import Mount from docker.errors import APIError - from json import loads + from lean.components.util.json_parser import load_json_from_file_or_string logger = container.logger - + + # Set Docker timeout if specified + if docker_timeout is not None: + container.docker_manager.set_timeout(docker_timeout) + project_manager = container.project_manager algorithm_file = project_manager.find_algorithm_file(project, not_throw = True) @@ -195,7 +210,12 @@ def research(project: Path, run_options["commands"].append("./start.sh") # Add known additional run options from the extra docker config - LeanRunner.parse_extra_docker_config(run_options, loads(extra_docker_config)) + # Parse extra Docker configuration from string or file + parsed_extra_docker_config = load_json_from_file_or_string( + json_string=extra_docker_config if extra_docker_config != "{}" else None, + json_file=extra_docker_config_file + ) + LeanRunner.parse_extra_docker_config(run_options, parsed_extra_docker_config) try: container.docker_manager.run_image(research_image, **run_options) diff --git a/lean/components/docker/docker_manager.py b/lean/components/docker/docker_manager.py index dbb95391..ff948cf4 100644 --- a/lean/components/docker/docker_manager.py +++ b/lean/components/docker/docker_manager.py @@ -27,16 +27,25 @@ class DockerManager: """The DockerManager contains methods to manage and run Docker images.""" - def __init__(self, logger: Logger, temp_manager: TempManager, platform_manager: PlatformManager) -> None: + def __init__(self, logger: Logger, temp_manager: TempManager, platform_manager: PlatformManager, timeout: int = 60) -> None: """Creates a new DockerManager instance. :param logger: the logger to use when printing messages :param temp_manager: the TempManager instance used when creating temporary directories :param platform_manager: the PlatformManager used when checking which operating system is in use + :param timeout: the timeout in seconds for Docker client operations (default: 60) """ self._logger = logger self._temp_manager = temp_manager self._platform_manager = platform_manager + self._timeout = timeout + + def set_timeout(self, timeout: int) -> None: + """Set the timeout for Docker client operations. + + :param timeout: The timeout in seconds for Docker client operations + """ + self._timeout = timeout def get_image_labels(self, image: str) -> str: docker_image = self._get_docker_client().images.get(image) @@ -570,7 +579,16 @@ def _get_docker_client(self): try: from docker import from_env - docker_client = from_env() + from os import environ + + # Check for environment variable override + try: + timeout = int(environ.get("DOCKER_CLIENT_TIMEOUT", self._timeout)) + except ValueError: + # Fall back to instance timeout on invalid value + timeout = self._timeout + + docker_client = from_env(timeout=timeout) except Exception: raise error diff --git a/lean/components/util/json_parser.py b/lean/components/util/json_parser.py new file mode 100644 index 00000000..05c7268e --- /dev/null +++ b/lean/components/util/json_parser.py @@ -0,0 +1,94 @@ +# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. +# Lean CLI v1.0. Copyright 2021 QuantConnect Corporation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pathlib import Path +from typing import Dict, Any, Optional +from json import loads, JSONDecodeError + + +def parse_json_safely(json_string: str) -> Dict[str, Any]: + """ + Attempts to parse a JSON string with multiple fallback strategies. + + This function is designed to handle JSON strings that may have been + mangled by Windows shells (PowerShell/CMD) which strip or escape quotes. + + :param json_string: The JSON string to parse + :return: Parsed dictionary + :raises ValueError: If all parsing attempts fail + """ + if not json_string or json_string.strip() == "": + return {} + + # Try standard JSON parsing first + try: + return loads(json_string) + except JSONDecodeError as e: + original_error = str(e) + + # Try fixing common Windows shell issues + # Try single quotes to double quotes (common Windows PowerShell issue) + try: + return loads(json_string.replace("'", '"')) + except JSONDecodeError: + pass + + # If all attempts fail, provide helpful error message + raise ValueError( + f"Failed to parse JSON configuration. Original error: {original_error}\n" + f"Input: {json_string}\n\n" + f"On Windows, JSON strings may be mangled by the shell. Consider using --extra-docker-config-file instead.\n" + f"Example: Create a file 'docker-config.json' with your configuration and use:\n" + f" --extra-docker-config-file docker-config.json" + ) + + +def load_json_from_file_or_string( + json_string: Optional[str] = None, + json_file: Optional[Path] = None +) -> Dict[str, Any]: + """ + Loads JSON configuration from either a string or a file. + + If both json_file and json_string are provided, json_file takes precedence. + + :param json_string: JSON string to parse (optional) + :param json_file: Path to JSON file (optional) + :return: Parsed dictionary, or empty dict if both parameters are None + :raises ValueError: If parsing fails or if file doesn't exist + """ + # Validate that both parameters aren't provided (though we allow it, file takes precedence) + if json_file is not None and json_string is not None: + # Log a warning would be ideal, but we'll prioritize file as documented + pass + + if json_file is not None: + if not json_file.exists(): + raise ValueError(f"Configuration file not found: {json_file}") + + try: + with open(json_file, 'r', encoding='utf-8') as f: + content = f.read() + return loads(content) + except JSONDecodeError as e: + raise ValueError( + f"Failed to parse JSON from file {json_file}: {e}\n" + f"Please ensure the file contains valid JSON." + ) + except Exception as e: + raise ValueError(f"Failed to read file {json_file}: {e}") + + if json_string is not None: + return parse_json_safely(json_string) + + return {} diff --git a/lean/container.py b/lean/container.py index bc2df0b6..dd752b7e 100644 --- a/lean/container.py +++ b/lean/container.py @@ -102,7 +102,10 @@ def initialize(self, self.docker_manager = docker_manager if not self.docker_manager: - self.docker_manager = DockerManager(self.logger, self.temp_manager, self.platform_manager) + from os import environ + # Get timeout from environment variable, default to 60 seconds + timeout = int(environ.get("DOCKER_CLIENT_TIMEOUT", 60)) + self.docker_manager = DockerManager(self.logger, self.temp_manager, self.platform_manager, timeout) self.project_manager = ProjectManager(self.logger, self.project_config_manager,