feat: add ability to configure logging dir (#2266)

Author: eakmanrq

sqlmesh/__init__.py

@@ -11,6 +11,7 @@
 import typing as t
 from datetime import datetime
 from enum import Enum
+from pathlib import Path
 
 from sqlmesh.core.dialect import extend_sqlglot
 
@@ -93,7 +94,7 @@ def is_notebook(self) -> bool:
 
 
 LOG_FORMAT = "%(asctime)s - %(threadName)s - %(name)s - %(levelname)s - %(message)s (%(filename)s:%(lineno)d)"
-LOG_PREFIX = "logs/sqlmesh_"
+LOG_FILENAME_PREFIX = "sqlmesh_"
 
 
 # SO: https://stackoverflow.com/questions/384076/how-can-i-color-python-logging-output
@@ -126,6 +127,7 @@ def configure_logging(
     write_to_stdout: bool = False,
     write_to_file: bool = True,
     log_limit: int = c.DEFAULT_LOG_LIMIT,
+    log_file_dir: t.Optional[t.Union[str, Path]] = None,
 ) -> None:
     logger = logging.getLogger()
     debug = force_debug or debug_mode_enabled()
@@ -141,9 +143,11 @@ def configure_logging(
     )
     logger.addHandler(stdout_handler)
 
+    log_file_dir = log_file_dir or c.DEFAULT_LOG_FILE_DIR
+    log_path_prefix = Path(log_file_dir) / LOG_FILENAME_PREFIX
     if write_to_file:
-        os.makedirs("logs", exist_ok=True)
-        filename = f"{LOG_PREFIX}{datetime.now().strftime('%Y_%m_%d_%H_%M_%S')}.log"
+        os.makedirs(str(log_file_dir), exist_ok=True)
+        filename = f"{log_path_prefix}{datetime.now().strftime('%Y_%m_%d_%H_%M_%S')}.log"
         file_handler = logging.FileHandler(filename, mode="w", encoding="utf-8")
         # the log files should always log at least info so that users will always have
         # minimal info for debugging even if they specify "ignore_warnings"
@@ -152,7 +156,7 @@ def configure_logging(
         logger.addHandler(file_handler)
 
     if log_limit > 0:
-        for path in list(sorted(glob.glob(f"{LOG_PREFIX}*.log"), reverse=True))[log_limit:]:
+        for path in list(sorted(glob.glob(f"{log_path_prefix}*.log"), reverse=True))[log_limit:]:
             os.remove(path)
 
     if debug:

sqlmesh/cli/main.py

@@ -52,6 +52,11 @@ def _sqlmesh_version() -> str:
     is_flag=True,
     help="Display logs in stdout.",
 )
+@click.option(
+    "--log-file-dir",
+    type=str,
+    help="The directory to write log files to.",
+)
 @click.pass_context
 @error_handler
 def cli(
@@ -62,6 +67,7 @@ def cli(
     ignore_warnings: bool = False,
     debug: bool = False,
     log_to_stdout: bool = False,
+    log_file_dir: t.Optional[str] = None,
 ) -> None:
     """SQLMesh command line tool."""
     if "--help" in sys.argv:
@@ -79,7 +85,9 @@ def cli(
 
     configs = load_configs(config, Context.CONFIG_TYPE, paths)
     log_limit = list(configs.values())[0].log_limit
-    configure_logging(debug, ignore_warnings, log_to_stdout, log_limit=log_limit)
+    configure_logging(
+        debug, ignore_warnings, log_to_stdout, log_limit=log_limit, log_file_dir=log_file_dir
+    )
 
     try:
         context = Context(

sqlmesh/core/constants.py

@@ -36,6 +36,9 @@
 DEFAULT_LOG_LIMIT = 20
 """The default number of logs to keep."""
 
+DEFAULT_LOG_FILE_DIR = "logs"
+"""The default directory for log files."""
+
 AUDITS = "audits"
 MACROS = "macros"
 METRICS = "metrics"

sqlmesh/magics.py

@@ -91,6 +91,7 @@ def _shell(self) -> t.Any:
     @argument("--gateway", type=str, help="The name of the gateway.")
     @argument("--ignore-warnings", action="store_true", help="Ignore warnings.")
     @argument("--debug", action="store_true", help="Enable debug mode.")
+    @argument("--log-file-dir", type=str, help="The directory to write the log file to.")
     @line_magic
     def context(self, line: str) -> None:
         """Sets the context in the user namespace."""
@@ -99,7 +100,9 @@ def context(self, line: str) -> None:
         args = parse_argstring(self.context, line)
         configs = load_configs(args.config, Context.CONFIG_TYPE, args.paths)
         log_limit = list(configs.values())[0].log_limit
-        configure_logging(args.debug, args.ignore_warnings, log_limit=log_limit)
+        configure_logging(
+            args.debug, args.ignore_warnings, log_limit=log_limit, log_file_dir=args.log_file_dir
+        )
         try:
             context = Context(paths=args.paths, config=configs, gateway=args.gateway)
             self._shell.user_ns["context"] = context