Fix(windows): Allow 'sqlmesh clean' to delete cache files that exceed 260 chars

Author: erindru

sqlmesh/core/context.py

@@ -139,6 +139,7 @@
 )
 from sqlmesh.utils.config import print_config
 from sqlmesh.utils.jinja import JinjaMacroRegistry
+from sqlmesh.utils.windows import IS_WINDOWS, fix_windows_path
 
 if t.TYPE_CHECKING:
     import pandas as pd
@@ -2590,12 +2591,18 @@ def table_name(
         )
 
     def clear_caches(self) -> None:
-        for path in self.configs:
-            cache_path = path / c.CACHE
-            if cache_path.exists():
-                rmtree(cache_path)
-        if self.cache_dir.exists():
-            rmtree(self.cache_dir)
+        paths_to_remove = [path / c.CACHE for path in self.configs]
+        paths_to_remove.append(self.cache_dir)
+
+        if IS_WINDOWS:                       
+            paths_to_remove = [
+                fix_windows_path(path)
+                for path in paths_to_remove
+            ]            
+
+        for path in paths_to_remove:
+            if path.exists():
+                rmtree(path)
 
         if isinstance(self._state_sync, CachingStateSync):
             self._state_sync.clear_cache()

sqlmesh/utils/windows.py

@@ -3,12 +3,22 @@
 
 IS_WINDOWS = platform.system() == "Windows"
 
+WINDOWS_LONGPATH_PREFIX = "\\\\?\\"
+
 
 def fix_windows_path(path: Path) -> Path:
     """
     Windows paths are limited to 260 characters: https://learn.microsoft.com/en-us/windows/win32/fileio/maximum-file-path-limitation
     Users can change this by updating a registry entry but we cant rely on that.
-    We can quite commonly generate a cache file path that exceeds 260 characters which causes a FileNotFound error.
-    If we prefix the path with "\\?\" then we can have paths up to 32,767 characters
-    """
-    return Path("\\\\?\\" + str(path.absolute()))
+
+    SQLMesh quite commonly generates cache file paths that exceed 260 characters and thus cause a FileNotFound error.
+    If we prefix paths with "\\?\" then we can have paths up to 32,767 characters.
+
+    Note that this prefix also means that relative paths no longer work. From the above docs:
+     > Because you cannot use the "\\?\" prefix with a relative path, relative paths are always limited to a total of MAX_PATH characters.
+    
+    So we call path.resolve() to resolve the relative sections before appending the prefix to work around this
+    """    
+    if path.parts and not path.parts[0].startswith(WINDOWS_LONGPATH_PREFIX):
+        path = Path(WINDOWS_LONGPATH_PREFIX + str(path.absolute()))
+    return path.resolve()

tests/core/test_context.py

@@ -62,6 +62,8 @@
     NoChangesPlanError,
 )
 from sqlmesh.utils.metaprogramming import Executable
+from sqlmesh.utils.windows import IS_WINDOWS, fix_windows_path
+from sqlmesh.utils import yaml
 from tests.utils.test_helpers import use_terminal_console
 from tests.utils.test_filesystem import create_temp_file
 
@@ -700,6 +702,45 @@ def test_clear_caches(tmp_path: pathlib.Path):
     assert not cache_dir.exists()
 
 
+def test_clear_caches_with_long_base_path(tmp_path: pathlib.Path):
+    base_path = tmp_path / ("abcde" * 50)
+    assert (
+        len(str(base_path.absolute())) > 260
+    )  # Paths longer than 260 chars trigger problems on Windows
+    
+    default_cache_dir = base_path / c.CACHE
+    custom_cache_dir = base_path / ".test_cache"    
+
+    # note: we create the Context here so it doesnt get passed any "fixed" paths
+    ctx = Context(config=Config(cache_dir=str(custom_cache_dir)), paths=base_path)
+
+    if IS_WINDOWS:
+        # fix these so we can use them in this test
+        default_cache_dir = fix_windows_path(default_cache_dir)
+        custom_cache_dir = fix_windows_path(custom_cache_dir)        
+
+    default_cache_dir.mkdir(parents=True)
+    custom_cache_dir.mkdir(parents=True)
+
+    default_cache_file = default_cache_dir / "cache.txt"
+    custom_cache_file = custom_cache_dir / "cache.txt"
+
+    default_cache_file.write_text("test")
+    custom_cache_file.write_text("test")
+
+    assert default_cache_file.exists()        
+    assert custom_cache_file.exists()    
+    assert default_cache_dir.exists()        
+    assert custom_cache_dir.exists()           
+    
+    ctx.clear_caches()
+
+    assert not default_cache_file.exists()        
+    assert not custom_cache_file.exists()    
+    assert not default_cache_dir.exists()        
+    assert not custom_cache_dir.exists()      
+
+
 def test_cache_path_configurations(tmp_path: pathlib.Path):
     project_dir = tmp_path / "project"
     project_dir.mkdir(parents=True)

tests/utils/test_windows.py

@@ -0,0 +1,33 @@
+import pytest
+from pathlib import Path
+from sqlmesh.utils.windows import IS_WINDOWS, WINDOWS_LONGPATH_PREFIX, fix_windows_path
+
+@pytest.mark.skipif(not IS_WINDOWS, reason="pathlib.Path only produces WindowsPath objects on Windows")
+def test_fix_windows_path():
+    short_path = Path("c:\\foo")
+    short_path_prefixed = Path(WINDOWS_LONGPATH_PREFIX + "c:\\foo")
+
+    segments = "\\".join(["bar", "baz", "bing"] * 50)
+    long_path = Path("c:\\" + segments)    
+    long_path_prefixed = Path(WINDOWS_LONGPATH_PREFIX + "c:\\" + segments)
+
+    assert len(str(short_path.absolute)) < 260
+    assert len(str(long_path.absolute)) > 260
+
+    # paths less than 260 chars are still prefixed because they may be being used as a base path
+    assert fix_windows_path(short_path) == short_path_prefixed
+
+    # paths greater than 260 characters don't work at all without the prefix
+    assert fix_windows_path(long_path) == long_path_prefixed
+
+    # multiple calls dont keep appending the same prefix
+    assert fix_windows_path(fix_windows_path(fix_windows_path(long_path_prefixed))) == long_path_prefixed
+
+    # paths with relative sections need to have relative sections resolved before they can be used
+    # since the \\?\ prefix doesnt work for paths with relative sections
+    assert fix_windows_path(Path("c:\\foo\..\\bar")) == Path(WINDOWS_LONGPATH_PREFIX + "c:\\bar")
+
+    # also check that relative sections are still resolved if they are added to a previously prefixed path
+    base = fix_windows_path(Path("c:\\foo"))
+    assert base == Path(WINDOWS_LONGPATH_PREFIX + "c:\\foo")
+    assert fix_windows_path(base / ".." / "bar") == Path(WINDOWS_LONGPATH_PREFIX + "c:\\bar")