feat: Rewrite CQL2 implementation for SFEOS

Author: Yuri Zmytrakov

stac_fastapi/core/stac_fastapi/core/core.py

@@ -875,6 +875,12 @@ async def post_search(
                 query_fields = get_properties_from_cql2_filter(cql2_filter)
                 await self.queryables_cache.validate(query_fields)
                 search = await self.database.apply_cql2_filter(search, cql2_filter)
+                date_str = getattr(search, "_cql2_date_str", None)
+                if date_str is not None:
+                    datetime_parsed = format_datetime_range(date_str=date_str)
+                    search, datetime_search = self.database.apply_datetime_filter(
+                        search=search, datetime=datetime_parsed
+                    )
             except HTTPException:
                 raise
             except Exception as e:

stac_fastapi/core/stac_fastapi/core/extensions/filter.py

@@ -13,8 +13,9 @@
 # defines spatial operators (S_INTERSECTS, S_CONTAINS, S_WITHIN, S_DISJOINT).
 # """
 
+from dataclasses import dataclass
 from enum import Enum
-from typing import Any, Dict
+from typing import Any, Dict, List, Optional
 
 DEFAULT_QUERYABLES: Dict[str, Dict[str, Any]] = {
     "id": {
@@ -90,3 +91,62 @@ class SpatialOp(str, Enum):
     S_CONTAINS = "s_contains"
     S_WITHIN = "s_within"
     S_DISJOINT = "s_disjoint"
+
+
+@dataclass
+class CqlNode:
+    """Base class."""
+
+    pass
+
+
+@dataclass
+class LogicalNode(CqlNode):
+    """Logical operators (AND, OR, NOT)."""
+
+    op: LogicalOp
+    children: List["CqlNode"]
+
+
+@dataclass
+class ComparisonNode(CqlNode):
+    """Comparison operators (=, <>, <, <=, >, >=, is null)."""
+
+    op: ComparisonOp
+    field: str
+    value: Any
+
+
+@dataclass
+class AdvancedComparisonNode(CqlNode):
+    """Advanced comparison operators (like, between, in)."""
+
+    op: AdvancedComparisonOp
+    field: str
+    value: Any
+
+
+@dataclass
+class SpatialNode(CqlNode):
+    """Spatial operators."""
+
+    op: SpatialOp
+    field: str
+    geometry: Dict[str, Any]
+
+
+@dataclass
+class DateTimeRangeNode(CqlNode):
+    """Datetime range queries."""
+
+    field: str = "properties.datetime"
+    start: Optional[str] = None
+    end: Optional[str] = None
+
+
+@dataclass
+class DateTimeExactNode(CqlNode):
+    """Exact datetime queries."""
+
+    field: str = "properties.datetime"
+    value: Optional[str] = None

stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py

@@ -52,6 +52,12 @@
     merge_to_operations,
     operations_to_script,
 )
+from stac_fastapi.sfeos_helpers.filter import (
+    Cql2AstParser,
+    DatetimeOptimizer,
+    to_es_via_ast,
+)
+from stac_fastapi.sfeos_helpers.filter.datetime_optimizer import extract_from_ast
 from stac_fastapi.sfeos_helpers.mappings import (
     AGGREGATION_MAPPING,
     COLLECTIONS_INDEX,
@@ -683,11 +689,11 @@ async def apply_cql2_filter(
         self, search: Search, _filter: Optional[Dict[str, Any]]
     ):
         """
-        Apply a CQL2 filter to an Opensearch Search object.
+        Apply a CQL2 filter to an OpenSearch Search object.
 
-        This method transforms a dictionary representing a CQL2 filter into an Opensearch query
-        and applies it to the provided Search object. If the filter is None, the original Search
-        object is returned unmodified.
+        This method transforms a CQL2 filter dictionary into an OpenSearch query using
+        an AST tree-based approach. If the filter is None, the original Search object is returned
+        unmodified.
 
         Args:
             search (Search): The Opensearch Search object to which the filter will be applied.
@@ -701,8 +707,26 @@ async def apply_cql2_filter(
                     otherwise the original Search object.
         """
         if _filter is not None:
-            es_query = filter_module.to_es(await self.get_queryables_mapping(), _filter)
-            search = search.filter(es_query)
+            queryables_mapping = await self.get_queryables_mapping()
+
+            try:
+                parser = Cql2AstParser(queryables_mapping)
+                ast = parser.parse(_filter)
+
+                optimizer = DatetimeOptimizer()
+                optimized_ast = optimizer.optimize_query_structure(ast)
+
+                date_str = extract_from_ast(optimized_ast, "datetime")
+                es_query = to_es_via_ast(queryables_mapping, optimized_ast)
+
+                search = search.filter(es_query)
+                search._cql2_date_str = date_str
+
+            except Exception:
+                # Fallback to dictionary-based approach
+                es_query = filter_module.to_es(queryables_mapping, _filter)
+                search = search.filter(es_query)
+                return search
 
         return search
 

stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/filter/__init__.py

@@ -11,6 +11,8 @@
 - cql2.py: CQL2 pattern conversion helpers
 - transform.py: Query transformation functions
 - client.py: Filter client implementation
+- ast_parser.py: AST parser for CQL2 queries
+- datetime_optimizer.py: Datetime optimization for query structure
 
 When adding new functionality to this package, consider:
 1. Will this code be used by both Elasticsearch and OpenSearch implementations?
@@ -22,6 +24,14 @@
 - Parameter names should be consistent across similar functions
 """
 
+from stac_fastapi.core.extensions.filter import (
+    AdvancedComparisonOp,
+    ComparisonOp,
+    LogicalOp,
+)
+
+from .ast_parser import Cql2AstParser
+from .ast_transform import to_es_via_ast
 from .client import EsAsyncBaseFiltersClient
 
 # Re-export the main functions and classes for backward compatibility
@@ -31,6 +41,7 @@
     cql2_like_to_es,
     valid_like_substitutions,
 )
+from .datetime_optimizer import DatetimeOptimizer
 from .transform import to_es, to_es_field
 
 __all__ = [
@@ -40,5 +51,12 @@
     "_replace_like_patterns",
     "to_es_field",
     "to_es",
+    "to_es_via_ast",
     "EsAsyncBaseFiltersClient",
+    "Cql2AstParser",
+    "AdvancedComparisonOp",
+    "ComparisonOp",
+    "LogicalOp",
+    "DatetimeOptimizer",
+    "extract_from_ast",
 ]

stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/filter/ast_parser.py

@@ -0,0 +1,113 @@
+"""AST parser for CQL2 queries."""
+
+import json
+from typing import Any, Dict, Union
+
+from stac_fastapi.core.extensions.filter import (
+    AdvancedComparisonNode,
+    AdvancedComparisonOp,
+    ComparisonNode,
+    ComparisonOp,
+    CqlNode,
+    LogicalNode,
+    LogicalOp,
+    SpatialNode,
+    SpatialOp,
+)
+
+
+class Cql2AstParser:
+    """Parse CQL2 into AST tree."""
+
+    def __init__(self, queryables_mapping: Dict[str, Any]):
+        """Initialize the CQL2 AST parser."""
+        self.queryables_mapping = queryables_mapping
+
+    def parse(self, cql: Union[str, Dict[str, Any]]) -> CqlNode:
+        """Parse CQL2 into AST tree.
+
+        Args:
+            cql: CQL2 expression as string/dictionary
+
+        Returns:
+            Node of AST tree
+        """
+        if isinstance(cql, str):
+            data: Dict[str, Any] = json.loads(cql)
+            return self._parse_node(data)
+
+        return self._parse_node(cql)
+
+    def _parse_node(self, node: Dict[str, Any]) -> CqlNode:
+        """Parse a single CQL2 node into AST."""
+        if "op" in node and node["op"] in ["and", "or", "not"]:
+            op = LogicalOp(node["op"])
+            args = node.get("args", [])
+
+            if op == LogicalOp.NOT:
+                children = [self._parse_node(args[0])] if args else []
+            else:
+                children = [self._parse_node(arg) for arg in args]
+
+            return LogicalNode(op=op, children=children)
+
+        elif "op" in node and node["op"] in ["=", "<>", "<", "<=", ">", ">=", "isNull"]:
+            op = ComparisonOp(node["op"])
+            args = node.get("args", [])
+
+            if isinstance(args[0], dict) and "property" in args[0]:
+                field = args[0]["property"]
+            else:
+                field = str(args[0])
+
+            value = args[1] if len(args) > 1 else None
+
+            return ComparisonNode(op=op, field=field, value=value)
+
+        elif "op" in node and node["op"] in ["like", "between", "in"]:
+            op = AdvancedComparisonOp(node["op"])
+            args = node.get("args", [])
+
+            if isinstance(args[0], dict) and "property" in args[0]:
+                field = args[0]["property"]
+            else:
+                field = str(args[0])
+
+            if op == AdvancedComparisonOp.BETWEEN:
+                if len(args) != 3:
+                    raise ValueError(
+                        f"BETWEEN operator requires (property, lower, upper), got {args}"
+                    )
+                value = (args[1], args[2])
+
+            elif op == AdvancedComparisonOp.IN:
+                if not isinstance(args[1], list):
+                    raise ValueError(f"IN operator expects list, got {type(args[1])}")
+                value = args[1]
+
+            elif op == AdvancedComparisonOp.LIKE:
+                if len(args) != 2:
+                    raise ValueError(
+                        f"LIKE operator requires (property, pattern), got {args}"
+                    )
+                value = args[1]
+
+            return AdvancedComparisonNode(op=op, field=field, value=value)
+
+        elif "op" in node and node["op"] in [
+            "s_intersects",
+            "s_contains",
+            "s_within",
+            "s_disjoint",
+        ]:
+            op = SpatialOp(node["op"])
+            args = node.get("args", [])
+
+            if isinstance(args[0], dict) and "property" in args[0]:
+                field = args[0]["property"]
+            else:
+                field = str(args[0])
+
+            geometry = args[1]
+
+            return SpatialNode(op=op, field=field, geometry=geometry)