feat: move queryables cache and validation logic to core

Author: Andrzej Pijanowski

README.md

@@ -367,7 +367,7 @@ You can customize additional settings in your `.env` file:
 | `STAC_INDEX_ASSETS` | Controls if Assets are indexed when added to Elasticsearch/Opensearch. This allows asset fields to be included in search queries. | `false` | Optional |
 | `USE_DATETIME` | Configures the datetime search behavior in SFEOS. When enabled, searches both datetime field and falls back to start_datetime/end_datetime range for items with null datetime. When disabled, searches only by start_datetime/end_datetime range. | `true` | Optional |
 | `USE_DATETIME_NANOS` | Enables nanosecond precision handling for `datetime` field searches as per the `date_nanos` type. When `False`, it uses 3 millisecond precision as per the type `date`. | `true` | Optional |
-| `EXCLUDED_FROM_QUERYABLES` | Comma-separated list of fully qualified field names to exclude from the queryables endpoint and filtering. Use full paths like `properties.auth:schemes,properties.storage:schemes`. Excluded fields and their nested children will not be exposed in queryables. | None | Optional |
+| `EXCLUDED_FROM_QUERYABLES` | Comma-separated list of fully qualified field names to exclude from the queryables endpoint and filtering. Use full paths like `properties.auth:schemes,properties.storage:schemes`. Excluded fields and their nested children will not be exposed in queryables. If `VALIDATE_QUERYABLES` is enabled, these fields will also be considered invalid for filtering. | None | Optional |
 | `EXCLUDED_FROM_ITEMS` | Specifies fields to exclude from STAC item responses. Supports comma-separated field names and dot notation for nested fields (e.g., `private_data,properties.confidential,assets.internal`). | `None` | Optional |
 | `VALIDATE_QUERYABLES` | Enable validation of query parameters against the collection's queryables. If set to `true`, the API will reject queries containing fields that are not defined in the collection's queryables. | `false` | Optional |
 | `QUERYABLES_CACHE_TTL` | Time-to-live (in seconds) for the queryables cache. Used when `VALIDATE_QUERYABLES` is enabled. | `3600` | Optional |

stac_fastapi/core/stac_fastapi/core/base_database_logic.py

@@ -138,3 +138,8 @@ async def delete_collection(
     ) -> None:
         """Delete a collection from the database."""
         pass
+
+    @abc.abstractmethod
+    async def get_queryables_mapping(self, collection_id: str = "*") -> Dict[str, Any]:
+        """Retrieve mapping of Queryables for search."""
+        pass

stac_fastapi/core/stac_fastapi/core/core.py

@@ -24,6 +24,10 @@
 from stac_fastapi.core.base_settings import ApiBaseSettings
 from stac_fastapi.core.datetime_utils import format_datetime_range
 from stac_fastapi.core.models.links import PagingLinks
+from stac_fastapi.core.queryables import (
+    QueryablesCache,
+    get_properties_from_cql2_filter,
+)
 from stac_fastapi.core.redis_utils import redis_pagination_links
 from stac_fastapi.core.serializers import CollectionSerializer, ItemSerializer
 from stac_fastapi.core.session import Session
@@ -39,11 +43,6 @@
     BulkTransactionMethod,
     Items,
 )
-from stac_fastapi.sfeos_helpers.queryables import (
-    get_properties_from_cql2_filter,
-    initialize_queryables_cache,
-    validate_queryables,
-)
 from stac_fastapi.types import stac as stac_types
 from stac_fastapi.types.conformance import BASE_CONFORMANCE_CLASSES
 from stac_fastapi.types.core import AsyncBaseCoreClient
@@ -95,7 +94,7 @@ class CoreClient(AsyncBaseCoreClient):
 
     def __attrs_post_init__(self):
         """Initialize the queryables cache."""
-        initialize_queryables_cache(self.database)
+        self.queryables_cache = QueryablesCache(self.database)
 
     def _landing_page(
         self,
@@ -826,7 +825,7 @@ async def post_search(
 
         if hasattr(search_request, "query") and getattr(search_request, "query"):
             query_fields = set(getattr(search_request, "query").keys())
-            await validate_queryables(query_fields)
+            await self.queryables_cache.validate(query_fields)
             for field_name, expr in getattr(search_request, "query").items():
                 field = "properties__" + field_name
                 for op, value in expr.items():
@@ -846,7 +845,7 @@ async def post_search(
         if cql2_filter is not None:
             try:
                 query_fields = get_properties_from_cql2_filter(cql2_filter)
-                await validate_queryables(query_fields)
+                await self.queryables_cache.validate(query_fields)
                 search = await self.database.apply_cql2_filter(search, cql2_filter)
             except HTTPException:
                 raise

…stac_fastapi/sfeos_helpers/queryables.py …api/core/stac_fastapi/core/queryables.pystac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/queryables.py renamed to stac_fastapi/core/stac_fastapi/core/queryables.py stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/queryables.py renamed to stac_fastapi/core/stac_fastapi/core/queryables.py

@@ -3,12 +3,10 @@
 import asyncio
 import os
 import time
-from typing import Any, Dict, List, Optional, Set
+from typing import Any, Dict, List, Set
 
 from fastapi import HTTPException
 
-from stac_fastapi.core.base_database_logic import BaseDatabaseLogic
-
 
 class QueryablesCache:
     """A thread-safe, time-based cache for queryable properties."""
@@ -99,40 +97,6 @@ async def validate(self, fields: Set[str]) -> None:
             )
 
 
-_queryables_cache_instance: Optional[QueryablesCache] = None
-
-
-def initialize_queryables_cache(database_logic: BaseDatabaseLogic):
-    """
-    Initialize the global queryables cache.
-
-    :param database_logic: An instance of DatabaseLogic.
-    """
-    global _queryables_cache_instance
-    if _queryables_cache_instance is None:
-        _queryables_cache_instance = QueryablesCache(database_logic)
-
-
-async def all_queryables() -> Set[str]:
-    """Get all queryable properties from the cache."""
-    if _queryables_cache_instance is None:
-        raise Exception("Queryables cache not initialized.")
-    return await _queryables_cache_instance.get_all_queryables()
-
-
-async def validate_queryables(fields: Set[str]) -> None:
-    """Validate if the provided fields are queryable."""
-    if _queryables_cache_instance is None:
-        return
-    await _queryables_cache_instance.validate(fields)
-
-
-def reload_queryables_settings():
-    """Reload queryables settings from environment variables."""
-    if _queryables_cache_instance:
-        _queryables_cache_instance.reload_settings()
-
-
 def get_properties_from_cql2_filter(cql2_filter: Dict[str, Any]) -> Set[str]:
     """Recursively extract property names from a CQL2 filter."""
     props: Set[str] = set()

stac_fastapi/tests/api/test_api_query_validation.py

@@ -4,15 +4,34 @@
 
 import pytest
 
-from stac_fastapi.sfeos_helpers.queryables import reload_queryables_settings
+if os.getenv("BACKEND", "elasticsearch").lower() == "opensearch":
+    from stac_fastapi.opensearch.app import app_config
+else:
+    from stac_fastapi.elasticsearch.app import app_config
+
+
+def get_core_client():
+    if os.getenv("BACKEND", "elasticsearch").lower() == "opensearch":
+        from stac_fastapi.opensearch.app import app_config
+    else:
+        from stac_fastapi.elasticsearch.app import app_config
+    return app_config["client"]
+
+
+def reload_queryables_settings():
+    client = get_core_client()
+    if hasattr(client, "queryables_cache"):
+        client.queryables_cache.reload_settings()
 
 
 @pytest.fixture(autouse=True)
 def enable_validation():
+
+    client = app_config["client"]
     with mock.patch.dict(os.environ, {"VALIDATE_QUERYABLES": "true"}):
-        reload_queryables_settings()
+        client.queryables_cache.reload_settings()
         yield
-    reload_queryables_settings()
+    client.queryables_cache.reload_settings()
 
 
 @pytest.mark.asyncio
@@ -72,6 +91,7 @@ async def test_validate_queryables_excluded(app_client, ctx):
     """Test that excluded queryables are rejected when validation is enabled."""
 
     excluded_field = "eo:cloud_cover"
+    client = app_config["client"]
 
     with mock.patch.dict(
         os.environ,
@@ -81,7 +101,7 @@ async def test_validate_queryables_excluded(app_client, ctx):
             "QUERYABLES_CACHE_TTL": "0",
         },
     ):
-        reload_queryables_settings()
+        client.queryables_cache.reload_settings()
 
         query = {"query": {excluded_field: {"lt": 10}}}
         resp = await app_client.post("/search", json=query)
@@ -93,4 +113,4 @@ async def test_validate_queryables_excluded(app_client, ctx):
         resp = await app_client.post("/search", json=query)
         assert resp.status_code == 200
 
-    reload_queryables_settings()
+    client.queryables_cache.reload_settings()

…i/tests/sfeos_helpers/test_queryables.py …ac_fastapi/tests/core/test_queryables.pystac_fastapi/tests/sfeos_helpers/test_queryables.py renamed to stac_fastapi/tests/core/test_queryables.py stac_fastapi/tests/sfeos_helpers/test_queryables.py renamed to stac_fastapi/tests/core/test_queryables.py

@@ -5,14 +5,9 @@
 import pytest
 from fastapi import HTTPException
 
-import stac_fastapi.sfeos_helpers.queryables as queryables_module
-from stac_fastapi.sfeos_helpers.queryables import (
+from stac_fastapi.core.queryables import (
     QueryablesCache,
-    all_queryables,
     get_properties_from_cql2_filter,
-    initialize_queryables_cache,
-    reload_queryables_settings,
-    validate_queryables,
 )
 
 
@@ -102,61 +97,6 @@ async def test_validate_disabled(self, queryables_cache):
         await queryables_cache.validate({"invalid_prop"})
 
 
-class TestGlobalFunctions:
-    @pytest.fixture(autouse=True)
-    def reset_global_cache(self):
-        original = queryables_module._queryables_cache_instance
-        queryables_module._queryables_cache_instance = None
-        yield
-
-        queryables_module._queryables_cache_instance = original
-
-    def test_initialize_queryables_cache(self):
-        db_logic = MagicMock()
-        initialize_queryables_cache(db_logic)
-        assert queryables_module._queryables_cache_instance is not None
-        assert queryables_module._queryables_cache_instance._db_logic == db_logic
-
-    @pytest.mark.asyncio
-    async def test_all_queryables_not_initialized(self):
-        with pytest.raises(Exception) as excinfo:
-            await all_queryables()
-        assert "Queryables cache not initialized" in str(excinfo.value)
-
-    @pytest.mark.asyncio
-    async def test_all_queryables(self):
-        db_logic = MagicMock()
-        db_logic.get_queryables_mapping = AsyncMock(return_value={"p1": "t1"})
-        initialize_queryables_cache(db_logic)
-
-        queryables_module._queryables_cache_instance.validation_enabled = True
-
-        res = await all_queryables()
-        assert res == {"p1"}
-
-    @pytest.mark.asyncio
-    async def test_validate_queryables(self):
-        db_logic = MagicMock()
-        db_logic.get_queryables_mapping = AsyncMock(return_value={"p1": "t1"})
-        initialize_queryables_cache(db_logic)
-        queryables_module._queryables_cache_instance.validation_enabled = True
-
-        await validate_queryables({"p1"})
-
-        with pytest.raises(HTTPException):
-            await validate_queryables({"invalid"})
-
-    def test_reload_queryables_settings(self):
-        db_logic = MagicMock()
-        initialize_queryables_cache(db_logic)
-
-        with patch.dict(os.environ, {"VALIDATE_QUERYABLES": "false"}):
-            reload_queryables_settings()
-            assert (
-                queryables_module._queryables_cache_instance.validation_enabled is False
-            )
-
-
 def test_get_properties_from_cql2_filter():
     # Simple prop
     cql2 = {"op": "=", "args": [{"property": "prop1"}, "value"]}