From 01274459fa7374e164305ba6a848c0cdfddceaf8 Mon Sep 17 00:00:00 2001 From: vincentsarago Date: Thu, 18 Sep 2025 18:15:59 +0200 Subject: [PATCH 01/10] switch to cql2 --- CHANGES.md | 2 ++ pyproject.toml | 2 +- tipg/collections.py | 23 ++++++++++++++--------- tipg/dependencies.py | 11 +++++------ 4 files changed, 22 insertions(+), 16 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 13590ab1..fdbe026e 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -8,6 +8,8 @@ Note: Minor version `0.X.0` update might break the API, It's recommended to pin ## [unreleased] +* switch from pygeofilter to cql2 + ## [1.3.1] - 2026-02-26 * fix: bbox filter when collection's geometry is not in EPSG:4326 CRS diff --git a/pyproject.toml b/pyproject.toml index bb71acca..409f74d7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,6 +22,7 @@ classifiers = [ dynamic = ["version"] dependencies = [ "orjson", + "cql2", "asyncpg>=0.23.0", "buildpg>=0.3", "fastapi>=0.100.0", @@ -30,7 +31,6 @@ dependencies = [ "pydantic>=2.4,<3.0", "pydantic-settings~=2.0", "geojson-pydantic>=1.0,<3.0", - "pygeofilter>=0.2.0,<0.3.0", "ciso8601~=2.3", "starlette-cramjam>=0.4,<0.6", ] diff --git a/tipg/collections.py b/tipg/collections.py index 054f2cf5..8b8f862d 100644 --- a/tipg/collections.py +++ b/tipg/collections.py @@ -11,9 +11,10 @@ from buildpg import funcs as pg_funcs from buildpg import logic, render from ciso8601 import parse_rfc3339 +from cql2 import Expr +from geojson_pydantic.geometries import Polygon from morecantile import Tile, TileMatrixSet from pydantic import BaseModel, Field, model_validator -from pygeofilter.ast import AstType from pyproj import Transformer from tipg.errors import ( @@ -24,8 +25,6 @@ InvalidPropertyName, MissingDatetimeColumn, ) -from tipg.filter.evaluate import to_filter -from tipg.filter.filters import bbox_to_wkt from tipg.logger import logger from tipg.model import Extent from tipg.settings import ( @@ -44,6 +43,12 @@ TransformerFromCRS = lru_cache(Transformer.from_crs) +def bbox_to_wkt(bbox: List[float], srid: int = 4326) -> str: + """Return WKT representation of a BBOX.""" + poly = Polygon.from_bounds(*bbox) # type:ignore + return f"SRID={srid};{poly.wkt}" + + def debug_query(q, *p): """Utility to print raw statement to use for debugging.""" @@ -522,7 +527,7 @@ def _where( # noqa: C901 datetime: Optional[List[str]] = None, bbox: Optional[List[float]] = None, properties: Optional[List[Tuple[str, Any]]] = None, - cql: Optional[AstType] = None, + cql: Optional[Expr] = None, geom: Optional[str] = None, dt: Optional[str] = None, tile: Optional[Tile] = None, @@ -594,7 +599,7 @@ def _where( # noqa: C901 # `CQL` filter if cql is not None: - wheres.append(to_filter(cql, [p.name for p in self.properties])) + wheres.append(cql.to_sql()) if tile and tms and geometry_column: # Get tile bounds in the TMS coordinate system @@ -698,7 +703,7 @@ async def _features_query( bbox_filter: Optional[List[float]] = None, datetime_filter: Optional[List[str]] = None, properties_filter: Optional[List[Tuple[str, str]]] = None, - cql_filter: Optional[AstType] = None, + cql_filter: Optional[Expr] = None, sortby: Optional[str] = None, properties: Optional[List[str]] = None, geom: Optional[str] = None, @@ -753,7 +758,7 @@ async def _features_count_query( bbox_filter: Optional[List[float]] = None, datetime_filter: Optional[List[str]] = None, properties_filter: Optional[List[Tuple[str, str]]] = None, - cql_filter: Optional[AstType] = None, + cql_filter: Optional[Expr] = None, geom: Optional[str] = None, dt: Optional[str] = None, function_parameters: Optional[Dict[str, str]], @@ -785,7 +790,7 @@ async def features( bbox_filter: Optional[List[float]] = None, datetime_filter: Optional[List[str]] = None, properties_filter: Optional[List[Tuple[str, str]]] = None, - cql_filter: Optional[AstType] = None, + cql_filter: Optional[Expr] = None, sortby: Optional[str] = None, properties: Optional[List[str]] = None, geom: Optional[str] = None, @@ -864,7 +869,7 @@ async def get_tile( datetime_filter: Optional[List[str]] = None, properties_filter: Optional[List[Tuple[str, str]]] = None, function_parameters: Optional[Dict[str, str]] = None, - cql_filter: Optional[AstType] = None, + cql_filter: Optional[Expr] = None, sortby: Optional[str] = None, properties: Optional[List[str]] = None, geom: Optional[str] = None, diff --git a/tipg/dependencies.py b/tipg/dependencies.py index a82feeb7..cce31768 100644 --- a/tipg/dependencies.py +++ b/tipg/dependencies.py @@ -3,12 +3,11 @@ import re from typing import Annotated, Dict, List, Literal, Optional, Tuple, get_args +import orjson from ciso8601 import parse_rfc3339 +from cql2 import Expr from morecantile import Tile from morecantile import tms as default_tms -from pygeofilter.ast import AstType -from pygeofilter.parsers.cql2_json import parse as cql2_json_parser -from pygeofilter.parsers.cql2_text import parse as cql2_text_parser from tipg.collections import Catalog, Collection, CollectionList from tipg.errors import InvalidBBox, MissingCollectionCatalog, MissingFunctionParameter @@ -289,14 +288,14 @@ def filter_query( alias="filter-lang", ), ] = None, -) -> Optional[AstType]: +) -> Optional[Dict]: """Parse Filter Query.""" if query is not None: if filter_lang == "cql2-json": - return cql2_json_parser(query) + return Expr(orjson.loads(query)) # default to cql2-text - return cql2_text_parser(query) + return Expr(query) return None From 53ce692dd957050f9f0a8f47ea5f2828f96df82e Mon Sep 17 00:00:00 2001 From: vincentsarago Date: Tue, 23 Sep 2025 11:13:30 -0600 Subject: [PATCH 02/10] update --- tipg/dependencies.py | 2 +- tipg/factory.py | 6 +- tipg/filter/__init__.py | 1 - tipg/filter/evaluate.py | 151 ------------------- tipg/filter/filters.py | 325 ---------------------------------------- 5 files changed, 4 insertions(+), 481 deletions(-) delete mode 100644 tipg/filter/__init__.py delete mode 100644 tipg/filter/evaluate.py delete mode 100644 tipg/filter/filters.py diff --git a/tipg/dependencies.py b/tipg/dependencies.py index cce31768..a3919616 100644 --- a/tipg/dependencies.py +++ b/tipg/dependencies.py @@ -288,7 +288,7 @@ def filter_query( alias="filter-lang", ), ] = None, -) -> Optional[Dict]: +) -> Optional[Expr]: """Parse Filter Query.""" if query is not None: if filter_lang == "cql2-json": diff --git a/tipg/factory.py b/tipg/factory.py index 3959ffee..9cf86fbe 100644 --- a/tipg/factory.py +++ b/tipg/factory.py @@ -19,10 +19,10 @@ import jinja2 import orjson +from cql2 import Expr from morecantile import Tile, TileMatrixSet from morecantile import tms as default_tms from morecantile.defaults import TileMatrixSets -from pygeofilter.ast import AstType from tipg import model from tipg.collections import Collection, CollectionList @@ -794,7 +794,7 @@ async def items( # noqa: C901 bbox_filter: Annotated[Optional[List[float]], Depends(bbox_query)], datetime_filter: Annotated[Optional[List[str]], Depends(datetime_query)], properties: Annotated[Optional[List[str]], Depends(properties_query)], - cql_filter: Annotated[Optional[AstType], Depends(filter_query)], + cql_filter: Annotated[Optional[Expr], Depends(filter_query)], sortby: Annotated[Optional[str], Depends(sortby_query)], geom_column: Annotated[ Optional[str], @@ -1646,7 +1646,7 @@ async def collection_get_tile( properties: Annotated[ Optional[List[str]], Depends(properties_query) ] = None, - cql_filter: Annotated[Optional[AstType], Depends(filter_query)] = None, + cql_filter: Annotated[Optional[Expr], Depends(filter_query)] = None, sortby: Annotated[Optional[str], Depends(sortby_query)] = None, geom_column: Annotated[ Optional[str], diff --git a/tipg/filter/__init__.py b/tipg/filter/__init__.py deleted file mode 100644 index 98f1bebd..00000000 --- a/tipg/filter/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""tipg.filter""" diff --git a/tipg/filter/evaluate.py b/tipg/filter/evaluate.py deleted file mode 100644 index 5852ec25..00000000 --- a/tipg/filter/evaluate.py +++ /dev/null @@ -1,151 +0,0 @@ -"""tipg.filter.evaluate.""" - -from datetime import date, datetime, time, timedelta - -from pygeofilter import ast, values -from pygeofilter.backends.evaluator import Evaluator, handle - -from tipg.filter import filters - -LITERALS = (str, float, int, bool, datetime, date, time, timedelta) - - -class BuildPGEvaluator(Evaluator): # noqa: D101 - def __init__(self, field_mapping): # noqa: D107 - self.field_mapping = field_mapping - - @handle(ast.Not) - def not_(self, node, sub): # noqa: D102 - return filters.negate(sub) - - @handle(ast.And, ast.Or) - def combination(self, node, lhs, rhs): # noqa: D102 - return filters.combine((lhs, rhs), node.op.value) - - @handle(ast.Comparison, subclasses=True) - def comparison(self, node, lhs, rhs): # noqa: D102 - return filters.runop( - lhs, - rhs, - node.op.value, - ) - - @handle(ast.Between) - def between(self, node, lhs, low, high): # noqa: D102 - return filters.between(lhs, low, high, node.not_) - - @handle(ast.Like) - def like(self, node, lhs): # noqa: D102 - return filters.like( - lhs, - node.pattern, - not node.nocase, - node.not_, - ) - - @handle(ast.In) - def in_(self, node, lhs, *options): # noqa: D102 - return filters.runop( - lhs, - options, - "in", - node.not_, - ) - - @handle(ast.IsNull) - def null(self, node, lhs): # noqa: D102 - if isinstance(lhs, list): - lhs = filters.attribute(lhs[0].name, self.field_mapping) - return filters.isnull(lhs) - - # @handle(ast.ExistsPredicateNode) - # def exists(self, node, lhs): - # if self.use_getattr: - # result = hasattr(self.obj, node.lhs.name) - # else: - # result = lhs in self.obj - - # if node.not_: - # result = not result - # return result - - @handle(ast.TemporalPredicate, subclasses=True) - def temporal(self, node, lhs, rhs): # noqa: D102 - return filters.temporal( - lhs, - rhs, - node.op.value, - ) - - @handle(ast.SpatialComparisonPredicate, subclasses=True) - def spatial_operation(self, node, lhs, rhs): # noqa: D102 - return filters.spatial( - lhs, - rhs, - node.op.name, - ) - - @handle(ast.Relate) - def spatial_pattern(self, node, lhs, rhs): # noqa: D102 - return filters.spatial( - lhs, - rhs, - "RELATE", - pattern=node.pattern, - ) - - @handle(ast.SpatialDistancePredicate, subclasses=True) - def spatial_distance(self, node, lhs, rhs): # noqa: D102 - return filters.spatial( - lhs, - rhs, - node.op.value, - distance=node.distance, - units=node.units, - ) - - @handle(ast.BBox) - def bbox(self, node, lhs): # noqa: D102 - return filters.bbox(lhs, node.minx, node.miny, node.maxx, node.maxy, node.crs) - - @handle(ast.Attribute) - def attribute(self, node): # noqa: D102 - return filters.attribute(node.name, self.field_mapping) - - @handle(ast.Arithmetic, subclasses=True) - def arithmetic(self, node, lhs, rhs): # noqa: D102 - return filters.runop(lhs, rhs, node.op.value) - - @handle(ast.Function) - def function(self, node, *arguments): # noqa: D102 - return filters.func(node.name, *arguments) - - @handle(*values.LITERALS) - def literal(self, node): # noqa: D102 - return filters.literal(node) - - @handle(values.Interval) - def interval(self, node, start, end): # noqa: D102 - return filters.literal((start, end)) - - @handle(values.Geometry) - def geometry(self, node): # noqa: D102 - return filters.parse_geometry(node.__geo_interface__) - - @handle(values.Envelope) - def envelope(self, node): # noqa: D102 - return filters.parse_bbox([node.x1, node.y1, node.x2, node.y2]) - - -def to_filter(ast, field_mapping=None): # noqa: D102 - """Helper function to translate ECQL AST to Django Query expressions. - - :param ast: the abstract syntax tree - :param field_mapping: a dict mapping from the filter name to the Django field lookup. - :param mapping_choices: a dict mapping field lookups to choices. - :type ast: :class:`Node` - :returns: a Django query object - :rtype: :class:`django.db.models.Q` - - """ - return BuildPGEvaluator(field_mapping).evaluate(ast) diff --git a/tipg/filter/filters.py b/tipg/filter/filters.py deleted file mode 100644 index 2e306e8e..00000000 --- a/tipg/filter/filters.py +++ /dev/null @@ -1,325 +0,0 @@ -"""tipg.filter.filters""" - -import re -from datetime import timedelta -from functools import reduce -from inspect import signature -from typing import Any, Callable, Dict, List - -from buildpg import V -from buildpg.funcs import AND as and_ -from buildpg.funcs import NOT as not_ -from buildpg.funcs import OR as or_ -from buildpg.funcs import any -from buildpg.logic import Func -from geojson_pydantic.geometries import Polygon, parse_geometry_obj - - -def bbox_to_wkt(bbox: List[float], srid: int = 4326) -> str: - """Return WKT representation of a BBOX.""" - poly = Polygon.from_bounds(*bbox) # type:ignore - return f"SRID={srid};{poly.wkt}" - - -def parse_geometry(geom: Dict[str, Any]) -> str: - """Parse geometry object and return WKT.""" - wkt = parse_geometry_obj(geom).wkt # type:ignore - sridtxt = "" if wkt.startswith("SRID=") else "SRID=4326;" - return f"{sridtxt}{wkt}" - - -# ------------------------------------------------------------------------------ -# Filters -# ------------------------------------------------------------------------------ -class Operator: - """Filter Operators.""" - - OPERATORS: Dict[str, Callable] = { - "==": lambda f, a: f == a, - "=": lambda f, a: f == a, - "eq": lambda f, a: f == a, - "!=": lambda f, a: f != a, - "<>": lambda f, a: f != a, - "ne": lambda f, a: f != a, - ">": lambda f, a: f > a, - "gt": lambda f, a: f > a, - "<": lambda f, a: f < a, - "lt": lambda f, a: f < a, - ">=": lambda f, a: f >= a, - "ge": lambda f, a: f >= a, - "<=": lambda f, a: f <= a, - "le": lambda f, a: f <= a, - "like": lambda f, a: f.like(a), - "ilike": lambda f, a: f.ilike(a), - "not_ilike": lambda f, a: ~f.ilike(a), - "in": lambda f, a: f == any(a), - "not_in": lambda f, a: ~f == any(a), - "any": lambda f, a: f.any(a), - "not_any": lambda f, a: f.not_(f.any(a)), - "INTERSECTS": lambda f, a: Func( - "st_intersects", - f, - Func("st_transform", a, Func("st_srid", f)), - ), - "DISJOINT": lambda f, a: Func( - "st_disjoint", f, Func("st_transform", a, Func("st_srid", f)) - ), - "CONTAINS": lambda f, a: Func( - "st_contains", f, Func("st_transform", a, Func("st_srid", f)) - ), - "WITHIN": lambda f, a: Func( - "st_within", f, Func("st_transform", a, Func("st_srid", f)) - ), - "TOUCHES": lambda f, a: Func( - "st_touches", f, Func("st_transform", a, Func("st_srid", f)) - ), - "CROSSES": lambda f, a: Func( - "st_crosses", - f, - Func("st_transform", a, Func("st_srid", f)), - ), - "OVERLAPS": lambda f, a: Func( - "st_overlaps", - f, - Func("st_transform", a, Func("st_srid", f)), - ), - "EQUALS": lambda f, a: Func( - "st_equals", - f, - Func("st_transform", a, Func("st_srid", f)), - ), - "RELATE": lambda f, a, pattern: Func( - "st_relate", f, Func("st_transform", a, Func("st_srid", f)), pattern - ), - "DWITHIN": lambda f, a, distance: Func( - "st_dwithin", f, Func("st_transform", a, Func("st_srid", f)), distance - ), - "BEYOND": lambda f, a, distance: ~Func( - "st_dwithin", f, Func("st_transform", a, Func("st_srid", f)), distance - ), - "+": lambda f, a: f + a, - "-": lambda f, a: f - a, - "*": lambda f, a: f * a, - "/": lambda f, a: f / a, - } - - def __init__(self, operator: str = None): - """Init.""" - if not operator: - operator = "==" - - if operator not in self.OPERATORS: - raise Exception("Operator `{}` not valid.".format(operator)) - - self.operator = operator - self.function = self.OPERATORS[operator] - self.arity = len(signature(self.function).parameters) - - -def func(name, *args): - """Return results of running SQL function with arguments.""" - return Func(name, *args) - - -def combine(sub_filters, combinator: str = "AND"): - """Combine filters using a logical combinator - - :param sub_filters: the filters to combine - :param combinator: a string: "AND" / "OR" - :return: the combined filter - - """ - assert combinator in ("AND", "OR") - _op = and_ if combinator == "AND" else or_ - - def test(acc, q): - return _op(acc, q) - - return reduce(test, sub_filters) - - -def negate(sub_filter): - """Negate a filter, opposing its meaning. - - :param sub_filter: the filter to negate - :return: the negated filter - - """ - return not_(sub_filter) - - -def runop(lhs, rhs=None, op: str = "=", negate: bool = False): - """Compare a filter with an expression using a comparison operation. - - :param lhs: the field to compare - :param rhs: the filter expression - :param op: a string denoting the operation. - :return: a comparison expression object - - """ - _op = Operator(op) - - if negate: - return not_(_op.function(lhs, rhs)) - return _op.function(lhs, rhs) - - -def between(lhs, low, high, negate=False): - """Create a filter to match elements that have a value within a certain range. - - :param lhs: the field to compare - :param low: the lower value of the range - :param high: the upper value of the range - :param not_: whether the range shall be inclusive (the default) or exclusive - :return: a comparison expression object - - """ - l_op = Operator("<=") - g_op = Operator(">=") - if negate: - return not_(and_(g_op.function(lhs, low), l_op.function(lhs, high))) - - return and_(g_op.function(lhs, low), l_op.function(lhs, high)) - - -def like(lhs, rhs, case=False, negate=False): - """Create a filter to filter elements according to a string attribute using wildcard expressions. - - :param lhs: the field to compare - :param rhs: the wildcard pattern: a string containing any number of '%' characters as wildcards. - :param case: whether the lookup shall be done case sensitively or not - :param not_: whether the range shall be inclusive (the default) or exclusive - :return: a comparison expression object - - """ - if case: - _op = Operator("like") - else: - _op = Operator("ilike") - - if negate: - return not_(_op.function(lhs, rhs)) - - return _op.function(lhs, rhs) - - -def temporal(lhs, time_or_period, op): - """Create a temporal filter for the given temporal attribute. - - :param lhs: the field to compare - :type lhs: :class:`django.db.models.F` - :param time_or_period: the time instant or time span to use as a filter - :type time_or_period: :class:`datetime.datetime` or a tuple of two datetimes or a tuple of one datetime and one :class:`datetime.timedelta` - :param op: the comparison operation. one of ``"BEFORE"``, ``"BEFORE OR DURING"``, ``"DURING"``, ``"DURING OR AFTER"``, ``"AFTER"``. - :type op: str - :return: a comparison expression object - :rtype: :class:`django.db.models.Q` - - """ - low = None - high = None - equal = None - if op in ("BEFORE", "AFTER"): - if op == "BEFORE": - high = time_or_period - else: - low = time_or_period - elif op == "TEQUALS": - equal = time_or_period - else: - low, high = time_or_period - - if isinstance(low, timedelta): - low = high - low - if isinstance(high, timedelta): - high = low + high - if low is not None or high is not None: - if low is not None and high is not None: - return between(lhs, low, high) - elif low is not None: - return runop(lhs, low, ">=") - else: - return runop(lhs, high, "<=") - elif equal is not None: - return runop(lhs, equal, "==") - - -UNITS_LOOKUP = {"kilometers": "km", "meters": "m"} - - -def spatial(lhs, rhs, op, pattern=None, distance=None, units=None): - """Create a spatial filter for the given spatial attribute. - - :param lhs: the field to compare - :param rhs: the time instant or time span to use as a filter - :param op: the comparison operation. one of ``"INTERSECTS"``, ``"DISJOINT"``, `"CONTAINS"``, ``"WITHIN"``, ``"TOUCHES"``, ``"CROSSES"``, ``"OVERLAPS"``, ``"EQUALS"``, ``"RELATE"``, ``"DWITHIN"``, ``"BEYOND"`` - :param pattern: the spatial relation pattern - :param distance: the distance value for distance based lookups: ``"DWITHIN"`` and ``"BEYOND"`` - :param units: the units the distance is expressed in - :return: a comparison expression object - - """ - - _op = Operator(op) - if op == "RELATE": - return _op.function(lhs, rhs, pattern) - elif op in ("DWITHIN", "BEYOND"): - if units == "kilometers": - distance = distance / 1000 - elif units == "miles": - distance = distance / 1609 - return _op.function(lhs, rhs, distance) - else: - return _op.function(lhs, rhs) - - -def bbox(lhs, minx, miny, maxx, maxy, crs: int = 4326): - """Create a bounding box filter for the given spatial attribute. - - :param lhs: the field to compare - :param minx: the lower x part of the bbox - :param miny: the lower y part of the bbox - :param maxx: the upper x part of the bbox - :param maxy: the upper y part of the bbox - :param crs: the CRS the bbox is expressed in - :return: a comparison expression object - - """ - - return Func("st_intersects", lhs, bbox_to_wkt([minx, miny, maxx, maxy], crs)) - - -def quote_ident(s: str) -> str: - """quote.""" - if re.match(r"^[a-z]+$", s): - return s - if re.match(r"^[a-zA-Z][\w\d_]*$", s): - return f'"{s}"' - raise TypeError(f"{s} is not a valid identifier") - - -def attribute(name: str, fields: List[str]): - """Create an attribute lookup expression using a field mapping dictionary. - - :param name: the field filter name - :param field_mapping: the dictionary to use as a lookup. - - """ - if name in fields: - return V(name) - elif name.lower() == "true": - return True - elif name.lower() == "false": - return False - else: - raise TypeError(f"Field {name} not in table.") - - -def isnull(lhs): - """null value.""" - return lhs.is_(V("NULL")) - - -def literal(value): - """literal value.""" - return value From 48b3b07c6c5b4d4dde9376755345b756f9af6256 Mon Sep 17 00:00:00 2001 From: vincentsarago Date: Tue, 23 Sep 2025 11:17:12 -0600 Subject: [PATCH 03/10] no need to parse json --- tipg/dependencies.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tipg/dependencies.py b/tipg/dependencies.py index a3919616..70fbfe48 100644 --- a/tipg/dependencies.py +++ b/tipg/dependencies.py @@ -3,7 +3,6 @@ import re from typing import Annotated, Dict, List, Literal, Optional, Tuple, get_args -import orjson from ciso8601 import parse_rfc3339 from cql2 import Expr from morecantile import Tile @@ -291,10 +290,6 @@ def filter_query( ) -> Optional[Expr]: """Parse Filter Query.""" if query is not None: - if filter_lang == "cql2-json": - return Expr(orjson.loads(query)) - - # default to cql2-text return Expr(query) return None From fba7b4ec1375c15135d9197b6bd74f72f4774aba Mon Sep 17 00:00:00 2001 From: vincentsarago Date: Tue, 23 Sep 2025 16:24:28 -0600 Subject: [PATCH 04/10] all filters in cql --- tipg/collections.py | 369 +++++++++++++++++++------------------------- tipg/factory.py | 2 +- 2 files changed, 164 insertions(+), 207 deletions(-) diff --git a/tipg/collections.py b/tipg/collections.py index 8b8f862d..8edc00d9 100644 --- a/tipg/collections.py +++ b/tipg/collections.py @@ -3,7 +3,7 @@ import abc import datetime import re -from functools import lru_cache +from functools import lru_cache, reduce from typing import Any, Dict, List, Optional, Tuple, TypedDict, Union from buildpg import RawDangerous as raw @@ -328,6 +328,139 @@ def queryables(self) -> Dict: return {**geoms, **props} + def cql_where( # noqa: C901 + self, + ids: Optional[List[str]] = None, + datetime: Optional[List[str]] = None, + bbox: Optional[List[float]] = None, + properties: Optional[List[Tuple[str, Any]]] = None, + cql: Optional[Expr] = None, + geom: Optional[str] = None, + dt: Optional[str] = None, + tile: Optional[Tile] = None, + tms: Optional[TileMatrixSet] = None, + ) -> Expr: + """Construct WHERE query.""" + exprs = [] + + if cql: + exprs.append(cql) + + # `ids` filter + if ids: + # REF: https://github.com/developmentseed/cql2-rs/issues/91 + if len(ids) == 1: + exprs.append(Expr(f"{self.id_column.name} = {ids[0]}")) + else: + id_list = ", ".join(f"'{id_}'" for id_ in ids) + exprs.append(Expr(f"{self.id_column.name} IN {id_list}")) + + # `properties` filter + if properties is not None: + for prop, val in properties: + col = self.get_column(prop) + if not col: + raise InvalidPropertyName(f"Invalid property name: {prop}") + exprs.append(Expr(f"{prop}='{val}'")) + + # `bbox` filter + geometry_column = self.get_geometry_column(geom) + if bbox is not None and geometry_column is not None: + # TODO: should we use bbox_to_wkt(bbox) + exprs.append( + Expr( + f"S_INTERSECTS({geometry_column.name}, {', '.join(map(str, bbox))})" + ) + ) + print(exprs[0].reduce().to_sql()) + + # `datetime` filter + if datetime: + if not self.datetime_columns: + raise MissingDatetimeColumn( + "Must have timestamp/timestamptz/date typed column to filter with datetime." + ) + + datetime_column = self.get_datetime_column(dt) + if not datetime_column: + raise InvalidDatetimeColumnName(f"Invalid Datetime Column: {dt}.") + + if len(datetime) == 1: + # NOTE: should we do parse_rfc3339(datetime[0]) + exprs.append(Expr(f"{datetime_column.name}=TIMESTAMP('{datetime[0]}')")) + + else: + start = ( + parse_rfc3339(datetime[0]) + if datetime[0] not in ["..", ""] + else None + ) + end = ( + parse_rfc3339(datetime[1]) + if datetime[1] not in ["..", ""] + else None + ) + if start is None and end is None: + raise InvalidDatetime( + "Double open-ended datetime intervals are not allowed." + ) + + if start is not None and end is not None and start > end: + raise InvalidDatetime( + "Start datetime cannot be before end datetime." + ) + + if (start and end) and start > end: + raise ValueError("Invalid datetime range: start must be <= end") + + startstr, endstr = datetime[:2] + if startstr not in ["..", ""]: + exprs.append( + Expr(f"{datetime_column.name}>=TIMESTAMP('{startstr}')") + ) + if endstr: + exprs.append(Expr(f"{datetime_column.name}<=TIMESTAMP('{endstr}')")) + + # if tile and tms and geometry_column: + # # Get tile bounds in the TMS coordinate system + # bbox = tms.xy_bounds(tile) + # left, bottom, right, top = bbox + + # # If the geometry column’s SRID does not match the TMS CRS, transform the bounds: + # # Use a fallback of 4326 if tms.crs.to_epsg() returns a falsey value. + # tms_epsg = tms.crs.to_epsg() or 4326 + # if geometry_column.srid != tms_epsg: + # transformer = TransformerFromCRS( + # tms_epsg, geometry_column.srid, always_xy=True + # ) + + # left, bottom, right, top = transformer.transform_bounds( + # left, bottom, right, top + # ) + + # wheres.append( + # logic.Func( + # "ST_Intersects", + # logic.Func( + # "ST_Segmentize", + # logic.Func( + # "ST_MakeEnvelope", + # left, + # bottom, + # right, + # top, + # geometry_column.srid, + # ), + # right - left, + # ), + # logic.V(geometry_column.name), + # ) + # ) + if exprs: + return reduce(lambda x, y: x + y, exprs).reduce() + + return None + @abc.abstractmethod async def features(self, *args, **kwargs) -> ItemList: """Get Items.""" @@ -521,156 +654,6 @@ def _geom( return g - def _where( # noqa: C901 - self, - ids: Optional[List[str]] = None, - datetime: Optional[List[str]] = None, - bbox: Optional[List[float]] = None, - properties: Optional[List[Tuple[str, Any]]] = None, - cql: Optional[Expr] = None, - geom: Optional[str] = None, - dt: Optional[str] = None, - tile: Optional[Tile] = None, - tms: Optional[TileMatrixSet] = None, - ): - """Construct WHERE query.""" - wheres = [logic.S(True)] - - # `ids` filter - if ids is not None: - if len(ids) == 1: - wheres.append( - logic.V(self.id_column.name) - == pg_funcs.cast(pg_funcs.cast(ids[0], "text"), self.id_column.type) - ) - else: - w = [ - logic.V(self.id_column.name) - == logic.S( - pg_funcs.cast(pg_funcs.cast(i, "text"), self.id_column.type) - ) - for i in ids - ] - wheres.append(pg_funcs.OR(*w)) - - # `properties filter - if properties is not None: - w = [] - for prop, val in properties: - col = self.get_column(prop) - if not col: - raise InvalidPropertyName(f"Invalid property name: {prop}") - - w.append( - logic.V(col.name) - == logic.S(pg_funcs.cast(pg_funcs.cast(val, "text"), col.type)) - ) - - if w: - wheres.append(pg_funcs.AND(*w)) - - # `bbox` filter - geometry_column = self.get_geometry_column(geom) - if bbox is not None and geometry_column is not None: - wheres.append( - logic.Func( - "ST_Intersects", - logic.V(geometry_column.name), - logic.Func( - "ST_Transform", - logic.S(bbox_to_wkt(bbox)), - logic.Func("ST_SRID", logic.V(geometry_column.name)), - ), - ) - ) - - # `datetime` filter - if datetime: - if not self.datetime_columns: - raise MissingDatetimeColumn( - "Must have timestamp/timestamptz/date typed column to filter with datetime." - ) - - datetime_column = self.get_datetime_column(dt) - if not datetime_column: - raise InvalidDatetimeColumnName(f"Invalid Datetime Column: {dt}.") - - wheres.append(self._datetime_filter_to_sql(datetime, datetime_column.name)) - - # `CQL` filter - if cql is not None: - wheres.append(cql.to_sql()) - - if tile and tms and geometry_column: - # Get tile bounds in the TMS coordinate system - bbox = tms.xy_bounds(tile) - left, bottom, right, top = bbox - - # If the geometry column’s SRID does not match the TMS CRS, transform the bounds: - # Use a fallback of 4326 if tms.crs.to_epsg() returns a falsey value. - tms_epsg = tms.crs.to_epsg() or 4326 - if geometry_column.srid != tms_epsg: - transformer = TransformerFromCRS( - tms_epsg, geometry_column.srid, always_xy=True - ) - - left, bottom, right, top = transformer.transform_bounds( - left, bottom, right, top - ) - - wheres.append( - logic.Func( - "ST_Intersects", - logic.Func( - "ST_Segmentize", - logic.Func( - "ST_MakeEnvelope", - left, - bottom, - right, - top, - geometry_column.srid, - ), - right - left, - ), - logic.V(geometry_column.name), - ) - ) - - return clauses.Where(pg_funcs.AND(*wheres)) - - def _datetime_filter_to_sql(self, interval: List[str], dt_name: str): - if len(interval) == 1: - return logic.V(dt_name) == logic.S( - pg_funcs.cast(parse_rfc3339(interval[0]), "timestamptz") - ) - - else: - start = ( - parse_rfc3339(interval[0]) if interval[0] not in ["..", ""] else None - ) - end = parse_rfc3339(interval[1]) if interval[1] not in ["..", ""] else None - - if start is None and end is None: - raise InvalidDatetime( - "Double open-ended datetime intervals are not allowed." - ) - - if start is not None and end is not None and start > end: - raise InvalidDatetime("Start datetime cannot be before end datetime.") - - if not start: - return logic.V(dt_name) <= logic.S(pg_funcs.cast(end, "timestamptz")) - - elif not end: - return logic.V(dt_name) >= logic.S(pg_funcs.cast(start, "timestamptz")) - - else: - return pg_funcs.AND( - logic.V(dt_name) >= logic.S(pg_funcs.cast(start, "timestamptz")), - logic.V(dt_name) < logic.S(pg_funcs.cast(end, "timestamptz")), - ) - def _sortby(self, sortby: Optional[str]): sorts = [] if sortby: @@ -699,15 +682,10 @@ async def _features_query( self, conn: asyncpg.Connection, *, - ids_filter: Optional[List[str]] = None, - bbox_filter: Optional[List[float]] = None, - datetime_filter: Optional[List[str]] = None, - properties_filter: Optional[List[Tuple[str, str]]] = None, - cql_filter: Optional[Expr] = None, + where: Optional[str] = None, sortby: Optional[str] = None, properties: Optional[List[str]] = None, geom: Optional[str] = None, - dt: Optional[str] = None, limit: Optional[int] = None, offset: Optional[int] = None, bbox_only: Optional[bool] = None, @@ -728,15 +706,7 @@ async def _features_query( geom_as_wkt=geom_as_wkt, ), self._from(function_parameters), - self._where( - ids=ids_filter, - datetime=datetime_filter, - bbox=bbox_filter, - properties=properties_filter, - cql=cql_filter, - geom=geom, - dt=dt, - ), + clauses.Where(where or logic.S(True)), self._sortby(sortby), clauses.Limit(limit), clauses.Offset(offset), @@ -754,28 +724,14 @@ async def _features_count_query( self, conn: asyncpg.Connection, *, - ids_filter: Optional[List[str]] = None, - bbox_filter: Optional[List[float]] = None, - datetime_filter: Optional[List[str]] = None, - properties_filter: Optional[List[Tuple[str, str]]] = None, - cql_filter: Optional[Expr] = None, - geom: Optional[str] = None, - dt: Optional[str] = None, + where: Optional[str], function_parameters: Optional[Dict[str, str]], ) -> int: """Build features COUNT query.""" c = clauses.Clauses( self._select_count(), self._from(function_parameters), - self._where( - ids=ids_filter, - datetime=datetime_filter, - bbox=bbox_filter, - properties=properties_filter, - cql=cql_filter, - geom=geom, - dt=dt, - ), + clauses.Where(where or logic.S(True)), ) q, p = render(":c", c=c) @@ -816,31 +772,30 @@ async def features( f"Limit can not be set higher than the `tipg_max_features_per_query` setting of {features_settings.max_features_per_query}" ) + where_filter = self.cql_where( + ids=ids_filter, + datetime=datetime_filter, + bbox=bbox_filter, + properties=properties_filter, + cql=cql_filter, + geom=geom, + dt=dt, + ) + matched = await self._features_count_query( conn, - ids_filter=ids_filter, - datetime_filter=datetime_filter, - bbox_filter=bbox_filter, - properties_filter=properties_filter, + where=where_filter.to_sql() if where_filter else None, function_parameters=function_parameters, - cql_filter=cql_filter, - geom=geom, - dt=dt, ) features = [ f async for f in self._features_query( conn, - ids_filter=ids_filter, - datetime_filter=datetime_filter, - bbox_filter=bbox_filter, - properties_filter=properties_filter, - cql_filter=cql_filter, + where=where_filter.to_sql() if where_filter else None, sortby=sortby, properties=properties, geom=geom, - dt=dt, limit=limit, offset=offset, bbox_only=bbox_only, @@ -888,6 +843,18 @@ async def get_tile( f"Limit can not be set higher than the `tipg_max_features_per_tile` setting of {mvt_settings.max_features_per_tile}" ) + where_filter = self.cql_where( + ids=ids_filter, + datetime=datetime_filter, + bbox=bbox_filter, + properties=properties_filter, + cql=cql_filter, + geom=geom, + dt=dt, + tms=tms, + tile=tile, + ) + c = clauses.Clauses( self._select_mvt( properties=properties, @@ -896,17 +863,7 @@ async def get_tile( tile=tile, ), self._from(function_parameters), - self._where( - ids=ids_filter, - datetime=datetime_filter, - bbox=bbox_filter, - properties=properties_filter, - cql=cql_filter, - geom=geom, - dt=dt, - tms=tms, - tile=tile, - ), + clauses.Where(where_filter.to_sql() if where_filter else logic.S(True)), clauses.Limit(limit), ) diff --git a/tipg/factory.py b/tipg/factory.py index 9cf86fbe..9bd8e4f3 100644 --- a/tipg/factory.py +++ b/tipg/factory.py @@ -1103,9 +1103,9 @@ async def item( async with request.app.state.pool.acquire() as conn: item_list = await collection.features( conn, + cql_filter=Expr(f"{collection.id_column.name} = {itemId}"), bbox_only=bbox_only, simplify=simplify, - ids_filter=[itemId], properties=properties, function_parameters=function_parameters_query(request, collection), geom=geom_column, From 2a5f7c8a8e90b5ecf1bc2bc02fa6b8d1433483a2 Mon Sep 17 00:00:00 2001 From: Bennett Kanuka Date: Wed, 27 May 2026 18:02:59 -0400 Subject: [PATCH 05/10] in-progress. technically working? --- pyproject.toml | 2 +- tipg/collections.py | 285 ++++++++++++++++++++++++++++++------------- tipg/dependencies.py | 14 ++- tipg/factory.py | 2 +- 4 files changed, 213 insertions(+), 90 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 409f74d7..518cc01f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,7 +22,7 @@ classifiers = [ dynamic = ["version"] dependencies = [ "orjson", - "cql2", + "cql2>=0.5,<0.6", "asyncpg>=0.23.0", "buildpg>=0.3", "fastapi>=0.100.0", diff --git a/tipg/collections.py b/tipg/collections.py index 8edc00d9..ef882bae 100644 --- a/tipg/collections.py +++ b/tipg/collections.py @@ -12,7 +12,6 @@ from buildpg import logic, render from ciso8601 import parse_rfc3339 from cql2 import Expr -from geojson_pydantic.geometries import Polygon from morecantile import Tile, TileMatrixSet from pydantic import BaseModel, Field, model_validator from pyproj import Transformer @@ -43,16 +42,90 @@ TransformerFromCRS = lru_cache(Transformer.from_crs) -def bbox_to_wkt(bbox: List[float], srid: int = 4326) -> str: - """Return WKT representation of a BBOX.""" - poly = Polygon.from_bounds(*bbox) # type:ignore - return f"SRID={srid};{poly.wkt}" +_INT_PG_TYPES = { + "smallint", + "integer", + "bigint", + "smallserial", + "serial", + "bigserial", +} +_FLOAT_PG_TYPES = { + "real", + "double precision", + "decimal", + "numeric", + "float8", +} + +# TODO: This doesn't seem right. There must be a simpler or canonical way of doing this. +def _coerce_value(val: Any, pg_type: str) -> Any: + """Cast a URL-string filter value into the column's native Python type. + + Postgres applies implicit casts for single ``col = 'literal'`` comparisons, + but ``col = ANY(text[])`` does not coerce array elements, so we have to + convert here to keep IN-style filters working against numeric columns. + """ + if not isinstance(val, str): + return val + if pg_type in _INT_PG_TYPES: + return int(val) + if pg_type in _FLOAT_PG_TYPES: + return float(val) + return val + + +# TODO: Understand how tipg did bbox intersects before me. Manually constructing the Polygon is too ugly to be right. +def _s_intersects_bbox( + prop: str, west: float, south: float, east: float, north: float +) -> Expr: + """Build a cql2 S_INTERSECTS expression against a 4326 polygon envelope.""" + return Expr( + { + "op": "s_intersects", + "args": [ + {"property": prop}, + { + "type": "Polygon", + "coordinates": [ + [ + [west, south], + [east, south], + [east, north], + [west, north], + [west, south], + ] + ], + }, + ], + } + ) + + +# TODO: Do we need this? Can we do just one ST_TRANSFORM and use that geom for the rest of the query? +def _wrap_geom_property_to_4326(node: Any, geom_col_name: str) -> Any: + """Walk a cql2-json tree, replacing references to ``geom_col_name`` with + ``ST_Transform(geom_col, 4326)`` so the surrounding 4326-space predicates + line up with a non-4326 stored geometry column. + """ + if isinstance(node, dict): + if node.get("property") == geom_col_name and len(node) == 1: + return { + "op": "st_transform", + "args": [{"property": geom_col_name}, 4326], + } + return {k: _wrap_geom_property_to_4326(v, geom_col_name) for k, v in node.items()} + if isinstance(node, list): + return [_wrap_geom_property_to_4326(item, geom_col_name) for item in node] + return node def debug_query(q, *p): """Utility to print raw statement to use for debugging.""" - qsub = re.sub(r"\$([0-9]+)", r"{\1}", q) + # Escape literal `{` and `}` (cql2 emits GeoJSON literals containing them) + # then turn `$N` placeholders into `{N}` format slots. + qsub = re.sub(r"\$([0-9]+)", r"{\1}", q.replace("{", "{{").replace("}", "}}")) def quote_str(s): """Quote strings.""" @@ -339,40 +412,46 @@ def cql_where( # noqa: C901 dt: Optional[str] = None, tile: Optional[Tile] = None, tms: Optional[TileMatrixSet] = None, - ) -> Expr: - """Construct WHERE query.""" - exprs = [] + ) -> Optional[Expr]: + """Construct WHERE query as a cql2 Expr.""" + exprs: List[Expr] = [] - if cql: + if cql is not None: exprs.append(cql) # `ids` filter if ids: - # REF: https://github.com/developmentseed/cql2-rs/issues/91 + id_prop = {"property": self.id_column.name} if len(ids) == 1: - exprs.append(Expr(f"{self.id_column.name} = {ids[0]}")) + # Single `=` relies on Postgres' implicit text→col cast, + # which yields the natural type-error message for bad input. + exprs.append(Expr({"op": "=", "args": [id_prop, ids[0]]})) else: - id_list = ", ".join(f"'{id_}'" for id_ in ids) - exprs.append(Expr(f"{self.id_column.name} IN {id_list}")) + # `= ANY(text[])` does not coerce element types, so we have + # to convert URL strings to the column's native type here. + typed_ids = [_coerce_value(i, self.id_column.type) for i in ids] + exprs.append(Expr({"op": "in", "args": [id_prop, typed_ids]})) # `properties` filter if properties is not None: for prop, val in properties: - col = self.get_column(prop) - if not col: + if not self.get_column(prop): raise InvalidPropertyName(f"Invalid property name: {prop}") - exprs.append(Expr(f"{prop}='{val}'")) + exprs.append( + Expr({"op": "=", "args": [{"property": prop}, val]}) + ) # `bbox` filter geometry_column = self.get_geometry_column(geom) if bbox is not None and geometry_column is not None: - # TODO: should we use bbox_to_wkt(bbox) + # TODO: Do we need to handle bbox with 6 elements? + if len(bbox) == 6: + west, south, _, east, north, _ = bbox + else: + west, south, east, north = bbox exprs.append( - Expr( - f"S_INTERSECTS({geometry_column.name}, {', '.join(map(str, bbox))})" - ) + _s_intersects_bbox(geometry_column.name, west, south, east, north) ) - print(exprs[0].reduce().to_sql()) # `datetime` filter if datetime: @@ -385,21 +464,29 @@ def cql_where( # noqa: C901 if not datetime_column: raise InvalidDatetimeColumnName(f"Invalid Datetime Column: {dt}.") - if len(datetime) == 1: - # NOTE: should we do parse_rfc3339(datetime[0]) - exprs.append(Expr(f"{datetime_column.name}=TIMESTAMP('{datetime[0]}')")) + dt_prop = {"property": datetime_column.name} + if len(datetime) == 1: + parse_rfc3339(datetime[0]) + exprs.append( + Expr( + { + "op": "=", + "args": [dt_prop, {"timestamp": datetime[0]}], + } + ) + ) else: + start_str, end_str = datetime[0], datetime[1] start = ( - parse_rfc3339(datetime[0]) - if datetime[0] not in ["..", ""] + parse_rfc3339(start_str) + if start_str not in ["..", ""] else None ) end = ( - parse_rfc3339(datetime[1]) - if datetime[1] not in ["..", ""] - else None + parse_rfc3339(end_str) if end_str not in ["..", ""] else None ) + if start is None and end is None: raise InvalidDatetime( "Double open-ended datetime intervals are not allowed." @@ -410,54 +497,75 @@ def cql_where( # noqa: C901 "Start datetime cannot be before end datetime." ) - if (start and end) and start > end: - raise ValueError("Invalid datetime range: start must be <= end") - - startstr, endstr = datetime[:2] - if startstr not in ["..", ""]: + if start is not None: exprs.append( - Expr(f"{datetime_column.name}>=TIMESTAMP('{startstr}')") + Expr( + { + "op": ">=", + "args": [dt_prop, {"timestamp": start_str}], + } + ) + ) + if end is not None: + # TODO: Understand the correct way to handle inclusive/exclusive end + # Closed interval uses exclusive upper bound to keep + # the half-open `[start, end)` semantics tipg has always + # used; the open-ended `../` form remains inclusive + # (`<=`) + op = "<" if start is not None else "<=" + exprs.append( + Expr( + { + "op": op, + "args": [dt_prop, {"timestamp": end_str}], + } + ) + ) + + # `tile` envelope filter — reproject tile bounds (TMS CRS) to 4326 + if tile and tms and geometry_column: + bounds = tms.xy_bounds(tile) + west, south, east, north = ( + bounds.left, + bounds.bottom, + bounds.right, + bounds.top, + ) + tms_epsg = tms.crs.to_epsg() or 4326 + if tms_epsg != 4326: + transformer = TransformerFromCRS(tms_epsg, 4326, always_xy=True) + west, south, east, north = transformer.transform_bounds( + west, south, east, north + ) + exprs.append( + _s_intersects_bbox(geometry_column.name, west, south, east, north) + ) + + # For non-4326 geometry columns, wrap every reference to the geom + # column in `ST_Transform(geom, 4326)` so the whole WHERE evaluates + # in 4326 space (all of our built-in spatial filters above use 4326 + # polygons, and the user is told to supply spatial literals in 4326). + + # Note: this loses the spatial index on the geom column for non-4326 + # collections; an index-friendly variant would transform the + # polygon-literal side to geom's SRID instead. + + # TODO: Should we transform the bbox or polygon CRS instead? + if geometry_column is not None and geometry_column.srid not in (None, 4326): + exprs = [ + Expr( + _wrap_geom_property_to_4326( + e.to_json(), geometry_column.name ) - if endstr: - exprs.append(Expr(f"{datetime_column.name}<=TIMESTAMP('{endstr}')")) - - # if tile and tms and geometry_column: - # # Get tile bounds in the TMS coordinate system - # bbox = tms.xy_bounds(tile) - # left, bottom, right, top = bbox - - # # If the geometry column’s SRID does not match the TMS CRS, transform the bounds: - # # Use a fallback of 4326 if tms.crs.to_epsg() returns a falsey value. - # tms_epsg = tms.crs.to_epsg() or 4326 - # if geometry_column.srid != tms_epsg: - # transformer = TransformerFromCRS( - # tms_epsg, geometry_column.srid, always_xy=True - # ) - - # left, bottom, right, top = transformer.transform_bounds( - # left, bottom, right, top - # ) - - # wheres.append( - # logic.Func( - # "ST_Intersects", - # logic.Func( - # "ST_Segmentize", - # logic.Func( - # "ST_MakeEnvelope", - # left, - # bottom, - # right, - # top, - # geometry_column.srid, - # ), - # right - left, - # ), - # logic.V(geometry_column.name), - # ) - # ) + ) + for e in exprs + ] + if exprs: - return reduce(lambda x, y: x + y, exprs).reduce() + # NOTE: do not call .reduce() — cql2-rs constant-folds some + # predicates (e.g. `"numeric" IS NULL`) incorrectly. + # TODO: Open a bug in cql2-rs for this. + return reduce(lambda x, y: x + y, exprs) return None @@ -553,7 +661,8 @@ def _select_mvt( """Create MVT from intersecting geometries.""" geom = pg_funcs.cast(logic.V(geometry_column.name), "geometry") - # make sure the geometries do not overflow the TMS bbox + # make sure the geometries do not overflow the TMS bbox — `tms.bbox` + # is 4326, so we transform any non-4326 geom to 4326 before clipping. if not tms.is_valid(tile): geom = logic.Func( "ST_Intersection", @@ -636,7 +745,7 @@ def _geom( g = pg_funcs.cast(logic.V(geometry_column.name), "geometry") - # Reproject to WGS64 if needed + # Reproject to WGS84 if needed — GeoJSON output is always 4326 if geometry_column.srid != 4326: g = logic.Func("ST_Transform", g, pg_funcs.cast(4326, "int")) @@ -678,11 +787,19 @@ def _sortby(self, sortby: Optional[str]): return clauses.OrderBy(*sorts) + # TODO: get rid of buildpg + @staticmethod + def _where_clause(where: Optional[Expr]): + """Wrap a cql2 Expr (or None) into a buildpg Where clause.""" + if where is None: + return clauses.Where(logic.S(True)) + return clauses.Where(raw(where.to_sql())) + async def _features_query( self, conn: asyncpg.Connection, *, - where: Optional[str] = None, + where: Optional[Expr] = None, sortby: Optional[str] = None, properties: Optional[List[str]] = None, geom: Optional[str] = None, @@ -706,7 +823,7 @@ async def _features_query( geom_as_wkt=geom_as_wkt, ), self._from(function_parameters), - clauses.Where(where or logic.S(True)), + self._where_clause(where), self._sortby(sortby), clauses.Limit(limit), clauses.Offset(offset), @@ -724,14 +841,14 @@ async def _features_count_query( self, conn: asyncpg.Connection, *, - where: Optional[str], + where: Optional[Expr] = None, function_parameters: Optional[Dict[str, str]], ) -> int: """Build features COUNT query.""" c = clauses.Clauses( self._select_count(), self._from(function_parameters), - clauses.Where(where or logic.S(True)), + self._where_clause(where), ) q, p = render(":c", c=c) @@ -784,7 +901,7 @@ async def features( matched = await self._features_count_query( conn, - where=where_filter.to_sql() if where_filter else None, + where=where_filter, function_parameters=function_parameters, ) @@ -792,7 +909,7 @@ async def features( f async for f in self._features_query( conn, - where=where_filter.to_sql() if where_filter else None, + where=where_filter, sortby=sortby, properties=properties, geom=geom, @@ -863,7 +980,7 @@ async def get_tile( tile=tile, ), self._from(function_parameters), - clauses.Where(where_filter.to_sql() if where_filter else logic.S(True)), + self._where_clause(where_filter), clauses.Limit(limit), ) diff --git a/tipg/dependencies.py b/tipg/dependencies.py index 70fbfe48..926137da 100644 --- a/tipg/dependencies.py +++ b/tipg/dependencies.py @@ -288,11 +288,17 @@ def filter_query( ), ] = None, ) -> Optional[Expr]: - """Parse Filter Query.""" - if query is not None: - return Expr(query) + """Parse Filter Query. - return None + User-supplied filters are normalized through cql2-json so spatial literals + (`POLYGON(...)`, `POINT(...)`, etc.) compile to ``ST_GeomFromGeoJSON`` + (SRID 4326) rather than ``ST_GeomFromText`` (SRID 0), which would + otherwise produce a mixed-SRID error against the 4326 geometry columns. + """ + if query is None: + return None + + return Expr(Expr(query).to_json()) def sortby_query( diff --git a/tipg/factory.py b/tipg/factory.py index 9bd8e4f3..bb7879e2 100644 --- a/tipg/factory.py +++ b/tipg/factory.py @@ -1103,7 +1103,7 @@ async def item( async with request.app.state.pool.acquire() as conn: item_list = await collection.features( conn, - cql_filter=Expr(f"{collection.id_column.name} = {itemId}"), + ids_filter=[itemId], bbox_only=bbox_only, simplify=simplify, properties=properties, From 57c9df10ded11cd4d2b1662bfabeb27b21cc21c3 Mon Sep 17 00:00:00 2001 From: Bennett Kanuka Date: Fri, 29 May 2026 18:24:09 -0400 Subject: [PATCH 06/10] additional work. still in progress --- tipg/collections.py | 227 +++++++++++++++++++++++++------------------ tipg/dependencies.py | 19 +++- 2 files changed, 146 insertions(+), 100 deletions(-) diff --git a/tipg/collections.py b/tipg/collections.py index ef882bae..a065ba30 100644 --- a/tipg/collections.py +++ b/tipg/collections.py @@ -23,6 +23,7 @@ InvalidLimit, InvalidPropertyName, MissingDatetimeColumn, + NotFound, ) from tipg.logger import logger from tipg.model import Extent @@ -42,81 +43,103 @@ TransformerFromCRS = lru_cache(Transformer.from_crs) -_INT_PG_TYPES = { - "smallint", - "integer", - "bigint", - "smallserial", - "serial", - "bigserial", -} -_FLOAT_PG_TYPES = { - "real", - "double precision", - "decimal", - "numeric", - "float8", -} +_INT_PG_TYPES = frozenset( + { + "smallint", + "integer", + "bigint", + "smallserial", + "serial", + "bigserial", + } +) + -# TODO: This doesn't seem right. There must be a simpler or canonical way of doing this. -def _coerce_value(val: Any, pg_type: str) -> Any: - """Cast a URL-string filter value into the column's native Python type. +def _coerce_id(val: str, pg_type: str) -> Union[str, int]: + """Parse a URL-supplied id into the primary-key column's Python type. - Postgres applies implicit casts for single ``col = 'literal'`` comparisons, - but ``col = ANY(text[])`` does not coerce array elements, so we have to - convert here to keep IN-style filters working against numeric columns. + IDs always arrive as strings (URL path or query parameters); the + underlying column is either text or integer. cql2 treats Python ints + and strs as different literal kinds (and PostgreSQL ``= ANY(text[])`` + does not coerce array elements at runtime), so we normalize here. + + A non-integer string against an integer column means the id can't + exist, so map the parse failure onto a 404 rather than letting the + eventual PostgreSQL ``invalid input syntax`` error bubble up as a 500. """ - if not isinstance(val, str): + if pg_type not in _INT_PG_TYPES: return val - if pg_type in _INT_PG_TYPES: + try: return int(val) - if pg_type in _FLOAT_PG_TYPES: - return float(val) - return val + except ValueError as exc: + raise NotFound(f"Invalid id {val!r} for {pg_type} column.") from exc -# TODO: Understand how tipg did bbox intersects before me. Manually constructing the Polygon is too ugly to be right. def _s_intersects_bbox( - prop: str, west: float, south: float, east: float, north: float + prop: str, + west: float, + south: float, + east: float, + north: float, + srid: Optional[int] = None, ) -> Expr: - """Build a cql2 S_INTERSECTS expression against a 4326 polygon envelope.""" - return Expr( - { - "op": "s_intersects", - "args": [ - {"property": prop}, - { - "type": "Polygon", - "coordinates": [ - [ - [west, south], - [east, south], - [east, north], - [west, north], - [west, south], - ] - ], - }, - ], - } - ) + """Build a cql2 S_INTERSECTS expression against a polygon envelope. + + Coordinates are taken to be in ``srid``; if ``srid`` is non-4326, the + polygon literal is wrapped in ``ST_SetSRID`` so PostGIS does not pick + up ``ST_GeomFromGeoJSON``'s 4326 default. Pass ``srid=None`` or + ``srid=4326`` for the no-op case. + """ + polygon: Any = { + "type": "Polygon", + "coordinates": [ + [ + [west, south], + [east, south], + [east, north], + [west, north], + [west, south], + ] + ], + } + if srid is not None and srid != 4326: + polygon = {"op": "st_setsrid", "args": [polygon, srid]} + return Expr({"op": "s_intersects", "args": [{"property": prop}, polygon]}) + + +_GEOJSON_GEOMETRY_TYPES = frozenset( + { + "Point", + "MultiPoint", + "LineString", + "MultiLineString", + "Polygon", + "MultiPolygon", + "GeometryCollection", + } +) -# TODO: Do we need this? Can we do just one ST_TRANSFORM and use that geom for the rest of the query? -def _wrap_geom_property_to_4326(node: Any, geom_col_name: str) -> Any: - """Walk a cql2-json tree, replacing references to ``geom_col_name`` with - ``ST_Transform(geom_col, 4326)`` so the surrounding 4326-space predicates - line up with a non-4326 stored geometry column. +def _transform_cql_geom_literals(node: Any, target_srid: int) -> Any: + """Walk a user-supplied cql2-json tree and wrap every GeoJSON geometry + literal in ``ST_Transform(, target_srid)``. + + Per OGC API Features Part 3 Req 7, geometries in a CQL filter are + CRS84 (≈ EPSG:4326). Wrapping the literal side (rather than the + column side) keeps PostGIS' spatial index on the column usable and + lets the planner fold the transform on the constant once. """ if isinstance(node, dict): - if node.get("property") == geom_col_name and len(node) == 1: - return { - "op": "st_transform", - "args": [{"property": geom_col_name}, 4326], - } - return {k: _wrap_geom_property_to_4326(v, geom_col_name) for k, v in node.items()} + if node.get("type") in _GEOJSON_GEOMETRY_TYPES and ( + "coordinates" in node or "geometries" in node + ): + return {"op": "st_transform", "args": [node, target_srid]} + return { + k: _transform_cql_geom_literals(v, target_srid) + for k, v in node.items() + } if isinstance(node, list): - return [_wrap_geom_property_to_4326(item, geom_col_name) for item in node] + return [_transform_cql_geom_literals(item, target_srid) for item in node] return node @@ -415,21 +438,31 @@ def cql_where( # noqa: C901 ) -> Optional[Expr]: """Construct WHERE query as a cql2 Expr.""" exprs: List[Expr] = [] + geometry_column = self.get_geometry_column(geom) + + # Per OGC API Features Part 3 Req 7, geometries in `cql` and `bbox` + # are CRS84 (≈ EPSG:4326). When the stored column is in another + # SRID, wrap their literals in `ST_Transform(literal, )` + # so the comparison happens in the column's native CRS — PostGIS + # folds the transform on the constant side and the spatial index + # on the column stays usable. + target_srid = geometry_column.srid if geometry_column is not None else None + needs_srid_wrap = target_srid is not None and target_srid != 4326 if cql is not None: + if needs_srid_wrap: + cql = Expr( + _transform_cql_geom_literals(cql.to_json(), target_srid) + ) exprs.append(cql) # `ids` filter if ids: id_prop = {"property": self.id_column.name} - if len(ids) == 1: - # Single `=` relies on Postgres' implicit text→col cast, - # which yields the natural type-error message for bad input. - exprs.append(Expr({"op": "=", "args": [id_prop, ids[0]]})) + typed_ids = [_coerce_id(i, self.id_column.type) for i in ids] + if len(typed_ids) == 1: + exprs.append(Expr({"op": "=", "args": [id_prop, typed_ids[0]]})) else: - # `= ANY(text[])` does not coerce element types, so we have - # to convert URL strings to the column's native type here. - typed_ids = [_coerce_value(i, self.id_column.type) for i in ids] exprs.append(Expr({"op": "in", "args": [id_prop, typed_ids]})) # `properties` filter @@ -441,16 +474,24 @@ def cql_where( # noqa: C901 Expr({"op": "=", "args": [{"property": prop}, val]}) ) - # `bbox` filter - geometry_column = self.get_geometry_column(geom) + # `bbox` filter — bbox is CRS84 (4326) per OGC API Features. + # Reproject the four corner coords to the column's CRS in Python + # so the polygon literal already carries target-CRS coordinates; + # ST_SetSRID then just tags it, no PostGIS-side transform needed. if bbox is not None and geometry_column is not None: - # TODO: Do we need to handle bbox with 6 elements? if len(bbox) == 6: west, south, _, east, north, _ = bbox else: west, south, east, north = bbox + if needs_srid_wrap: + transformer = TransformerFromCRS(4326, target_srid, always_xy=True) + west, south, east, north = transformer.transform_bounds( + west, south, east, north + ) exprs.append( - _s_intersects_bbox(geometry_column.name, west, south, east, north) + _s_intersects_bbox( + geometry_column.name, west, south, east, north, srid=target_srid + ) ) # `datetime` filter @@ -522,7 +563,11 @@ def cql_where( # noqa: C901 ) ) - # `tile` envelope filter — reproject tile bounds (TMS CRS) to 4326 + # `tile` envelope filter — reproject tile bounds directly from the + # TMS CRS to the stored column's CRS (skipping the 4326 round-trip) + # and tag the resulting polygon literal with `ST_SetSRID` so it + # carries the column's SRID rather than the 4326 default that + # `ST_GeomFromGeoJSON` would otherwise apply. if tile and tms and geometry_column: bounds = tms.xy_bounds(tile) west, south, east, north = ( @@ -532,34 +577,24 @@ def cql_where( # noqa: C901 bounds.top, ) tms_epsg = tms.crs.to_epsg() or 4326 - if tms_epsg != 4326: - transformer = TransformerFromCRS(tms_epsg, 4326, always_xy=True) + tile_target_srid = target_srid if target_srid is not None else 4326 + if tms_epsg != tile_target_srid: + transformer = TransformerFromCRS( + tms_epsg, tile_target_srid, always_xy=True + ) west, south, east, north = transformer.transform_bounds( west, south, east, north ) exprs.append( - _s_intersects_bbox(geometry_column.name, west, south, east, north) - ) - - # For non-4326 geometry columns, wrap every reference to the geom - # column in `ST_Transform(geom, 4326)` so the whole WHERE evaluates - # in 4326 space (all of our built-in spatial filters above use 4326 - # polygons, and the user is told to supply spatial literals in 4326). - - # Note: this loses the spatial index on the geom column for non-4326 - # collections; an index-friendly variant would transform the - # polygon-literal side to geom's SRID instead. - - # TODO: Should we transform the bbox or polygon CRS instead? - if geometry_column is not None and geometry_column.srid not in (None, 4326): - exprs = [ - Expr( - _wrap_geom_property_to_4326( - e.to_json(), geometry_column.name - ) + _s_intersects_bbox( + geometry_column.name, + west, + south, + east, + north, + srid=tile_target_srid, ) - for e in exprs - ] + ) if exprs: # NOTE: do not call .reduce() — cql2-rs constant-folds some diff --git a/tipg/dependencies.py b/tipg/dependencies.py index 926137da..42938bd7 100644 --- a/tipg/dependencies.py +++ b/tipg/dependencies.py @@ -290,10 +290,21 @@ def filter_query( ) -> Optional[Expr]: """Parse Filter Query. - User-supplied filters are normalized through cql2-json so spatial literals - (`POLYGON(...)`, `POINT(...)`, etc.) compile to ``ST_GeomFromGeoJSON`` - (SRID 4326) rather than ``ST_GeomFromText`` (SRID 0), which would - otherwise produce a mixed-SRID error against the 4326 geometry columns. + Per OGC API - Features - Part 3: Filtering, Requirement 7 + (``/req/filter/filter-crs-wgs84``), when no ``filter-crs`` is supplied + the server "SHALL process all geometries in the filter expression using + CRS84". tipg does not implement the ``filter-crs`` parameter, so we + always treat filter geometries as CRS84 (≈ EPSG:4326). + + To make that assumption explicit in the generated SQL, we round-trip + user input through cql2-json before re-parsing. The cql2 library emits + ``ST_GeomFromText(...)`` (SRID 0) when an ``Expr`` was parsed from + cql2-text, but ``ST_GeomFromGeoJSON(...)`` (SRID 4326) when parsed + from cql2-json — and SRID 0 literals fail any mixed-SRID PostGIS + comparison. Re-parsing the JSON form forces every spatial literal into + the SRID-4326 emit path. ``_where`` later wraps these 4326 literals in + ``ST_Transform(..., )`` if the target column is in a + different CRS, so the index on the column side is preserved. """ if query is None: return None From cd67753447f627370006f4b858ec6b52a97afee2 Mon Sep 17 00:00:00 2001 From: Bennett Kanuka Date: Fri, 29 May 2026 19:56:56 -0400 Subject: [PATCH 07/10] remove buildpg --- pyproject.toml | 3 +- tipg/collections.py | 419 +++++++++++++++++++++----------------------- tipg/database.py | 4 +- 3 files changed, 203 insertions(+), 223 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 518cc01f..8358cd7a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,7 +24,6 @@ dependencies = [ "orjson", "cql2>=0.5,<0.6", "asyncpg>=0.23.0", - "buildpg>=0.3", "fastapi>=0.100.0", "jinja2>=2.11.2,<4.0.0", "morecantile>=5.0,<7.0", @@ -95,7 +94,7 @@ exclude_lines = [ [tool.isort] profile = "black" known_first_party = ["tipg"] -known_third_party = ["geojson_pydantic", "buildpg", "pydantic"] +known_third_party = ["geojson_pydantic", "pydantic"] forced_separate = [ "fastapi", "starlette", diff --git a/tipg/collections.py b/tipg/collections.py index a065ba30..6a6245ed 100644 --- a/tipg/collections.py +++ b/tipg/collections.py @@ -6,10 +6,7 @@ from functools import lru_cache, reduce from typing import Any, Dict, List, Optional, Tuple, TypedDict, Union -from buildpg import RawDangerous as raw -from buildpg import asyncpg, clauses -from buildpg import funcs as pg_funcs -from buildpg import logic, render +import asyncpg from ciso8601 import parse_rfc3339 from cql2 import Expr from morecantile import Tile, TileMatrixSet @@ -43,6 +40,11 @@ TransformerFromCRS = lru_cache(Transformer.from_crs) +def _quote_ident(name: str) -> str: + """Quote a PostgreSQL identifier (column/table/schema) safely.""" + return '"' + name.replace('"', '""') + '"' + + _INT_PG_TYPES = frozenset( { "smallint", @@ -636,26 +638,48 @@ class PgCollection(Collection): dbschema: str = Field(alias="schema") - def _select_no_geo(self, properties: Optional[List[str]], addid: bool = True): - nocomma = False - columns = self.columns(properties) - if columns: - sel = logic.as_sql_block(clauses.Select(columns)) - else: - sel = logic.as_sql_block(raw("SELECT ")) - nocomma = True + @property + def _qualified_name(self) -> str: + """SQL-quoted ``"schema"."table"`` identifier for this collection.""" + return f"{_quote_ident(self.dbschema)}.{_quote_ident(self.table)}" - if addid: - if self.id_column: - id_clause = logic.V(self.id_column.name).as_("tipg_id") - else: - id_clause = raw(" ROW_NUMBER () OVER () AS tipg_id ") - if nocomma: - sel = clauses.Clauses(sel, id_clause) - else: - sel = sel.comma(id_clause) + def _select_property_columns(self, properties: Optional[List[str]]) -> List[str]: + """Quoted column-name fragments for the property columns to SELECT.""" + return [_quote_ident(c) for c in self.columns(properties)] + + def _select_id(self) -> str: + """SQL fragment that selects the primary-key column as ``tipg_id``. + + Falls back to ``ROW_NUMBER()`` when the collection has no primary key. + """ + if self.id_column: + return f"{_quote_ident(self.id_column.name)} AS tipg_id" + return "ROW_NUMBER() OVER () AS tipg_id" + + def _geom_expr( + self, + geometry_column: Optional[Column], + bbox_only: Optional[bool], + simplify: Optional[float], + ) -> Optional[str]: + """Build the SQL expression for the geometry column (reprojected to + 4326 if needed, optionally bbox-only or simplified). + """ + if geometry_column is None: + return None + + g = f"CAST({_quote_ident(geometry_column.name)} AS geometry)" + + if geometry_column.srid != 4326: + g = f"ST_Transform({g}, 4326)" + + if bbox_only: + g = f"ST_Envelope({g})" + elif simplify: + s = float(simplify) + g = f"ST_SnapToGrid(ST_Simplify({g}, {s}), {s})" - return logic.as_sql_block(sel) + return g def _select( self, @@ -664,27 +688,26 @@ def _select( bbox_only: Optional[bool], simplify: Optional[float], geom_as_wkt: bool = False, - ): - sel = self._select_no_geo(properties) + ) -> str: + """Build the SELECT clause for the main features query.""" + cols = self._select_property_columns(properties) + cols.append(self._select_id()) - geom = self._geom(geometry_column, bbox_only, simplify) + geom = self._geom_expr(geometry_column, bbox_only, simplify) if geom_as_wkt: - if geom: - sel = sel.comma(logic.Func("ST_AsEWKT", geom).as_("tipg_geom")) - else: - sel = sel.comma(pg_funcs.cast(None, "text").as_("tipg_geom")) - + cols.append( + f"ST_AsEWKT({geom}) AS tipg_geom" + if geom + else "CAST(NULL AS text) AS tipg_geom" + ) else: - if geom: - sel = sel.comma( - pg_funcs.cast(logic.Func("ST_AsGeoJSON", geom), "json").as_( - "tipg_geom" - ) - ) - else: - sel = sel.comma(pg_funcs.cast(None, "json").as_("tipg_geom")) + cols.append( + f"CAST(ST_AsGeoJSON({geom}) AS json) AS tipg_geom" + if geom + else "CAST(NULL AS json) AS tipg_geom" + ) - return sel + return "SELECT " + ", ".join(cols) def _select_mvt( self, @@ -692,113 +715,84 @@ def _select_mvt( geometry_column: Column, tms: TileMatrixSet, tile: Tile, - ): - """Create MVT from intersecting geometries.""" - geom = pg_funcs.cast(logic.V(geometry_column.name), "geometry") - - # make sure the geometries do not overflow the TMS bbox — `tms.bbox` - # is 4326, so we transform any non-4326 geom to 4326 before clipping. + ) -> str: + """Build the SELECT clause that emits an MVT geometry per row.""" + cols = self._select_property_columns(properties) + + geom = f"CAST({_quote_ident(geometry_column.name)} AS geometry)" + + # For tiles that fall outside the TMS's natural domain (e.g. an + # over-zoomed corner tile), clip the source geometry to the TMS's + # geographic bbox before reprojecting — otherwise ST_AsMVTGeom can + # produce garbage near the antimeridian / poles. ``tms.bbox`` is in + # the TMS's ``geographic_crs``, which is NOT always EPSG:4326 (e.g. + # CanadianNAD83_LCC uses 4269, EuropeanETRS89_LAEAQuad uses 4258, + # NZTM2000Quad uses 4167). When the geographic CRS has no EPSG code + # (WorldCRS84Quad → OGC:CRS84), 4326 is a safe fallback since the + # coordinate values are identical (only axis order differs, and + # PostGIS treats 4326 as lon/lat). if not tms.is_valid(tile): - geom = logic.Func( - "ST_Intersection", - logic.Func("ST_MakeEnvelope", *tms.bbox, 4326), - logic.Func( - "ST_Transform", - geom, - pg_funcs.cast(4326, "int"), - ), + west, south, east, north = tms.bbox + geo_srid = tms.geographic_crs.to_epsg() or 4326 + geom = ( + f"ST_Intersection(" + f"ST_MakeEnvelope({west}, {south}, {east}, {north}, {geo_srid}), " + f"ST_Transform({geom}, {geo_srid})" + f")" ) - # Transform the geometries to TMS CRS using EPSG code + # Reproject to TMS CRS — prefer EPSG code, fall back to PROJ string. if tms_srid := tms.crs.to_epsg(): - transform_logic = logic.Func( - "ST_Transform", - geom, - pg_funcs.cast(tms_srid, "int"), - ) - - # Transform the geometries to TMS CRS using PROJ String + transformed = f"ST_Transform({geom}, {tms_srid})" else: - tms_proj = tms.crs.to_proj4() - transform_logic = logic.Func( - "ST_Transform", - geom, - pg_funcs.cast(tms_proj, "text"), - ) + tms_proj = tms.crs.to_proj4().replace("'", "''") + transformed = f"ST_Transform({geom}, '{tms_proj}')" bbox = tms.xy_bounds(tile) - sel = self._select_no_geo(properties, addid=False).comma( - logic.Func( - "ST_AsMVTGeom", - transform_logic, - logic.Func( - "ST_Segmentize", - logic.Func( - "ST_MakeEnvelope", - bbox.left, - bbox.bottom, - bbox.right, - bbox.top, - ), - bbox.right - bbox.left, - ), - mvt_settings.tile_resolution, - mvt_settings.tile_buffer, - mvt_settings.tile_clip, - ).as_("geom") + envelope = ( + f"ST_Segmentize(" + f"ST_MakeEnvelope({bbox.left}, {bbox.bottom}, {bbox.right}, {bbox.top}), " + f"{bbox.right - bbox.left}" + f")" + ) + cols.append( + f"ST_AsMVTGeom(" + f"{transformed}, {envelope}, " + f"{int(mvt_settings.tile_resolution)}, " + f"{int(mvt_settings.tile_buffer)}, " + f"{'TRUE' if mvt_settings.tile_clip else 'FALSE'}" + f") AS geom" ) - return sel - - def _select_count(self): - return clauses.Select(pg_funcs.count("*")) - - def _from(self, function_parameters: Optional[Dict[str, str]]): - if self.type == "Function": - if not function_parameters: - return clauses.From(self.id) + raw("()") - params = [] - for p in self.parameters: - if p.name in function_parameters: - params.append( - pg_funcs.cast( - pg_funcs.cast(function_parameters[p.name], "text"), - p.type, - ) - ) - return clauses.From(logic.Func(self.id, *params)) - return clauses.From(self.id) + return "SELECT " + ", ".join(cols) - def _geom( + def _from( self, - geometry_column: Optional[Column], - bbox_only: Optional[bool], - simplify: Optional[float], - ): - if geometry_column is None: - return None - - g = pg_funcs.cast(logic.V(geometry_column.name), "geometry") - - # Reproject to WGS84 if needed — GeoJSON output is always 4326 - if geometry_column.srid != 4326: - g = logic.Func("ST_Transform", g, pg_funcs.cast(4326, "int")) - - # Return BBOX Only - if bbox_only: - g = logic.Func("ST_Envelope", g) - - # Simplify the geometry - elif simplify: - g = logic.Func( - "ST_SnapToGrid", - logic.Func("ST_Simplify", g, simplify), - simplify, - ) - - return g - - def _sortby(self, sortby: Optional[str]): + function_parameters: Optional[Dict[str, str]], + params: List[Any], + ) -> str: + """Build the FROM clause. Function-table parameters are appended to + ``params`` and referenced via ``$N`` placeholders. + """ + name = self._qualified_name + if self.type != "Function": + return f"FROM {name}" + + if not function_parameters: + return f"FROM {name}()" + + args = [] + for p in self.parameters: + if p.name in function_parameters: + params.append(function_parameters[p.name]) + # Double cast (value → text → target type): asyncpg sends + # Python strs as text, and the explicit cast to ``p.type`` + # then coerces to whatever the function signature expects. + args.append(f"CAST(CAST(${len(params)} AS text) AS {p.type})") + return f"FROM {name}({', '.join(args)})" + + def _sortby(self, sortby: Optional[str]) -> str: + """Build the ORDER BY clause.""" sorts = [] if sortby: for s in sortby.strip().split(","): @@ -806,29 +800,23 @@ def _sortby(self, sortby: Optional[str]): direction = parts["direction"] column = parts["column"].strip() - if self.get_column(column): - if direction == "-": - sorts.append(logic.V(column).desc()) - else: - sorts.append(logic.V(column)) - else: + if not self.get_column(column): raise InvalidPropertyName(f"Property {column} does not exist.") - + col_sql = _quote_ident(column) + sorts.append(f"{col_sql} DESC" if direction == "-" else col_sql) + elif self.id_column is not None: + sorts.append(_quote_ident(self.id_column.name)) else: - if self.id_column is not None: - sorts.append(logic.V(self.id_column.name)) - else: - sorts.append(logic.V(self.properties[0].name)) + sorts.append(_quote_ident(self.properties[0].name)) - return clauses.OrderBy(*sorts) + return "ORDER BY " + ", ".join(sorts) - # TODO: get rid of buildpg @staticmethod - def _where_clause(where: Optional[Expr]): - """Wrap a cql2 Expr (or None) into a buildpg Where clause.""" + def _where_clause(where: Optional[Expr]) -> str: + """Render a cql2 ``Expr`` (or ``None``) as a WHERE clause.""" if where is None: - return clauses.Where(logic.S(True)) - return clauses.Where(raw(where.to_sql())) + return "WHERE TRUE" + return f"WHERE {where.to_sql()}" async def _features_query( self, @@ -845,27 +833,29 @@ async def _features_query( geom_as_wkt: bool = False, function_parameters: Optional[Dict[str, str]], ): - """Build Features query.""" + """Build and run the Features query, yielding each row as a Feature.""" limit = limit or features_settings.default_features_limit offset = offset or 0 + params: List[Any] = [] - c = clauses.Clauses( - self._select( - properties=properties, - geometry_column=self.get_geometry_column(geom), - bbox_only=bbox_only, - simplify=simplify, - geom_as_wkt=geom_as_wkt, - ), - self._from(function_parameters), - self._where_clause(where), - self._sortby(sortby), - clauses.Limit(limit), - clauses.Offset(offset), + sql = " ".join( + [ + self._select( + properties=properties, + geometry_column=self.get_geometry_column(geom), + bbox_only=bbox_only, + simplify=simplify, + geom_as_wkt=geom_as_wkt, + ), + self._from(function_parameters, params), + self._where_clause(where), + self._sortby(sortby), + f"LIMIT {int(limit)}", + f"OFFSET {int(offset)}", + ] ) - q, p = render(":c", c=c) - for r in await conn.fetch(q, *p): + for r in await conn.fetch(sql, *params): props = dict(r) g = props.pop("tipg_geom") id = props.pop("tipg_id") @@ -879,16 +869,16 @@ async def _features_count_query( where: Optional[Expr] = None, function_parameters: Optional[Dict[str, str]], ) -> int: - """Build features COUNT query.""" - c = clauses.Clauses( - self._select_count(), - self._from(function_parameters), - self._where_clause(where), + """Run the COUNT(*) query for the current filter.""" + params: List[Any] = [] + sql = " ".join( + [ + "SELECT COUNT(*)", + self._from(function_parameters, params), + self._where_clause(where), + ] ) - - q, p = render(":c", c=c) - count = await conn.fetchval(q, *p) - return count + return await conn.fetchval(sql, *params) async def features( self, @@ -1007,30 +997,30 @@ async def get_tile( tile=tile, ) - c = clauses.Clauses( - self._select_mvt( - properties=properties, - geometry_column=geometry_column, - tms=tms, - tile=tile, - ), - self._from(function_parameters), - self._where_clause(where_filter), - clauses.Limit(limit), + params: List[Any] = [] + inner_sql = " ".join( + [ + self._select_mvt( + properties=properties, + geometry_column=geometry_column, + tms=tms, + tile=tile, + ), + self._from(function_parameters, params), + self._where_clause(where_filter), + f"LIMIT {int(limit)}", + ] ) - q, p = render( - """ - WITH - t AS (:c) - SELECT ST_AsMVT(t.*, :l) FROM t - """, - c=c, - l=self.table if mvt_settings.set_mvt_layername is True else "default", + layer = self.table if mvt_settings.set_mvt_layername is True else "default" + params.append(layer) + sql = ( + f"WITH t AS ({inner_sql}) " + f"SELECT ST_AsMVT(t.*, ${len(params)}) FROM t" ) - debug_query(q, *p) + debug_query(sql, *params) - tile = await conn.fetchval(q, *p) + tile = await conn.fetchval(sql, *params) return bytes(tile) @@ -1047,31 +1037,22 @@ async def pg_get_collection_index( # noqa: C901 query = f""" SELECT {settings.tipg_schema}.tipg_catalog( - :schemas, - :tables, - :exclude_tables, - :exclude_table_schemas, - :functions, - :exclude_functions, - :exclude_function_schemas, - :spatial, - :spatial_extent, - :datetime_extent + $1, $2, $3, $4, $5, $6, $7, $8, $9, $10 ); - """ # noqa: W605 + """ - rows = await conn.fetch_b( + rows = await conn.fetch( query, - schemas=schemas, - tables=settings.tables, - exclude_tables=settings.exclude_tables, - exclude_table_schemas=settings.exclude_table_schemas, - functions=settings.functions, - exclude_functions=settings.exclude_functions, - exclude_function_schemas=settings.exclude_function_schemas, - spatial=settings.only_spatial_tables, - spatial_extent=settings.spatial_extent, - datetime_extent=settings.datetime_extent, + schemas, + settings.tables, + settings.exclude_tables, + settings.exclude_table_schemas, + settings.functions, + settings.exclude_functions, + settings.exclude_function_schemas, + settings.only_spatial_tables, + settings.spatial_extent, + settings.datetime_extent, ) collections: List[Collection] = [] diff --git a/tipg/database.py b/tipg/database.py index 43fa0bb3..da2e0df7 100644 --- a/tipg/database.py +++ b/tipg/database.py @@ -4,8 +4,8 @@ from importlib.resources import files as resources_files from typing import List, Optional +import asyncpg import orjson -from buildpg import asyncpg from tipg.logger import logger from tipg.settings import PostgresSettings @@ -82,7 +82,7 @@ async def connect_to_db( if not settings: settings = PostgresSettings() - app.state.pool = await asyncpg.create_pool_b( + app.state.pool = await asyncpg.create_pool( str(settings.database_url), min_size=settings.db_min_conn_size, max_size=settings.db_max_conn_size, From e3e875876230a8a1761cbd9b35df0123f0ab0f9e Mon Sep 17 00:00:00 2001 From: Bennett Kanuka Date: Mon, 1 Jun 2026 10:20:03 -0400 Subject: [PATCH 08/10] ruff-format --- tipg/collections.py | 24 ++++++------------------ 1 file changed, 6 insertions(+), 18 deletions(-) diff --git a/tipg/collections.py b/tipg/collections.py index 6a6245ed..9da8eb70 100644 --- a/tipg/collections.py +++ b/tipg/collections.py @@ -137,8 +137,7 @@ def _transform_cql_geom_literals(node: Any, target_srid: int) -> Any: ): return {"op": "st_transform", "args": [node, target_srid]} return { - k: _transform_cql_geom_literals(v, target_srid) - for k, v in node.items() + k: _transform_cql_geom_literals(v, target_srid) for k, v in node.items() } if isinstance(node, list): return [_transform_cql_geom_literals(item, target_srid) for item in node] @@ -453,9 +452,7 @@ def cql_where( # noqa: C901 if cql is not None: if needs_srid_wrap: - cql = Expr( - _transform_cql_geom_literals(cql.to_json(), target_srid) - ) + cql = Expr(_transform_cql_geom_literals(cql.to_json(), target_srid)) exprs.append(cql) # `ids` filter @@ -472,9 +469,7 @@ def cql_where( # noqa: C901 for prop, val in properties: if not self.get_column(prop): raise InvalidPropertyName(f"Invalid property name: {prop}") - exprs.append( - Expr({"op": "=", "args": [{"property": prop}, val]}) - ) + exprs.append(Expr({"op": "=", "args": [{"property": prop}, val]})) # `bbox` filter — bbox is CRS84 (4326) per OGC API Features. # Reproject the four corner coords to the column's CRS in Python @@ -522,13 +517,9 @@ def cql_where( # noqa: C901 else: start_str, end_str = datetime[0], datetime[1] start = ( - parse_rfc3339(start_str) - if start_str not in ["..", ""] - else None - ) - end = ( - parse_rfc3339(end_str) if end_str not in ["..", ""] else None + parse_rfc3339(start_str) if start_str not in ["..", ""] else None ) + end = parse_rfc3339(end_str) if end_str not in ["..", ""] else None if start is None and end is None: raise InvalidDatetime( @@ -1014,10 +1005,7 @@ async def get_tile( layer = self.table if mvt_settings.set_mvt_layername is True else "default" params.append(layer) - sql = ( - f"WITH t AS ({inner_sql}) " - f"SELECT ST_AsMVT(t.*, ${len(params)}) FROM t" - ) + sql = f"WITH t AS ({inner_sql}) " f"SELECT ST_AsMVT(t.*, ${len(params)}) FROM t" debug_query(sql, *params) tile = await conn.fetchval(sql, *params) From 9ecd4f8c9d9916306f98ec3f38ff415378987f37 Mon Sep 17 00:00:00 2001 From: Bennett Kanuka Date: Wed, 10 Jun 2026 13:16:52 -0400 Subject: [PATCH 09/10] Clarify TMS transformation --- tipg/collections.py | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/tipg/collections.py b/tipg/collections.py index 9da8eb70..88e939e5 100644 --- a/tipg/collections.py +++ b/tipg/collections.py @@ -447,12 +447,12 @@ def cql_where( # noqa: C901 # so the comparison happens in the column's native CRS — PostGIS # folds the transform on the constant side and the spatial index # on the column stays usable. - target_srid = geometry_column.srid if geometry_column is not None else None - needs_srid_wrap = target_srid is not None and target_srid != 4326 + col_srid = geometry_column.srid if geometry_column is not None else None + needs_srid_wrap = col_srid is not None and col_srid != 4326 if cql is not None: if needs_srid_wrap: - cql = Expr(_transform_cql_geom_literals(cql.to_json(), target_srid)) + cql = Expr(_transform_cql_geom_literals(cql.to_json(), col_srid)) exprs.append(cql) # `ids` filter @@ -481,13 +481,13 @@ def cql_where( # noqa: C901 else: west, south, east, north = bbox if needs_srid_wrap: - transformer = TransformerFromCRS(4326, target_srid, always_xy=True) + transformer = TransformerFromCRS(4326, col_srid, always_xy=True) west, south, east, north = transformer.transform_bounds( west, south, east, north ) exprs.append( _s_intersects_bbox( - geometry_column.name, west, south, east, north, srid=target_srid + geometry_column.name, west, south, east, north, srid=col_srid ) ) @@ -569,15 +569,20 @@ def cql_where( # noqa: C901 bounds.right, bounds.top, ) - tms_epsg = tms.crs.to_epsg() or 4326 - tile_target_srid = target_srid if target_srid is not None else 4326 - if tms_epsg != tile_target_srid: + # Bounds are now in TMS CRS + tms_srid = tms.crs.to_epsg() or 4326 + tile_target_srid = col_srid if col_srid is not None else 4326 + + # If the TMS CRS is different from the column's SRID, + # transform the bounds to the column's SRID. + if tms_srid != tile_target_srid: transformer = TransformerFromCRS( - tms_epsg, tile_target_srid, always_xy=True + tms_srid, tile_target_srid, always_xy=True ) west, south, east, north = transformer.transform_bounds( west, south, east, north ) + exprs.append( _s_intersects_bbox( geometry_column.name, From 7b80d526546c2fba5e00ca1b04a6a7373ba2d6b5 Mon Sep 17 00:00:00 2001 From: Bennett Kanuka Date: Wed, 10 Jun 2026 16:01:51 -0400 Subject: [PATCH 10/10] refactor: split large modules into focused packages Break up the three largest modules without changing behavior; old import paths keep working via back-compat shims. - collections.py split into: pgcollection.py (PgCollection), dbmodel.py (data models + abstract Collection base), filter.py (cql2->SQL filter building), and sqlhelpers.py (shared SQL helpers). collections.py now holds the catalog builders (pg_get_collection_index, register_collection_catalog) and re-exports the moved names. - model.py split into the models/ package (common, features, tiles); model.py is now a re-export shim. - factory.py split into the factories/ package (base, features, tiles, endpoints, utils); factory.py is now a re-export shim. --- CHANGES.md | 12 + docs/src/user_guide/factories.md | 10 +- tests/routes/test_tiles.py | 2 +- tipg/collections.py | 1043 +-------------- tipg/dbmodel.py | 295 +++++ tipg/dependencies.py | 2 +- tipg/extensions/viewer.py | 2 +- tipg/factories/__init__.py | 28 + tipg/factories/base.py | 225 ++++ tipg/factories/endpoints.py | 69 + tipg/factories/features.py | 874 +++++++++++++ tipg/factories/tiles.py | 830 ++++++++++++ tipg/factories/utils.py | 97 ++ tipg/factory.py | 2038 +----------------------------- tipg/filter.py | 296 +++++ tipg/middleware.py | 2 +- tipg/model.py | 962 +------------- tipg/models/__init__.py | 79 ++ tipg/models/common.py | 82 ++ tipg/models/features.py | 166 +++ tipg/models/tiles.py | 722 +++++++++++ tipg/pgcollection.py | 409 ++++++ tipg/sqlhelpers.py | 38 + 23 files changed, 4283 insertions(+), 4000 deletions(-) create mode 100644 tipg/dbmodel.py create mode 100644 tipg/factories/__init__.py create mode 100644 tipg/factories/base.py create mode 100644 tipg/factories/endpoints.py create mode 100644 tipg/factories/features.py create mode 100644 tipg/factories/tiles.py create mode 100644 tipg/factories/utils.py create mode 100644 tipg/filter.py create mode 100644 tipg/models/__init__.py create mode 100644 tipg/models/common.py create mode 100644 tipg/models/features.py create mode 100644 tipg/models/tiles.py create mode 100644 tipg/pgcollection.py create mode 100644 tipg/sqlhelpers.py diff --git a/CHANGES.md b/CHANGES.md index fdbe026e..d00cf79e 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -9,6 +9,18 @@ Note: Minor version `0.X.0` update might break the API, It's recommended to pin ## [unreleased] * switch from pygeofilter to cql2 +* split `tipg.collections` into focused modules + * move data models and the abstract `Collection` base class (`Column`, `Parameter`, `Feature`, `ItemList`, `Collection`, `CollectionList`, `Catalog`) to `tipg.dbmodel` + * move cql2 filter building (`cql_where`, ...) to `tipg.filter` + * move shared SQL helpers to `tipg.sqlhelpers` + * move the `PgCollection` implementation to `tipg.pgcollection` + * `tipg.collections` now defines the catalog builders `pg_get_collection_index` and `register_collection_catalog`, and re-exports the models, `PgCollection` and the settings singletons so existing `from tipg.collections import ...` imports keep working +* split the `tipg.model` response models into a `tipg.models` package + * `tipg.models.common` (`Link`), `tipg.models.features` (Features/Common models), `tipg.models.tiles` (tile models) + * `tipg.model` is now a back-compat shim re-exporting from `tipg.models`, so existing `from tipg.model import ...` imports keep working +* split the `tipg.factory` router factories into a `tipg.factories` package + * `tipg.factories.base` (`EndpointsFactory`, `FactoryExtension`), `tipg.factories.features` (`OGCFeaturesFactory`), `tipg.factories.tiles` (`OGCTilesFactory`), `tipg.factories.endpoints` (`Endpoints`), `tipg.factories.utils` (shared render helpers) + * `tipg.factory` is now a back-compat shim re-exporting from `tipg.factories`, so existing `from tipg.factory import ...` imports keep working ## [1.3.1] - 2026-02-26 diff --git a/docs/src/user_guide/factories.md b/docs/src/user_guide/factories.md index e71f3a9f..6ad3a5ec 100644 --- a/docs/src/user_guide/factories.md +++ b/docs/src/user_guide/factories.md @@ -72,9 +72,9 @@ app.include_router(endpoints.router) #### Creation Options -- **collections_dependency** (Callable[..., tipg.collections.CollectionList]): Callable which return a CollectionList dictionary +- **collections_dependency** (Callable[..., tipg.dbmodel.CollectionList]): Callable which return a CollectionList dictionary -- **collection_dependency** (Callable[..., tipg.collections.Collection]): Callable which return a Collection instance +- **collection_dependency** (Callable[..., tipg.dbmodel.Collection]): Callable which return a Collection instance - **with_common** (bool, optional): Create Full OGC Features API set of endpoints with OGC Common endpoints (landing `/` and conformance `/conformance`). Defaults to `True` @@ -111,7 +111,7 @@ app.include_router(endpoints.router) #### Creation Options -- **collection_dependency** (Callable[..., tipg.collections.Collection]): Callable which return a Collection instance +- **collection_dependency** (Callable[..., tipg.dbmodel.Collection]): Callable which return a Collection instance - **supported_tms** (morecantile.TileMatrixSets): morecantile TileMatrixSets instance (holds a set of TileMatrixSet documents) @@ -154,9 +154,9 @@ app.include_router(endpoints.router) #### Creation Options -- **collections_dependency** (Callable[..., tipg.collections.CollectionList]): Callable which return a CollectionList dictionary +- **collections_dependency** (Callable[..., tipg.dbmodel.CollectionList]): Callable which return a CollectionList dictionary -- **collection_dependency** (Callable[..., tipg.collections.Collection]): Callable which return a Collection instance +- **collection_dependency** (Callable[..., tipg.dbmodel.Collection]): Callable which return a Collection instance - **supported_tms** (morecantile.TileMatrixSets): morecantile TileMatrixSets instance (holds a set of TileMatrixSet documents) diff --git a/tests/routes/test_tiles.py b/tests/routes/test_tiles.py index b96a5c48..b328fc04 100644 --- a/tests/routes/test_tiles.py +++ b/tests/routes/test_tiles.py @@ -3,7 +3,7 @@ import mapbox_vector_tile import numpy as np -from tipg.collections import mvt_settings +from tipg.pgcollection import mvt_settings def test_tilejson(app): diff --git a/tipg/collections.py b/tipg/collections.py index 88e939e5..380076d5 100644 --- a/tipg/collections.py +++ b/tipg/collections.py @@ -1,1021 +1,46 @@ -"""tipg.dbmodel: database events.""" +"""tipg.collections: build and register the collection catalog. + +This module also re-exports the models, the ``PgCollection`` implementation and +the settings singletons that used to live here, so existing +``from tipg.collections import ...`` imports keep working after the split into +``tipg.dbmodel`` / ``tipg.pgcollection``. +""" -import abc import datetime -import re -from functools import lru_cache, reduce -from typing import Any, Dict, List, Optional, Tuple, TypedDict, Union +from typing import List, Optional import asyncpg -from ciso8601 import parse_rfc3339 -from cql2 import Expr -from morecantile import Tile, TileMatrixSet -from pydantic import BaseModel, Field, model_validator -from pyproj import Transformer -from tipg.errors import ( - InvalidDatetime, - InvalidDatetimeColumnName, - InvalidGeometryColumnName, - InvalidLimit, - InvalidPropertyName, - MissingDatetimeColumn, - NotFound, -) -from tipg.logger import logger -from tipg.model import Extent -from tipg.settings import ( - DatabaseSettings, - FeaturesSettings, - MVTSettings, - TableConfig, - TableSettings, +from tipg.dbmodel import ( + Catalog, + Collection, + CollectionList, + Column, + Feature, + ItemList, + Parameter, ) +from tipg.pgcollection import PgCollection, features_settings, mvt_settings +from tipg.settings import DatabaseSettings, TableConfig, TableSettings from fastapi import FastAPI -mvt_settings = MVTSettings() -features_settings = FeaturesSettings() - -TransformerFromCRS = lru_cache(Transformer.from_crs) - - -def _quote_ident(name: str) -> str: - """Quote a PostgreSQL identifier (column/table/schema) safely.""" - return '"' + name.replace('"', '""') + '"' - - -_INT_PG_TYPES = frozenset( - { - "smallint", - "integer", - "bigint", - "smallserial", - "serial", - "bigserial", - } -) - - -def _coerce_id(val: str, pg_type: str) -> Union[str, int]: - """Parse a URL-supplied id into the primary-key column's Python type. - - IDs always arrive as strings (URL path or query parameters); the - underlying column is either text or integer. cql2 treats Python ints - and strs as different literal kinds (and PostgreSQL ``= ANY(text[])`` - does not coerce array elements at runtime), so we normalize here. - - A non-integer string against an integer column means the id can't - exist, so map the parse failure onto a 404 rather than letting the - eventual PostgreSQL ``invalid input syntax`` error bubble up as a 500. - """ - if pg_type not in _INT_PG_TYPES: - return val - try: - return int(val) - except ValueError as exc: - raise NotFound(f"Invalid id {val!r} for {pg_type} column.") from exc - - -def _s_intersects_bbox( - prop: str, - west: float, - south: float, - east: float, - north: float, - srid: Optional[int] = None, -) -> Expr: - """Build a cql2 S_INTERSECTS expression against a polygon envelope. - - Coordinates are taken to be in ``srid``; if ``srid`` is non-4326, the - polygon literal is wrapped in ``ST_SetSRID`` so PostGIS does not pick - up ``ST_GeomFromGeoJSON``'s 4326 default. Pass ``srid=None`` or - ``srid=4326`` for the no-op case. - """ - polygon: Any = { - "type": "Polygon", - "coordinates": [ - [ - [west, south], - [east, south], - [east, north], - [west, north], - [west, south], - ] - ], - } - if srid is not None and srid != 4326: - polygon = {"op": "st_setsrid", "args": [polygon, srid]} - return Expr({"op": "s_intersects", "args": [{"property": prop}, polygon]}) - - -_GEOJSON_GEOMETRY_TYPES = frozenset( - { - "Point", - "MultiPoint", - "LineString", - "MultiLineString", - "Polygon", - "MultiPolygon", - "GeometryCollection", - } -) - - -def _transform_cql_geom_literals(node: Any, target_srid: int) -> Any: - """Walk a user-supplied cql2-json tree and wrap every GeoJSON geometry - literal in ``ST_Transform(, target_srid)``. - - Per OGC API Features Part 3 Req 7, geometries in a CQL filter are - CRS84 (≈ EPSG:4326). Wrapping the literal side (rather than the - column side) keeps PostGIS' spatial index on the column usable and - lets the planner fold the transform on the constant once. - """ - if isinstance(node, dict): - if node.get("type") in _GEOJSON_GEOMETRY_TYPES and ( - "coordinates" in node or "geometries" in node - ): - return {"op": "st_transform", "args": [node, target_srid]} - return { - k: _transform_cql_geom_literals(v, target_srid) for k, v in node.items() - } - if isinstance(node, list): - return [_transform_cql_geom_literals(item, target_srid) for item in node] - return node - - -def debug_query(q, *p): - """Utility to print raw statement to use for debugging.""" - - # Escape literal `{` and `}` (cql2 emits GeoJSON literals containing them) - # then turn `$N` placeholders into `{N}` format slots. - qsub = re.sub(r"\$([0-9]+)", r"{\1}", q.replace("{", "{{").replace("}", "}}")) - - def quote_str(s): - """Quote strings.""" - - if s is None: - return "null" - elif isinstance(s, str): - return f"'{s}'" - else: - return s - - p = [quote_str(s) for s in p] - logger.debug(qsub.format(None, *p)) - - -# Links to geojson schema -geojson_schema = { - "GEOMETRY": "https://geojson.org/schema/Geometry.json", - "POINT": "https://geojson.org/schema/Point.json", - "MULTIPOINT": "https://geojson.org/schema/MultiPoint.json", - "LINESTRING": "https://geojson.org/schema/LineString.json", - "MULTILINESTRING": "https://geojson.org/schema/MultiLineString.json", - "POLYGON": "https://geojson.org/schema/Polygon.json", - "MULTIPOLYGON": "https://geojson.org/schema/MultiPolygon.json", - "GEOMETRYCOLLECTION": "https://geojson.org/schema/GeometryCollection.json", -} - - -class Feature(TypedDict, total=False): - """Simple Feature model.""" - - type: str - # Geometry is either a dict or a str (wkt) - geometry: Optional[Union[Dict, str]] - properties: Optional[Dict] - id: Optional[Any] - bbox: Optional[List[float]] - - -class ItemList(TypedDict): - """Items.""" - - items: List[Feature] - matched: Optional[int] - next: Optional[int] - prev: Optional[int] - - -class Column(BaseModel): - """Model for database Column.""" - - name: str - type: str - description: Optional[str] = None - geometry_type: Optional[str] = None - srid: Optional[int] = None - bounds: Optional[List[float]] = None - mindt: Optional[str] = None - maxdt: Optional[str] = None - - @model_validator(mode="before") - def sridbounds_default(cls, values): - """Set default bounds and srid when this is a function.""" - if values.get("geometry_type"): - values["srid"] = values.get("srid", 4326) - values["bounds"] = values.get("bounds", [-180, -90, 180, 90]) - return values - - @property - def json_type(self) -> str: - """Return JSON field type.""" - if self.type.endswith("[]"): - return "array" - - if self.type in [ - "smallint", - "integer", - "bigint", - "decimal", - "numeric", - "real", - "double precision", - "smallserial", - "serial", - "bigserial", - # Float8 is not a Postgres type name but is the name we give - # internally do Double Precision type - # ref: https://github.com/developmentseed/tipg/pull/60/files#r1011863866 - "float8", - ]: - return "number" - - if self.type.startswith("bool"): - return "boolean" - - if any([self.type.startswith("json"), self.type.startswith("geo")]): - return "object" - - return "string" - - @property - def is_geometry(self) -> bool: - """Returns true if this property is a geometry column.""" - return self.type in ("geometry", "geography") - - @property - def is_datetime(self) -> bool: - """Returns true if this property is a datetime column.""" - return self.type in ("timestamp", "timestamptz", "date") - - -class Parameter(Column): - """Model for PostGIS function parameters.""" - - default: Optional[str] = None - - -class Collection(BaseModel, metaclass=abc.ABCMeta): - """Collection Base Class.""" - - type: str - id: str - table: str - title: Optional[str] = None - description: Optional[str] = None - table_columns: List[Column] = [] - properties: List[Column] = [] - id_column: Optional[Column] = None - geometry_column: Optional[Column] = None - datetime_column: Optional[Column] = None - parameters: List[Parameter] = [] - - @property - def extent(self) -> Optional[Extent]: - """Return extent.""" - extent: Dict[str, Any] = {} - if cols := self.geometry_columns: - if len(cols) == 1: - bbox = [cols[0].bounds] - else: - minx, miny, maxx, maxy = zip(*[col.bounds for col in cols]) - bbox = [ - [min(minx), min(miny), max(maxx), max(maxy)], - *[col.bounds for col in cols], - ] - - extent["spatial"] = { - "bbox": bbox, - # The extent calculated in Pg is in WGS84 LON,LAT order - # so we use `CRS84` as CRS - "crs": "http://www.opengis.net/def/crs/OGC/1.3/CRS84", - } - - if cols := [col for col in self.datetime_columns if col.mindt or col.maxdt]: - intervals = [] - if len(cols) == 1: - if cols[0].mindt or cols[0].maxdt: - intervals = [[cols[0].mindt, cols[0].maxdt]] - - else: - mindt = [col.mindt for col in cols if col.mindt] - maxdt = [col.maxdt for col in cols if col.maxdt] - intervals = [ - [min(mindt), max(maxdt)], - *[[col.mindt, col.maxdt] for col in cols if col.mindt or col.maxdt], - ] - - if intervals: - extent["temporal"] = {"interval": intervals} - - if extent: - return Extent(**extent) - - return None - - @property - def bounds(self) -> Optional[List[float]]: - """Return spatial bounds from collection extent.""" - if self.extent and self.extent.spatial: - return self.extent.spatial.bbox[0] - - return None - - @property - def dt_bounds(self) -> Optional[List[str]]: - """Return temporal bounds from collection extent.""" - if self.extent and self.extent.temporal: - return self.extent.temporal.interval[0] - - return None - - @property - def crs(self): - """Return crs of set geometry column.""" - if self.geometry_column: - return f"http://www.opengis.net/def/crs/EPSG/0/{self.geometry_column.srid}" - - @property - def geometry_columns(self) -> List[Column]: - """Return geometry columns.""" - return [c for c in self.table_columns if c.is_geometry] - - @property - def datetime_columns(self) -> List[Column]: - """Return datetime columns.""" - return [c for c in self.table_columns if c.is_datetime] - - def get_geometry_column(self, name: Optional[str] = None) -> Optional[Column]: - """Return the name of the first geometry column.""" - if (not self.geometry_columns) or (name and name.lower() == "none"): - return None - - if name is None: - return self.geometry_column - - for col in self.geometry_columns: - if col.name == name: - return col - - return None - - def get_datetime_column(self, name: Optional[str] = None) -> Optional[Column]: - """Return the Column for either the passed in tstz column or the first tstz column.""" - if not self.datetime_columns: - return None - - if name is None: - return self.datetime_column - - for col in self.datetime_columns: - if col.name == name: - return col - - return None - - def columns(self, properties: Optional[List[str]] = None) -> List[str]: - """Return table columns optionally filtered to only include columns from properties.""" - if properties in [[], [""]]: - return [] - - cols = [ - c.name for c in self.properties if c.type not in ["geometry", "geography"] - ] - if properties is None: - return cols - - return [c for c in cols if c in properties] - - def get_column(self, property_name: str) -> Optional[Column]: - """Return column info.""" - for p in self.properties: - if p.name == property_name: - return p - - return None - - @property - def queryables(self) -> Dict: - """Return the queryables.""" - if self.geometry_columns: - geoms = { - col.name: {"$ref": geojson_schema.get(col.geometry_type.upper(), "")} - for col in self.geometry_columns - } - else: - geoms = {} - - props = { - col.name: {"name": col.name, "type": col.json_type} - for col in self.properties - if col.name not in geoms - } - - return {**geoms, **props} - - def cql_where( # noqa: C901 - self, - ids: Optional[List[str]] = None, - datetime: Optional[List[str]] = None, - bbox: Optional[List[float]] = None, - properties: Optional[List[Tuple[str, Any]]] = None, - cql: Optional[Expr] = None, - geom: Optional[str] = None, - dt: Optional[str] = None, - tile: Optional[Tile] = None, - tms: Optional[TileMatrixSet] = None, - ) -> Optional[Expr]: - """Construct WHERE query as a cql2 Expr.""" - exprs: List[Expr] = [] - geometry_column = self.get_geometry_column(geom) - - # Per OGC API Features Part 3 Req 7, geometries in `cql` and `bbox` - # are CRS84 (≈ EPSG:4326). When the stored column is in another - # SRID, wrap their literals in `ST_Transform(literal, )` - # so the comparison happens in the column's native CRS — PostGIS - # folds the transform on the constant side and the spatial index - # on the column stays usable. - col_srid = geometry_column.srid if geometry_column is not None else None - needs_srid_wrap = col_srid is not None and col_srid != 4326 - - if cql is not None: - if needs_srid_wrap: - cql = Expr(_transform_cql_geom_literals(cql.to_json(), col_srid)) - exprs.append(cql) - - # `ids` filter - if ids: - id_prop = {"property": self.id_column.name} - typed_ids = [_coerce_id(i, self.id_column.type) for i in ids] - if len(typed_ids) == 1: - exprs.append(Expr({"op": "=", "args": [id_prop, typed_ids[0]]})) - else: - exprs.append(Expr({"op": "in", "args": [id_prop, typed_ids]})) - - # `properties` filter - if properties is not None: - for prop, val in properties: - if not self.get_column(prop): - raise InvalidPropertyName(f"Invalid property name: {prop}") - exprs.append(Expr({"op": "=", "args": [{"property": prop}, val]})) - - # `bbox` filter — bbox is CRS84 (4326) per OGC API Features. - # Reproject the four corner coords to the column's CRS in Python - # so the polygon literal already carries target-CRS coordinates; - # ST_SetSRID then just tags it, no PostGIS-side transform needed. - if bbox is not None and geometry_column is not None: - if len(bbox) == 6: - west, south, _, east, north, _ = bbox - else: - west, south, east, north = bbox - if needs_srid_wrap: - transformer = TransformerFromCRS(4326, col_srid, always_xy=True) - west, south, east, north = transformer.transform_bounds( - west, south, east, north - ) - exprs.append( - _s_intersects_bbox( - geometry_column.name, west, south, east, north, srid=col_srid - ) - ) - - # `datetime` filter - if datetime: - if not self.datetime_columns: - raise MissingDatetimeColumn( - "Must have timestamp/timestamptz/date typed column to filter with datetime." - ) - - datetime_column = self.get_datetime_column(dt) - if not datetime_column: - raise InvalidDatetimeColumnName(f"Invalid Datetime Column: {dt}.") - - dt_prop = {"property": datetime_column.name} - - if len(datetime) == 1: - parse_rfc3339(datetime[0]) - exprs.append( - Expr( - { - "op": "=", - "args": [dt_prop, {"timestamp": datetime[0]}], - } - ) - ) - else: - start_str, end_str = datetime[0], datetime[1] - start = ( - parse_rfc3339(start_str) if start_str not in ["..", ""] else None - ) - end = parse_rfc3339(end_str) if end_str not in ["..", ""] else None - - if start is None and end is None: - raise InvalidDatetime( - "Double open-ended datetime intervals are not allowed." - ) - - if start is not None and end is not None and start > end: - raise InvalidDatetime( - "Start datetime cannot be before end datetime." - ) - - if start is not None: - exprs.append( - Expr( - { - "op": ">=", - "args": [dt_prop, {"timestamp": start_str}], - } - ) - ) - if end is not None: - # TODO: Understand the correct way to handle inclusive/exclusive end - # Closed interval uses exclusive upper bound to keep - # the half-open `[start, end)` semantics tipg has always - # used; the open-ended `../` form remains inclusive - # (`<=`) - op = "<" if start is not None else "<=" - exprs.append( - Expr( - { - "op": op, - "args": [dt_prop, {"timestamp": end_str}], - } - ) - ) - - # `tile` envelope filter — reproject tile bounds directly from the - # TMS CRS to the stored column's CRS (skipping the 4326 round-trip) - # and tag the resulting polygon literal with `ST_SetSRID` so it - # carries the column's SRID rather than the 4326 default that - # `ST_GeomFromGeoJSON` would otherwise apply. - if tile and tms and geometry_column: - bounds = tms.xy_bounds(tile) - west, south, east, north = ( - bounds.left, - bounds.bottom, - bounds.right, - bounds.top, - ) - # Bounds are now in TMS CRS - tms_srid = tms.crs.to_epsg() or 4326 - tile_target_srid = col_srid if col_srid is not None else 4326 - - # If the TMS CRS is different from the column's SRID, - # transform the bounds to the column's SRID. - if tms_srid != tile_target_srid: - transformer = TransformerFromCRS( - tms_srid, tile_target_srid, always_xy=True - ) - west, south, east, north = transformer.transform_bounds( - west, south, east, north - ) - - exprs.append( - _s_intersects_bbox( - geometry_column.name, - west, - south, - east, - north, - srid=tile_target_srid, - ) - ) - - if exprs: - # NOTE: do not call .reduce() — cql2-rs constant-folds some - # predicates (e.g. `"numeric" IS NULL`) incorrectly. - # TODO: Open a bug in cql2-rs for this. - return reduce(lambda x, y: x + y, exprs) - - return None - - @abc.abstractmethod - async def features(self, *args, **kwargs) -> ItemList: - """Get Items.""" - ... - - @abc.abstractmethod - async def get_tile(self, *args, **kwargs) -> bytes: - """Get MVT Tile.""" - ... - - -class CollectionList(TypedDict): - """Collections.""" - - collections: List[Collection] - matched: Optional[int] - next: Optional[int] - prev: Optional[int] - - -class Catalog(TypedDict): - """Internal Collection Catalog.""" - - collections: Dict[str, Collection] - last_updated: datetime.datetime - - -class PgCollection(Collection): - """Model for DB Table and Function.""" - - dbschema: str = Field(alias="schema") - - @property - def _qualified_name(self) -> str: - """SQL-quoted ``"schema"."table"`` identifier for this collection.""" - return f"{_quote_ident(self.dbschema)}.{_quote_ident(self.table)}" - - def _select_property_columns(self, properties: Optional[List[str]]) -> List[str]: - """Quoted column-name fragments for the property columns to SELECT.""" - return [_quote_ident(c) for c in self.columns(properties)] - - def _select_id(self) -> str: - """SQL fragment that selects the primary-key column as ``tipg_id``. - - Falls back to ``ROW_NUMBER()`` when the collection has no primary key. - """ - if self.id_column: - return f"{_quote_ident(self.id_column.name)} AS tipg_id" - return "ROW_NUMBER() OVER () AS tipg_id" - - def _geom_expr( - self, - geometry_column: Optional[Column], - bbox_only: Optional[bool], - simplify: Optional[float], - ) -> Optional[str]: - """Build the SQL expression for the geometry column (reprojected to - 4326 if needed, optionally bbox-only or simplified). - """ - if geometry_column is None: - return None - - g = f"CAST({_quote_ident(geometry_column.name)} AS geometry)" - - if geometry_column.srid != 4326: - g = f"ST_Transform({g}, 4326)" - - if bbox_only: - g = f"ST_Envelope({g})" - elif simplify: - s = float(simplify) - g = f"ST_SnapToGrid(ST_Simplify({g}, {s}), {s})" - - return g - - def _select( - self, - properties: Optional[List[str]], - geometry_column: Optional[Column], - bbox_only: Optional[bool], - simplify: Optional[float], - geom_as_wkt: bool = False, - ) -> str: - """Build the SELECT clause for the main features query.""" - cols = self._select_property_columns(properties) - cols.append(self._select_id()) - - geom = self._geom_expr(geometry_column, bbox_only, simplify) - if geom_as_wkt: - cols.append( - f"ST_AsEWKT({geom}) AS tipg_geom" - if geom - else "CAST(NULL AS text) AS tipg_geom" - ) - else: - cols.append( - f"CAST(ST_AsGeoJSON({geom}) AS json) AS tipg_geom" - if geom - else "CAST(NULL AS json) AS tipg_geom" - ) - - return "SELECT " + ", ".join(cols) - - def _select_mvt( - self, - properties: Optional[List[str]], - geometry_column: Column, - tms: TileMatrixSet, - tile: Tile, - ) -> str: - """Build the SELECT clause that emits an MVT geometry per row.""" - cols = self._select_property_columns(properties) - - geom = f"CAST({_quote_ident(geometry_column.name)} AS geometry)" - - # For tiles that fall outside the TMS's natural domain (e.g. an - # over-zoomed corner tile), clip the source geometry to the TMS's - # geographic bbox before reprojecting — otherwise ST_AsMVTGeom can - # produce garbage near the antimeridian / poles. ``tms.bbox`` is in - # the TMS's ``geographic_crs``, which is NOT always EPSG:4326 (e.g. - # CanadianNAD83_LCC uses 4269, EuropeanETRS89_LAEAQuad uses 4258, - # NZTM2000Quad uses 4167). When the geographic CRS has no EPSG code - # (WorldCRS84Quad → OGC:CRS84), 4326 is a safe fallback since the - # coordinate values are identical (only axis order differs, and - # PostGIS treats 4326 as lon/lat). - if not tms.is_valid(tile): - west, south, east, north = tms.bbox - geo_srid = tms.geographic_crs.to_epsg() or 4326 - geom = ( - f"ST_Intersection(" - f"ST_MakeEnvelope({west}, {south}, {east}, {north}, {geo_srid}), " - f"ST_Transform({geom}, {geo_srid})" - f")" - ) - - # Reproject to TMS CRS — prefer EPSG code, fall back to PROJ string. - if tms_srid := tms.crs.to_epsg(): - transformed = f"ST_Transform({geom}, {tms_srid})" - else: - tms_proj = tms.crs.to_proj4().replace("'", "''") - transformed = f"ST_Transform({geom}, '{tms_proj}')" - - bbox = tms.xy_bounds(tile) - envelope = ( - f"ST_Segmentize(" - f"ST_MakeEnvelope({bbox.left}, {bbox.bottom}, {bbox.right}, {bbox.top}), " - f"{bbox.right - bbox.left}" - f")" - ) - cols.append( - f"ST_AsMVTGeom(" - f"{transformed}, {envelope}, " - f"{int(mvt_settings.tile_resolution)}, " - f"{int(mvt_settings.tile_buffer)}, " - f"{'TRUE' if mvt_settings.tile_clip else 'FALSE'}" - f") AS geom" - ) - - return "SELECT " + ", ".join(cols) - - def _from( - self, - function_parameters: Optional[Dict[str, str]], - params: List[Any], - ) -> str: - """Build the FROM clause. Function-table parameters are appended to - ``params`` and referenced via ``$N`` placeholders. - """ - name = self._qualified_name - if self.type != "Function": - return f"FROM {name}" - - if not function_parameters: - return f"FROM {name}()" - - args = [] - for p in self.parameters: - if p.name in function_parameters: - params.append(function_parameters[p.name]) - # Double cast (value → text → target type): asyncpg sends - # Python strs as text, and the explicit cast to ``p.type`` - # then coerces to whatever the function signature expects. - args.append(f"CAST(CAST(${len(params)} AS text) AS {p.type})") - return f"FROM {name}({', '.join(args)})" - - def _sortby(self, sortby: Optional[str]) -> str: - """Build the ORDER BY clause.""" - sorts = [] - if sortby: - for s in sortby.strip().split(","): - parts = re.match("^(?P[+-]?)(?P.*)$", s).groupdict() # type:ignore - - direction = parts["direction"] - column = parts["column"].strip() - if not self.get_column(column): - raise InvalidPropertyName(f"Property {column} does not exist.") - col_sql = _quote_ident(column) - sorts.append(f"{col_sql} DESC" if direction == "-" else col_sql) - elif self.id_column is not None: - sorts.append(_quote_ident(self.id_column.name)) - else: - sorts.append(_quote_ident(self.properties[0].name)) - - return "ORDER BY " + ", ".join(sorts) - - @staticmethod - def _where_clause(where: Optional[Expr]) -> str: - """Render a cql2 ``Expr`` (or ``None``) as a WHERE clause.""" - if where is None: - return "WHERE TRUE" - return f"WHERE {where.to_sql()}" - - async def _features_query( - self, - conn: asyncpg.Connection, - *, - where: Optional[Expr] = None, - sortby: Optional[str] = None, - properties: Optional[List[str]] = None, - geom: Optional[str] = None, - limit: Optional[int] = None, - offset: Optional[int] = None, - bbox_only: Optional[bool] = None, - simplify: Optional[float] = None, - geom_as_wkt: bool = False, - function_parameters: Optional[Dict[str, str]], - ): - """Build and run the Features query, yielding each row as a Feature.""" - limit = limit or features_settings.default_features_limit - offset = offset or 0 - params: List[Any] = [] - - sql = " ".join( - [ - self._select( - properties=properties, - geometry_column=self.get_geometry_column(geom), - bbox_only=bbox_only, - simplify=simplify, - geom_as_wkt=geom_as_wkt, - ), - self._from(function_parameters, params), - self._where_clause(where), - self._sortby(sortby), - f"LIMIT {int(limit)}", - f"OFFSET {int(offset)}", - ] - ) - - for r in await conn.fetch(sql, *params): - props = dict(r) - g = props.pop("tipg_geom") - id = props.pop("tipg_id") - feature = Feature(type="Feature", geometry=g, id=id, properties=props) - yield feature - - async def _features_count_query( - self, - conn: asyncpg.Connection, - *, - where: Optional[Expr] = None, - function_parameters: Optional[Dict[str, str]], - ) -> int: - """Run the COUNT(*) query for the current filter.""" - params: List[Any] = [] - sql = " ".join( - [ - "SELECT COUNT(*)", - self._from(function_parameters, params), - self._where_clause(where), - ] - ) - return await conn.fetchval(sql, *params) - - async def features( - self, - conn: asyncpg.Connection, - *, - ids_filter: Optional[List[str]] = None, - bbox_filter: Optional[List[float]] = None, - datetime_filter: Optional[List[str]] = None, - properties_filter: Optional[List[Tuple[str, str]]] = None, - cql_filter: Optional[Expr] = None, - sortby: Optional[str] = None, - properties: Optional[List[str]] = None, - geom: Optional[str] = None, - dt: Optional[str] = None, - limit: Optional[int] = None, - offset: Optional[int] = None, - bbox_only: Optional[bool] = None, - simplify: Optional[float] = None, - geom_as_wkt: bool = False, - function_parameters: Optional[Dict[str, str]] = None, - ) -> ItemList: - """Build and run Pg query.""" - limit = limit or features_settings.default_features_limit - offset = offset or 0 - - function_parameters = function_parameters or {} - - if geom and geom.lower() != "none" and not self.get_geometry_column(geom): - raise InvalidGeometryColumnName(f"Invalid Geometry Column: {geom}.") - - if limit and limit > features_settings.max_features_per_query: - raise InvalidLimit( - f"Limit can not be set higher than the `tipg_max_features_per_query` setting of {features_settings.max_features_per_query}" - ) - - where_filter = self.cql_where( - ids=ids_filter, - datetime=datetime_filter, - bbox=bbox_filter, - properties=properties_filter, - cql=cql_filter, - geom=geom, - dt=dt, - ) - - matched = await self._features_count_query( - conn, - where=where_filter, - function_parameters=function_parameters, - ) - - features = [ - f - async for f in self._features_query( - conn, - where=where_filter, - sortby=sortby, - properties=properties, - geom=geom, - limit=limit, - offset=offset, - bbox_only=bbox_only, - simplify=simplify, - geom_as_wkt=geom_as_wkt, - function_parameters=function_parameters, - ) - ] - returned = len(features) - - return ItemList( - items=features, - matched=matched, - next=offset + returned if matched - returned > offset else None, - prev=max(offset - limit, 0) if offset else None, - ) - - async def get_tile( - self, - conn: asyncpg.Connection, - *, - tms: TileMatrixSet, - tile: Tile, - ids_filter: Optional[List[str]] = None, - bbox_filter: Optional[List[float]] = None, - datetime_filter: Optional[List[str]] = None, - properties_filter: Optional[List[Tuple[str, str]]] = None, - function_parameters: Optional[Dict[str, str]] = None, - cql_filter: Optional[Expr] = None, - sortby: Optional[str] = None, - properties: Optional[List[str]] = None, - geom: Optional[str] = None, - dt: Optional[str] = None, - limit: Optional[int] = None, - ): - """Build query to get Vector Tile.""" - limit = limit or mvt_settings.max_features_per_tile - - geometry_column = self.get_geometry_column(geom) - if not geometry_column: - raise InvalidGeometryColumnName(f"Invalid Geometry Column Name {geom}") - - if limit > mvt_settings.max_features_per_tile: - raise InvalidLimit( - f"Limit can not be set higher than the `tipg_max_features_per_tile` setting of {mvt_settings.max_features_per_tile}" - ) - - where_filter = self.cql_where( - ids=ids_filter, - datetime=datetime_filter, - bbox=bbox_filter, - properties=properties_filter, - cql=cql_filter, - geom=geom, - dt=dt, - tms=tms, - tile=tile, - ) - - params: List[Any] = [] - inner_sql = " ".join( - [ - self._select_mvt( - properties=properties, - geometry_column=geometry_column, - tms=tms, - tile=tile, - ), - self._from(function_parameters, params), - self._where_clause(where_filter), - f"LIMIT {int(limit)}", - ] - ) - - layer = self.table if mvt_settings.set_mvt_layername is True else "default" - params.append(layer) - sql = f"WITH t AS ({inner_sql}) " f"SELECT ST_AsMVT(t.*, ${len(params)}) FROM t" - debug_query(sql, *params) - - tile = await conn.fetchval(sql, *params) - - return bytes(tile) +__all__ = [ + # re-exported for backwards compatibility (see module docstring) + "Catalog", + "Collection", + "CollectionList", + "Column", + "Feature", + "ItemList", + "Parameter", + "PgCollection", + "features_settings", + "mvt_settings", + # defined here + "pg_get_collection_index", + "register_collection_catalog", +] async def pg_get_collection_index( # noqa: C901 diff --git a/tipg/dbmodel.py b/tipg/dbmodel.py new file mode 100644 index 00000000..fca7dd2b --- /dev/null +++ b/tipg/dbmodel.py @@ -0,0 +1,295 @@ +"""tipg.dbmodel: database-domain models and the abstract Collection base.""" + +import abc +import datetime +from typing import Any, Dict, List, Optional, TypedDict, Union + +from pydantic import BaseModel, model_validator + +from tipg.model import Extent + +# Links to geojson schema +geojson_schema = { + "GEOMETRY": "https://geojson.org/schema/Geometry.json", + "POINT": "https://geojson.org/schema/Point.json", + "MULTIPOINT": "https://geojson.org/schema/MultiPoint.json", + "LINESTRING": "https://geojson.org/schema/LineString.json", + "MULTILINESTRING": "https://geojson.org/schema/MultiLineString.json", + "POLYGON": "https://geojson.org/schema/Polygon.json", + "MULTIPOLYGON": "https://geojson.org/schema/MultiPolygon.json", + "GEOMETRYCOLLECTION": "https://geojson.org/schema/GeometryCollection.json", +} + + +class Feature(TypedDict, total=False): + """Simple Feature model.""" + + type: str + # Geometry is either a dict or a str (wkt) + geometry: Optional[Union[Dict, str]] + properties: Optional[Dict] + id: Optional[Any] + bbox: Optional[List[float]] + + +class ItemList(TypedDict): + """Items.""" + + items: List[Feature] + matched: Optional[int] + next: Optional[int] + prev: Optional[int] + + +class Column(BaseModel): + """Model for database Column.""" + + name: str + type: str + description: Optional[str] = None + geometry_type: Optional[str] = None + srid: Optional[int] = None + bounds: Optional[List[float]] = None + mindt: Optional[str] = None + maxdt: Optional[str] = None + + @model_validator(mode="before") + def sridbounds_default(cls, values): + """Set default bounds and srid when this is a function.""" + if values.get("geometry_type"): + values["srid"] = values.get("srid", 4326) + values["bounds"] = values.get("bounds", [-180, -90, 180, 90]) + return values + + @property + def json_type(self) -> str: + """Return JSON field type.""" + if self.type.endswith("[]"): + return "array" + + if self.type in [ + "smallint", + "integer", + "bigint", + "decimal", + "numeric", + "real", + "double precision", + "smallserial", + "serial", + "bigserial", + # Float8 is not a Postgres type name but is the name we give + # internally do Double Precision type + # ref: https://github.com/developmentseed/tipg/pull/60/files#r1011863866 + "float8", + ]: + return "number" + + if self.type.startswith("bool"): + return "boolean" + + if any([self.type.startswith("json"), self.type.startswith("geo")]): + return "object" + + return "string" + + @property + def is_geometry(self) -> bool: + """Returns true if this property is a geometry column.""" + return self.type in ("geometry", "geography") + + @property + def is_datetime(self) -> bool: + """Returns true if this property is a datetime column.""" + return self.type in ("timestamp", "timestamptz", "date") + + +class Parameter(Column): + """Model for PostGIS function parameters.""" + + default: Optional[str] = None + + +class Collection(BaseModel, metaclass=abc.ABCMeta): + """Collection Base Class.""" + + type: str + id: str + table: str + title: Optional[str] = None + description: Optional[str] = None + table_columns: List[Column] = [] + properties: List[Column] = [] + id_column: Optional[Column] = None + geometry_column: Optional[Column] = None + datetime_column: Optional[Column] = None + parameters: List[Parameter] = [] + + @property + def extent(self) -> Optional[Extent]: + """Return extent.""" + extent: Dict[str, Any] = {} + if cols := self.geometry_columns: + if len(cols) == 1: + bbox = [cols[0].bounds] + else: + minx, miny, maxx, maxy = zip(*[col.bounds for col in cols]) + bbox = [ + [min(minx), min(miny), max(maxx), max(maxy)], + *[col.bounds for col in cols], + ] + + extent["spatial"] = { + "bbox": bbox, + # The extent calculated in Pg is in WGS84 LON,LAT order + # so we use `CRS84` as CRS + "crs": "http://www.opengis.net/def/crs/OGC/1.3/CRS84", + } + + if cols := [col for col in self.datetime_columns if col.mindt or col.maxdt]: + intervals = [] + if len(cols) == 1: + if cols[0].mindt or cols[0].maxdt: + intervals = [[cols[0].mindt, cols[0].maxdt]] + + else: + mindt = [col.mindt for col in cols if col.mindt] + maxdt = [col.maxdt for col in cols if col.maxdt] + intervals = [ + [min(mindt), max(maxdt)], + *[[col.mindt, col.maxdt] for col in cols if col.mindt or col.maxdt], + ] + + if intervals: + extent["temporal"] = {"interval": intervals} + + if extent: + return Extent(**extent) + + return None + + @property + def bounds(self) -> Optional[List[float]]: + """Return spatial bounds from collection extent.""" + if self.extent and self.extent.spatial: + return self.extent.spatial.bbox[0] + + return None + + @property + def dt_bounds(self) -> Optional[List[str]]: + """Return temporal bounds from collection extent.""" + if self.extent and self.extent.temporal: + return self.extent.temporal.interval[0] + + return None + + @property + def crs(self): + """Return crs of set geometry column.""" + if self.geometry_column: + return f"http://www.opengis.net/def/crs/EPSG/0/{self.geometry_column.srid}" + + @property + def geometry_columns(self) -> List[Column]: + """Return geometry columns.""" + return [c for c in self.table_columns if c.is_geometry] + + @property + def datetime_columns(self) -> List[Column]: + """Return datetime columns.""" + return [c for c in self.table_columns if c.is_datetime] + + def get_geometry_column(self, name: Optional[str] = None) -> Optional[Column]: + """Return the name of the first geometry column.""" + if (not self.geometry_columns) or (name and name.lower() == "none"): + return None + + if name is None: + return self.geometry_column + + for col in self.geometry_columns: + if col.name == name: + return col + + return None + + def get_datetime_column(self, name: Optional[str] = None) -> Optional[Column]: + """Return the Column for either the passed in tstz column or the first tstz column.""" + if not self.datetime_columns: + return None + + if name is None: + return self.datetime_column + + for col in self.datetime_columns: + if col.name == name: + return col + + return None + + def columns(self, properties: Optional[List[str]] = None) -> List[str]: + """Return table columns optionally filtered to only include columns from properties.""" + if properties in [[], [""]]: + return [] + + cols = [ + c.name for c in self.properties if c.type not in ["geometry", "geography"] + ] + if properties is None: + return cols + + return [c for c in cols if c in properties] + + def get_column(self, property_name: str) -> Optional[Column]: + """Return column info.""" + for p in self.properties: + if p.name == property_name: + return p + + return None + + @property + def queryables(self) -> Dict: + """Return the queryables.""" + if self.geometry_columns: + geoms = { + col.name: {"$ref": geojson_schema.get(col.geometry_type.upper(), "")} + for col in self.geometry_columns + } + else: + geoms = {} + + props = { + col.name: {"name": col.name, "type": col.json_type} + for col in self.properties + if col.name not in geoms + } + + return {**geoms, **props} + + @abc.abstractmethod + async def features(self, *args, **kwargs) -> ItemList: + """Get Items.""" + ... + + @abc.abstractmethod + async def get_tile(self, *args, **kwargs) -> bytes: + """Get MVT Tile.""" + ... + + +class CollectionList(TypedDict): + """Collections.""" + + collections: List[Collection] + matched: Optional[int] + next: Optional[int] + prev: Optional[int] + + +class Catalog(TypedDict): + """Internal Collection Catalog.""" + + collections: Dict[str, Collection] + last_updated: datetime.datetime diff --git a/tipg/dependencies.py b/tipg/dependencies.py index 42938bd7..0ac86131 100644 --- a/tipg/dependencies.py +++ b/tipg/dependencies.py @@ -8,7 +8,7 @@ from morecantile import Tile from morecantile import tms as default_tms -from tipg.collections import Catalog, Collection, CollectionList +from tipg.dbmodel import Catalog, Collection, CollectionList from tipg.errors import InvalidBBox, MissingCollectionCatalog, MissingFunctionParameter from tipg.resources.enums import MediaType from tipg.settings import TMSSettings diff --git a/tipg/extensions/viewer.py b/tipg/extensions/viewer.py index 9f37c59a..f6d8c43c 100644 --- a/tipg/extensions/viewer.py +++ b/tipg/extensions/viewer.py @@ -4,7 +4,7 @@ from typing import Annotated, Optional from urllib.parse import urlencode -from tipg.collections import Collection +from tipg.dbmodel import Collection from tipg.factory import EndpointsFactory, FactoryExtension from fastapi import Depends, Query diff --git a/tipg/factories/__init__.py b/tipg/factories/__init__.py new file mode 100644 index 00000000..2cee6a65 --- /dev/null +++ b/tipg/factories/__init__.py @@ -0,0 +1,28 @@ +"""tipg.factories: router factories. + +The factory classes are organized into submodules: + +* ``base`` — ``EndpointsFactory`` (abstract) and ``FactoryExtension`` +* ``features`` — ``OGCFeaturesFactory`` +* ``tiles`` — ``OGCTilesFactory`` +* ``endpoints`` — ``Endpoints`` (combined Features + Tiles) +* ``utils`` — shared rendering helpers + +All public names are re-exported here. +""" + +from tipg.factories.base import EndpointsFactory, FactoryExtension +from tipg.factories.endpoints import Endpoints +from tipg.factories.features import OGCFeaturesFactory +from tipg.factories.tiles import OGCTilesFactory +from tipg.factories.utils import create_csv_rows, create_html_response + +__all__ = [ + "Endpoints", + "EndpointsFactory", + "FactoryExtension", + "OGCFeaturesFactory", + "OGCTilesFactory", + "create_csv_rows", + "create_html_response", +] diff --git a/tipg/factories/base.py b/tipg/factories/base.py new file mode 100644 index 00000000..832b939f --- /dev/null +++ b/tipg/factories/base.py @@ -0,0 +1,225 @@ +"""tipg.factories.base: abstract factory base classes.""" + +import abc +from dataclasses import dataclass, field +from typing import Annotated, Any, Callable, List, Optional + +from tipg import model +from tipg.dbmodel import Collection +from tipg.dependencies import CollectionParams, OutputType +from tipg.factories.utils import DEFAULT_TEMPLATES, create_html_response +from tipg.resources.enums import MediaType +from tipg.resources.response import ORJSONResponse + +from fastapi import APIRouter, Depends + +from starlette.requests import Request +from starlette.routing import compile_path, replace_params +from starlette.templating import Jinja2Templates, _TemplateResponse + +COMMON_CONFORMS = [ + "http://www.opengis.net/spec/ogcapi-common-1/1.0/conf/core", + "http://www.opengis.net/spec/ogcapi-common-1/1.0/conf/landingPage", + "http://www.opengis.net/spec/ogcapi-common-2/1.0/conf/collections", + "http://www.opengis.net/spec/ogcapi-common-2/1.0/conf/simple-query", + "http://www.opengis.net/spec/ogcapi-common-1/1.0/conf/json", + "http://www.opengis.net/spec/ogcapi-common-1/1.0/conf/html", + "http://www.opengis.net/spec/ogcapi-common-1/1.0/conf/oas30", +] + + +@dataclass +class FactoryExtension(metaclass=abc.ABCMeta): + """Factory Extension.""" + + @abc.abstractmethod + def register(self, factory: "EndpointsFactory"): + """Register extension to the factory.""" + ... + + +# ref: https://github.com/python/mypy/issues/5374 +@dataclass # type: ignore +class EndpointsFactory(metaclass=abc.ABCMeta): + """Endpoints Factory.""" + + # FastAPI router + router: APIRouter = field(default_factory=APIRouter) + + # collection dependency + collection_dependency: Callable[..., Collection] = CollectionParams + + # Router Prefix is needed to find the path for routes when prefixed + # e.g if you mount the route with `/foo` prefix, set router_prefix to foo + router_prefix: str = "" + + extensions: List[FactoryExtension] = field(default_factory=list) + + templates: Jinja2Templates = DEFAULT_TEMPLATES + + # Full application with Landing and Conformance + with_common: bool = True + + title: str = "OGC API" + + def __post_init__(self): + """Post Init: register route and configure specific options.""" + if self.with_common: + self._landing_route() + self._conformance_route() + + self.register_routes() + + # Register Extensions + for ext in self.extensions: + ext.register(self) + + def url_for(self, request: Request, name: str, **path_params: Any) -> str: + """Return full url (with prefix) for a specific handler.""" + url_path = self.router.url_path_for(name, **path_params) + + base_url = str(request.base_url) + if self.router_prefix: + prefix = self.router_prefix.lstrip("/") + # If we have prefix with custom path param we check and replace them with + # the path params provided + if "{" in prefix: + _, path_format, param_convertors = compile_path(prefix) + prefix, _ = replace_params( + path_format, param_convertors, request.path_params.copy() + ) + base_url += prefix + + return str(url_path.make_absolute_url(base_url=base_url)) + + def _create_html_response( + self, + request: Request, + data: Any, + template_name: str, + title: Optional[str] = None, + **kwargs: Any, + ) -> _TemplateResponse: + return create_html_response( + request, + data, + templates=self.templates, + template_name=template_name, + title=title, + router_prefix=self.router_prefix, + **kwargs, + ) + + @abc.abstractmethod + def register_routes(self): + """Register factory Routes.""" + ... + + @property + @abc.abstractmethod + def conforms_to(self) -> List[str]: + """Endpoints conformances.""" + ... + + @abc.abstractmethod + def links(self, request: Request) -> List[model.Link]: + """Register factory Routes.""" + ... + + def _conformance_route(self): + """Register Conformance (/conformance) route.""" + + @self.router.get( + "/conformance", + response_model=model.Conformance, + response_model_exclude_none=True, + response_class=ORJSONResponse, + responses={ + 200: { + "content": { + MediaType.json.value: {}, + MediaType.html.value: {}, + } + }, + }, + tags=["OGC Common"], + ) + def conformance( + request: Request, + output_type: Annotated[Optional[MediaType], Depends(OutputType)] = None, + ): + """Get conformance.""" + data = model.Conformance(conformsTo=[*COMMON_CONFORMS, *self.conforms_to]) + + if output_type == MediaType.html: + return self._create_html_response( + request, + data.model_dump(exclude_none=True, mode="json"), + template_name="conformance", + ) + + return data + + def _landing_route(self): + """Register Landing (/) and Conformance (/conformance) routes.""" + + @self.router.get( + "/", + response_model=model.Landing, + response_model_exclude_none=True, + response_class=ORJSONResponse, + responses={ + 200: { + "content": { + MediaType.json.value: {}, + MediaType.html.value: {}, + } + }, + }, + tags=["OGC Common"], + ) + def landing( + request: Request, + output_type: Annotated[Optional[MediaType], Depends(OutputType)] = None, + ): + """Get landing page.""" + data = model.Landing( + title=self.title, + links=[ + model.Link( + title="Landing Page", + href=self.url_for(request, "landing"), + type=MediaType.json, + rel="self", + ), + model.Link( + title="the API definition (JSON)", + href=str(request.url_for("openapi")), + type=MediaType.openapi30_json, + rel="service-desc", + ), + model.Link( + title="the API documentation", + href=str(request.url_for("swagger_ui_html")), + type=MediaType.html, + rel="service-doc", + ), + model.Link( + title="Conformance", + href=self.url_for(request, "conformance"), + type=MediaType.json, + rel="conformance", + ), + *self.links(request), + ], + ) + + if output_type == MediaType.html: + return self._create_html_response( + request, + data.model_dump(exclude_none=True, mode="json"), + template_name="landing", + title=self.title, + ) + + return data diff --git a/tipg/factories/endpoints.py b/tipg/factories/endpoints.py new file mode 100644 index 00000000..71772ba3 --- /dev/null +++ b/tipg/factories/endpoints.py @@ -0,0 +1,69 @@ +"""tipg.factories.endpoints: combined OGC Features + Tiles factory.""" + +from dataclasses import dataclass, field +from typing import Callable, List + +from morecantile import tms as default_tms +from morecantile.defaults import TileMatrixSets + +from tipg import model +from tipg.dbmodel import CollectionList +from tipg.dependencies import CollectionsParams +from tipg.factories.base import EndpointsFactory +from tipg.factories.features import OGCFeaturesFactory +from tipg.factories.tiles import OGCTilesFactory + +from starlette.requests import Request + + +@dataclass +class Endpoints(EndpointsFactory): + """OGC Features and Tiles Endpoints Factory.""" + + # OGC Features dependency + collections_dependency: Callable[..., CollectionList] = CollectionsParams + + # OGC Tiles dependency + supported_tms: TileMatrixSets = default_tms + with_tiles_viewer: bool = True + + ogc_features: OGCFeaturesFactory = field(init=False) + ogc_tiles: OGCTilesFactory = field(init=False) + + @property + def conforms_to(self) -> List[str]: + """Endpoints conformances.""" + return [ + *self.ogc_features.conforms_to, + *self.ogc_tiles.conforms_to, + ] + + def links(self, request: Request) -> List[model.Link]: + """List of available links.""" + return [ + *self.ogc_features.links(request), + *self.ogc_tiles.links(request), + ] + + def register_routes(self): + """Register factory Routes.""" + self.ogc_features = OGCFeaturesFactory( + collections_dependency=self.collections_dependency, + collection_dependency=self.collection_dependency, + router_prefix=self.router_prefix, + templates=self.templates, + # We do not want `/` and `/conformance` from the factory + with_common=False, + ) + self.router.include_router(self.ogc_features.router) + + self.ogc_tiles = OGCTilesFactory( + collection_dependency=self.collection_dependency, + router_prefix=self.router_prefix, + templates=self.templates, + supported_tms=self.supported_tms, + with_viewer=self.with_tiles_viewer, + # We do not want `/` and `/conformance` from the factory + with_common=False, + ) + self.router.include_router(self.ogc_tiles.router) diff --git a/tipg/factories/features.py b/tipg/factories/features.py new file mode 100644 index 00000000..e46bbd0f --- /dev/null +++ b/tipg/factories/features.py @@ -0,0 +1,874 @@ +"""tipg.factories.features: OGC API - Features endpoints factory.""" + +from dataclasses import dataclass +from typing import Annotated, Callable, Dict, List, Optional + +import orjson +from cql2 import Expr + +from tipg import model +from tipg.dbmodel import Collection, CollectionList +from tipg.dependencies import ( + CollectionsParams, + ItemsOutputType, + OutputType, + QueryablesOutputType, + bbox_query, + datetime_query, + filter_query, + function_parameters_query, + ids_query, + properties_filter_query, + properties_query, + sortby_query, +) +from tipg.errors import NoPrimaryKey, NotFound +from tipg.factories.base import EndpointsFactory +from tipg.factories.utils import create_csv_rows +from tipg.resources.enums import MediaType +from tipg.resources.response import ( + GeoJSONResponse, + ORJSONResponse, + SchemaJSONResponse, + orjsonDumps, +) +from tipg.settings import FeaturesSettings + +from fastapi import Depends, Path, Query + +from starlette.datastructures import QueryParams +from starlette.requests import Request +from starlette.responses import StreamingResponse +from starlette.routing import NoMatchFound + +features_settings = FeaturesSettings() + +FEATURES_CONFORMS = [ + "http://www.opengis.net/spec/ogcapi-features-1/1.0/conf/core", + "http://www.opengis.net/spec/ogcapi-features-1/1.0/conf/html", + "http://www.opengis.net/spec/ogcapi-features-1/1.0/conf/oas30", + "http://www.opengis.net/spec/ogcapi-features-1/1.0/conf/geojson", + "http://www.opengis.net/spec/ogcapi-features-3/1.0/conf/filter", + "http://www.opengis.net/spec/ogcapi-features-3/1.0/conf/features-filter", +] + + +@dataclass +class OGCFeaturesFactory(EndpointsFactory): + """OGC Features Endpoints Factory.""" + + # collections dependency + collections_dependency: Callable[..., CollectionList] = CollectionsParams + + @property + def conforms_to(self) -> List[str]: + """Factory conformances.""" + return FEATURES_CONFORMS + + def links(self, request: Request) -> List[model.Link]: + """OGC Features API links.""" + return [ + model.Link( + title="List of Collections", + href=self.url_for(request, "collections"), + type=MediaType.json, + rel="data", + ), + model.Link( + title="Collection metadata (Template URL)", + href=self.url_for( + request, + "collection", + collectionId="{collectionId}", + ), + type=MediaType.json, + rel="data", + templated=True, + ), + model.Link( + title="Collection queryables (Template URL)", + href=self.url_for( + request, + "queryables", + collectionId="{collectionId}", + ), + type=MediaType.schemajson, + rel="queryables", + templated=True, + ), + model.Link( + title="Collection Features (Template URL)", + href=self.url_for(request, "items", collectionId="{collectionId}"), + type=MediaType.geojson, + rel="data", + templated=True, + ), + model.Link( + title="Collection Feature (Template URL)", + href=self.url_for( + request, + "item", + collectionId="{collectionId}", + itemId="{itemId}", + ), + type=MediaType.geojson, + rel="data", + templated=True, + ), + ] + + def _additional_collection_tiles_links( + self, request: Request, collection: Collection + ) -> List[model.Link]: + links = [] + base_url = str(request.base_url) + try: + links.append( + model.Link( + rel="data", + title="Collection TileSets", + type=MediaType.json, + href=str( + request.app.url_path_for( + "collection_tileset_list", + collectionId=collection.id, + ).make_absolute_url(base_url=base_url) + ), + ), + ) + links.append( + model.Link( + rel="data", + title="Collection TileSet (Template URL)", + type=MediaType.json, + templated=True, + href=str( + request.app.url_path_for( + "collection_tileset", + collectionId=collection.id, + tileMatrixSetId="{tileMatrixSetId}", + ).make_absolute_url(base_url=base_url) + ), + ), + ) + except NoMatchFound: + pass + + try: + links.append( + model.Link( + title="Collection Map viewer (Template URL)", + href=str( + request.app.url_path_for( + "map_viewer", + collectionId=collection.id, + tileMatrixSetId="{tileMatrixSetId}", + ).make_absolute_url(base_url=base_url) + ), + type=MediaType.html, + rel="data", + templated=True, + ) + ) + + except NoMatchFound: + pass + + return links + + def register_routes(self): + """Register OGC Features endpoints.""" + self._collections_route() + self._collection_route() + self._queryables_route() + self._items_route() + self._item_route() + + def _collections_route(self): # noqa: C901 + @self.router.get( + "/collections", + response_model=model.Collections, + response_model_exclude_none=True, + response_class=ORJSONResponse, + responses={ + 200: { + "content": { + MediaType.json.value: {}, + MediaType.html.value: {}, + } + }, + }, + tags=["OGC Features API"], + ) + def collections( + request: Request, + collection_list: Annotated[ + CollectionList, + Depends(self.collections_dependency), + ], + output_type: Annotated[ + Optional[MediaType], + Depends(OutputType), + ] = None, + ): + """List of collections.""" + links: list = [ + model.Link( + href=self.url_for(request, "collections"), + rel="self", + type=MediaType.json, + ), + ] + + if next_token := collection_list["next"]: + query_params = QueryParams( + {**request.query_params, "offset": next_token} + ) + url = self.url_for(request, "collections") + f"?{query_params}" + links.append( + model.Link( + href=url, + rel="next", + type=MediaType.json, + title="Next page", + ), + ) + + if collection_list["prev"] is not None: + prev_token = collection_list["prev"] + qp = dict(request.query_params) + qp.pop("offset", None) + query_params = QueryParams({**qp, "offset": prev_token}) + url = self.url_for(request, "collections") + if query_params: + url += f"?{query_params}" + + links.append( + model.Link( + href=url, + rel="prev", + type=MediaType.json, + title="Previous page", + ), + ) + + data = model.Collections( + links=links, + numberMatched=collection_list["matched"], + numberReturned=len(collection_list["collections"]), + collections=[ + model.Collection( + id=collection.id, + title=collection.id, + description=collection.description, + extent=collection.extent, + links=[ + model.Link( + href=self.url_for( + request, + "collection", + collectionId=collection.id, + ), + rel="collection", + type=MediaType.json, + ), + model.Link( + href=self.url_for( + request, + "items", + collectionId=collection.id, + ), + rel="items", + type=MediaType.geojson, + ), + model.Link( + href=self.url_for( + request, + "queryables", + collectionId=collection.id, + ), + rel="queryables", + type=MediaType.schemajson, + ), + *self._additional_collection_tiles_links( + request, collection + ), + ], + ) + for collection in collection_list["collections"] + ], + ).model_dump(exclude_none=True, mode="json") + + if output_type == MediaType.html: + return self._create_html_response( + request, + data, + template_name="collections", + title="Collections list", + ) + + return ORJSONResponse(data) + + def _collection_route(self): + @self.router.get( + "/collections/{collectionId}", + response_model=model.Collection, + response_model_exclude_none=True, + response_class=ORJSONResponse, + responses={ + 200: { + "content": { + MediaType.json.value: {}, + MediaType.html.value: {}, + } + }, + }, + tags=["OGC Features API"], + ) + def collection( + request: Request, + collection: Annotated[Collection, Depends(self.collection_dependency)], + output_type: Annotated[Optional[MediaType], Depends(OutputType)] = None, + ): + """Metadata for a feature collection.""" + data = model.Collection( + id=collection.id, + title=collection.title, + description=collection.description, + extent=collection.extent, + links=[ + model.Link( + title="Collection", + href=self.url_for( + request, + "collection", + collectionId=collection.id, + ), + rel="self", + type=MediaType.json, + ), + model.Link( + title="Items", + href=self.url_for( + request, + "items", + collectionId=collection.id, + ), + rel="items", + type=MediaType.geojson, + ), + model.Link( + title="Items (CSV)", + href=self.url_for( + request, + "items", + collectionId=collection.id, + ) + + "?f=csv", + rel="alternate", + type=MediaType.csv, + ), + model.Link( + title="Items (GeoJSONSeq)", + href=self.url_for( + request, + "items", + collectionId=collection.id, + ) + + "?f=geojsonseq", + rel="alternate", + type=MediaType.geojsonseq, + ), + model.Link( + href=self.url_for( + request, + "queryables", + collectionId=collection.id, + ), + rel="queryables", + type=MediaType.schemajson, + ), + *self._additional_collection_tiles_links(request, collection), + ], + ).model_dump(exclude_none=True, mode="json") + + if output_type == MediaType.html: + return self._create_html_response( + request, + data, + template_name="collection", + title=f"{collection.id} collection", + ) + + return ORJSONResponse(data) + + def _queryables_route(self): + @self.router.get( + "/collections/{collectionId}/queryables", + response_model=model.Queryables, + response_model_exclude_none=True, + response_model_by_alias=True, + response_class=SchemaJSONResponse, + responses={ + 200: { + "content": { + MediaType.schemajson.value: {}, + MediaType.html.value: {}, + } + }, + }, + tags=["OGC Features API"], + ) + def queryables( + request: Request, + collection: Annotated[Collection, Depends(self.collection_dependency)], + output_type: Annotated[ + Optional[MediaType], Depends(QueryablesOutputType) + ] = None, + ): + """Queryables for a feature collection. + + ref: http://docs.ogc.org/DRAFTS/19-079r1.html#filter-queryables + """ + qs = "?" + str(request.query_params) if request.query_params else "" + self_url = self.url_for(request, "queryables", collectionId=collection.id) + data = model.Queryables( + title=collection.id, + link=self_url + qs, + properties=collection.queryables, + ).model_dump(exclude_none=True, mode="json", by_alias=True) + + if output_type == MediaType.html: + return self._create_html_response( + request, + data, + template_name="queryables", + title=f"{collection.id} queryables", + ) + + return SchemaJSONResponse(data) + + def _items_route(self): # noqa: C901 + @self.router.get( + "/collections/{collectionId}/items", + response_class=GeoJSONResponse, + responses={ + 200: { + "content": { + MediaType.geojson.value: {}, + MediaType.html.value: {}, + MediaType.csv.value: {}, + MediaType.json.value: {}, + MediaType.geojsonseq.value: {}, + MediaType.ndjson.value: {}, + }, + "model": model.Items, + }, + }, + tags=["OGC Features API"], + ) + async def items( # noqa: C901 + request: Request, + collection: Annotated[Collection, Depends(self.collection_dependency)], + ids_filter: Annotated[Optional[List[str]], Depends(ids_query)], + bbox_filter: Annotated[Optional[List[float]], Depends(bbox_query)], + datetime_filter: Annotated[Optional[List[str]], Depends(datetime_query)], + properties: Annotated[Optional[List[str]], Depends(properties_query)], + cql_filter: Annotated[Optional[Expr], Depends(filter_query)], + sortby: Annotated[Optional[str], Depends(sortby_query)], + geom_column: Annotated[ + Optional[str], + Query( + description="Select geometry column.", + alias="geom-column", + ), + ] = None, + datetime_column: Annotated[ + Optional[str], + Query( + description="Select datetime column.", + alias="datetime-column", + ), + ] = None, + limit: Annotated[ + int, + Query( + ge=0, + le=features_settings.max_features_per_query, + description="Limits the number of features in the response.", + ), + ] = features_settings.default_features_limit, + offset: Annotated[ + Optional[int], + Query( + ge=0, + description="Starts the response at an offset.", + ), + ] = None, + bbox_only: Annotated[ + Optional[bool], + Query( + description="Only return the bounding box of the feature.", + alias="bbox-only", + ), + ] = None, + simplify: Annotated[ + Optional[float], + Query( + description="Simplify the output geometry to given threshold in decimal degrees.", + ), + ] = None, + output_type: Annotated[ + Optional[MediaType], Depends(ItemsOutputType) + ] = None, + ): + output_type = output_type or MediaType.geojson + geom_as_wkt = output_type not in [ + MediaType.geojson, + MediaType.geojsonseq, + MediaType.html, + ] + + async with request.app.state.pool.acquire() as conn: + item_list = await collection.features( + conn, + ids_filter=ids_filter, + bbox_filter=bbox_filter, + datetime_filter=datetime_filter, + properties_filter=properties_filter_query(request, collection), + function_parameters=function_parameters_query(request, collection), + cql_filter=cql_filter, + sortby=sortby, + properties=properties, + limit=limit, + offset=offset, + geom=geom_column, + dt=datetime_column, + bbox_only=bbox_only, + simplify=simplify, + geom_as_wkt=geom_as_wkt, + ) + + if output_type in ( + MediaType.csv, + MediaType.json, + MediaType.ndjson, + ): + if any(f.get("geometry", None) is not None for f in item_list["items"]): + rows = ( + { + "collectionId": collection.id, + "itemId": f.get("id"), + **f.get("properties", {}), + "geometry": f.get("geometry", None), + } + for f in item_list["items"] + ) + else: + rows = ( + { + "collectionId": collection.id, + "itemId": f.get("id"), + **f.get("properties", {}), + } + for f in item_list["items"] + ) + + # CSV Response + if output_type == MediaType.csv: + return StreamingResponse( + create_csv_rows(rows), + media_type=MediaType.csv, + headers={ + "Content-Disposition": "attachment;filename=items.csv" + }, + ) + + # JSON Response + if output_type == MediaType.json: + return ORJSONResponse(list(rows)) + + # NDJSON Response + if output_type == MediaType.ndjson: + return StreamingResponse( + (orjsonDumps(row) + b"\n" for row in rows), + media_type=MediaType.ndjson, + headers={ + "Content-Disposition": "attachment;filename=items.ndjson" + }, + ) + + qs = "?" + str(request.query_params) if request.query_params else "" + links: List[Dict] = [ + { + "title": "Collection", + "href": self.url_for( + request, "collection", collectionId=collection.id + ), + "rel": "collection", + "type": "application/json", + }, + { + "title": "Items", + "href": self.url_for(request, "items", collectionId=collection.id) + + qs, + "rel": "self", + "type": "application/geo+json", + }, + ] + + if next_token := item_list["next"]: + query_params = QueryParams( + {**request.query_params, "offset": next_token} + ) + url = ( + self.url_for(request, "items", collectionId=collection.id) + + f"?{query_params}" + ) + links.append( + { + "href": url, + "rel": "next", + "type": "application/geo+json", + "title": "Next page", + }, + ) + + if item_list["prev"] is not None: + prev_token = item_list["prev"] + qp = dict(request.query_params) + qp.pop("offset") + query_params = QueryParams({**qp, "offset": prev_token}) + url = self.url_for(request, "items", collectionId=collection.id) + if query_params: + url += f"?{query_params}" + + links.append( + { + "href": url, + "rel": "prev", + "type": "application/geo+json", + "title": "Previous page", + }, + ) + + data = { + "type": "FeatureCollection", + "id": collection.id, + "title": collection.title or collection.id, + "description": collection.description + or collection.title + or collection.id, + "numberMatched": item_list["matched"], + "numberReturned": len(item_list["items"]), + "links": links, + "features": [ + { + **feature, # type: ignore + "links": [ + { + "title": "Collection", + "href": self.url_for( + request, + "collection", + collectionId=collection.id, + ), + "rel": "collection", + "type": "application/json", + }, + { + "title": "Item", + "href": self.url_for( + request, + "item", + collectionId=collection.id, + itemId=feature.get("id"), + ), + "rel": "item", + "type": "application/geo+json", + }, + ], + } + for feature in item_list["items"] + ], + } + + # HTML Response + if output_type == MediaType.html: + return self._create_html_response( + request, + orjson.loads(orjsonDumps(data).decode()), + template_name="items", + title=f"{collection.id} items", + ) + + # GeoJSONSeq Response + elif output_type == MediaType.geojsonseq: + return StreamingResponse( + (orjsonDumps(f) + b"\n" for f in data["features"]), # type: ignore + media_type=MediaType.geojsonseq, + headers={ + "Content-Disposition": "attachment;filename=items.geojson" + }, + ) + + # Default to GeoJSON Response + return GeoJSONResponse(data) + + def _item_route(self): + @self.router.get( + "/collections/{collectionId}/items/{itemId}", + response_class=GeoJSONResponse, + responses={ + 200: { + "content": { + MediaType.geojson.value: {}, + MediaType.html.value: {}, + MediaType.csv.value: {}, + MediaType.json.value: {}, + MediaType.geojsonseq.value: {}, + MediaType.ndjson.value: {}, + }, + "model": model.Item, + }, + }, + tags=["OGC Features API"], + ) + async def item( + request: Request, + collection: Annotated[Collection, Depends(self.collection_dependency)], + itemId: Annotated[str, Path(description="Item identifier")], + bbox_only: Annotated[ + Optional[bool], + Query( + description="Only return the bounding box of the feature.", + alias="bbox-only", + ), + ] = None, + simplify: Annotated[ + Optional[float], + Query( + description="Simplify the output geometry to given threshold in decimal degrees.", + ), + ] = None, + geom_column: Annotated[ + Optional[str], + Query( + description="Select geometry column.", + alias="geom-column", + ), + ] = None, + datetime_column: Annotated[ + Optional[str], + Query( + description="Select datetime column.", + alias="datetime-column", + ), + ] = None, + properties: Optional[List[str]] = Depends(properties_query), + output_type: Annotated[ + Optional[MediaType], Depends(ItemsOutputType) + ] = None, + ): + if collection.id_column is None: + raise NoPrimaryKey("No primary key is set on this table") + + output_type = output_type or MediaType.geojson + geom_as_wkt = output_type not in [ + MediaType.geojson, + MediaType.geojsonseq, + MediaType.html, + ] + + async with request.app.state.pool.acquire() as conn: + item_list = await collection.features( + conn, + ids_filter=[itemId], + bbox_only=bbox_only, + simplify=simplify, + properties=properties, + function_parameters=function_parameters_query(request, collection), + geom=geom_column, + dt=datetime_column, + geom_as_wkt=geom_as_wkt, + ) + + if not item_list["items"]: + raise NotFound( + f"Item {itemId} in Collection {collection.id} does not exist." + ) + + feature = item_list["items"][0] + + if output_type in ( + MediaType.csv, + MediaType.json, + MediaType.ndjson, + ): + row = { + "collectionId": collection.id, + "itemId": feature.get("id"), + **feature.get("properties", {}), + } + if feature.get("geometry") is not None: + row["geometry"] = (feature["geometry"],) + rows = iter([row]) + + # CSV Response + if output_type == MediaType.csv: + return StreamingResponse( + create_csv_rows(rows), + media_type=MediaType.csv, + headers={ + "Content-Disposition": "attachment;filename=items.csv" + }, + ) + + # JSON Response + if output_type == MediaType.json: + return ORJSONResponse(rows.__next__()) + + # NDJSON Response + if output_type == MediaType.ndjson: + return StreamingResponse( + (orjsonDumps(row) + b"\n" for row in rows), + media_type=MediaType.ndjson, + headers={ + "Content-Disposition": "attachment;filename=items.ndjson" + }, + ) + + data = { + **feature, # type: ignore + "links": [ + { + "href": self.url_for( + request, "collection", collectionId=collection.id + ), + "rel": "collection", + "type": "application/json", + }, + { + "href": self.url_for( + request, + "item", + collectionId=collection.id, + itemId=itemId, + ), + "rel": "self", + "type": "application/geo+json", + }, + ], + } + + # HTML Response + if output_type == MediaType.html: + return self._create_html_response( + request, + orjson.loads(orjsonDumps(data).decode()), + template_name="item", + title=f"{collection.id}/{itemId} item", + ) + + # Default to GeoJSON Response + return GeoJSONResponse(data) diff --git a/tipg/factories/tiles.py b/tipg/factories/tiles.py new file mode 100644 index 00000000..5eafafb4 --- /dev/null +++ b/tipg/factories/tiles.py @@ -0,0 +1,830 @@ +"""tipg.factories.tiles: OGC API - Tiles endpoints factory.""" + +from dataclasses import dataclass +from typing import Annotated, Any, Dict, List, Literal, Optional +from urllib.parse import urlencode + +from cql2 import Expr +from morecantile import Tile, TileMatrixSet +from morecantile import tms as default_tms +from morecantile.defaults import TileMatrixSets + +from tipg import model +from tipg.dbmodel import Collection +from tipg.dependencies import ( + OutputType, + TileParams, + bbox_query, + datetime_query, + filter_query, + function_parameters_query, + ids_query, + properties_filter_query, + properties_query, + sortby_query, +) +from tipg.errors import MissingGeometryColumn +from tipg.factories.base import EndpointsFactory +from tipg.resources.enums import MediaType +from tipg.resources.response import ORJSONResponse +from tipg.settings import MVTSettings, TMSSettings + +from fastapi import Depends, Path, Query + +from starlette.requests import Request +from starlette.responses import HTMLResponse, Response + +tms_settings = TMSSettings() +mvt_settings = MVTSettings() + +TILES_CONFORMS = [ + "http://www.opengis.net/spec/ogcapi-tiles-1/1.0/conf/core", + "http://www.opengis.net/spec/ogcapi-tiles-1/1.0/conf/oas30", + "http://www.opengis.net/spec/ogcapi-tiles-1/1.0/conf/mvt", + "http://www.opengis.net/spec/ogcapi-tiles-1/1.0/conf/tileset", + "http://www.opengis.net/spec/ogcapi-tiles-1/1.0/conf/tilesets-list", +] + + +@dataclass +class OGCTilesFactory(EndpointsFactory): + """OGC Tiles Endpoints Factory.""" + + supported_tms: TileMatrixSets = default_tms + with_viewer: bool = True + + @property + def conforms_to(self) -> List[str]: + """Factory conformances.""" + return TILES_CONFORMS + + def links(self, request: Request) -> List[model.Link]: + """OGC Tiles API links.""" + links = [ + model.Link( + title="Collection Vector Tiles (Template URL)", + href=self.url_for( + request, + "collection_get_tile", + collectionId="{collectionId}", + tileMatrixSetId="{tileMatrixSetId}", + z="{z}", + x="{x}", + y="{y}", + ), + type=MediaType.mvt, + rel="data", + templated=True, + ), + model.Link( + title="Collection TileSets (Template URL)", + href=self.url_for( + request, + "collection_tileset_list", + collectionId="{collectionId}", + ), + type=MediaType.json, + rel="data", + templated=True, + ), + model.Link( + title="Collection TileSet (Template URL)", + href=self.url_for( + request, + "collection_tileset", + collectionId="{collectionId}", + tileMatrixSetId="{tileMatrixSetId}", + ), + type=MediaType.json, + rel="data", + templated=True, + ), + ] + + if self.with_viewer: + links.append( + model.Link( + title="Collection Map viewer (Template URL)", + href=self.url_for( + request, + "map_viewer", + collectionId="{collectionId}", + tileMatrixSetId="{tileMatrixSetId}", + ), + type=MediaType.html, + rel="data", + templated=True, + ) + ) + + links += [ + model.Link( + title="TileMatrixSets", + href=self.url_for( + request, + "tilematrixsets", + ), + type=MediaType.json, + rel="http://www.opengis.net/def/rel/ogc/1.0/tiling-schemes", + ), + model.Link( + title="TileMatrixSet (Template URL)", + href=self.url_for( + request, + "tilematrixset", + tileMatrixSetId="{tileMatrixSetId}", + ), + type=MediaType.json, + rel="data", + templated=True, + ), + ] + + return links + + def register_routes(self): # noqa: C901 + """Register OGC Tiles endpoints.""" + self._tilematrixsets_routes() + self._tilesets_routes() + self._tile_routes() + self._tilejson_routes() + self._stylejson_routes() + + def _tilematrixsets_routes(self): + @self.router.get( + r"/tileMatrixSets", + response_model=model.TileMatrixSetList, + response_model_exclude_none=True, + summary="Retrieve the list of available tiling schemes (tile matrix sets).", + operation_id="getTileMatrixSetsList", + responses={ + 200: { + "content": { + MediaType.html.value: {}, + MediaType.json.value: {}, + }, + }, + }, + tags=["OGC Tiles API"], + ) + async def tilematrixsets( + request: Request, + output_type: Annotated[Optional[MediaType], Depends(OutputType)] = None, + ): + """ + OGC Specification: http://docs.opengeospatial.org/per/19-069.html#_tilematrixsets + """ + data = model.TileMatrixSetList( + tileMatrixSets=[ + model.TileMatrixSetRef( + id=tms_id, + title=f"Definition of {tms_id} tileMatrixSets", + links=[ + model.TileMatrixSetLink( + href=self.url_for( + request, + "tilematrixset", + tileMatrixSetId=tms_id, + ), + ) + ], + ) + for tms_id in self.supported_tms.list() + ] + ) + + if output_type == MediaType.html: + return self._create_html_response( + request, + data.model_dump(exclude_none=True, mode="json"), + template_name="tilematrixsets", + title="TileMatrixSets list", + ) + + return data + + @self.router.get( + "/tileMatrixSets/{tileMatrixSetId}", + response_model=TileMatrixSet, + response_model_exclude_none=True, + summary="Retrieve the definition of the specified tiling scheme (tile matrix set).", + operation_id="getTileMatrixSet", + responses={ + 200: { + "content": { + MediaType.html.value: {}, + MediaType.json.value: {}, + }, + }, + }, + tags=["OGC Tiles API"], + ) + async def tilematrixset( + request: Request, + tileMatrixSetId: Annotated[ + Literal[tuple(self.supported_tms.list())], + Path(description="Identifier for a supported TileMatrixSet."), + ], + output_type: Annotated[Optional[MediaType], Depends(OutputType)] = None, + ): + """ + OGC Specification: http://docs.opengeospatial.org/per/19-069.html#_tilematrixset + """ + tms = self.supported_tms.get(tileMatrixSetId) + + if output_type == MediaType.html: + return self._create_html_response( + request, + { + **tms.model_dump(exclude_none=True, mode="json"), + # For visualization purpose we add the tms bbox + "bbox": tms.bbox, + }, + template_name="tilematrixset", + title=f"{tileMatrixSetId} TileMatrixSet", + ) + + return tms + + def _tilesets_routes(self): + @self.router.get( + "/collections/{collectionId}/tiles", + response_model=model.TileSetList, + response_class=ORJSONResponse, + response_model_exclude_none=True, + responses={ + 200: { + "content": { + MediaType.json.value: {}, + MediaType.html.value: {}, + } + } + }, + summary="Retrieve a list of available vector tilesets for the specified collection.", + operation_id=".collection.vector.getTileSetsList", + tags=["OGC Tiles API"], + ) + async def collection_tileset_list( + request: Request, + collection: Annotated[Collection, Depends(self.collection_dependency)], + output_type: Annotated[Optional[MediaType], Depends(OutputType)] = None, + ): + """Retrieve a list of available vector tilesets for the specified collection.""" + collection_bbox = None + if bounds := collection.bounds: + collection_bbox = { + "lowerLeft": [bounds[0], bounds[1]], + "upperRight": [bounds[2], bounds[3]], + "crs": "http://www.opengis.net/def/crs/OGC/1.3/CRS84", + } + + data = model.TileSetList.model_validate( + { + "tilesets": [ + { + "title": f"'{collection.id}' tileset tiled using {tms} TileMatrixSet", + "dataType": "vector", + "crs": self.supported_tms.get(tms).crs, + "boundingBox": collection_bbox, + "links": [ + { + "href": self.url_for( + request, + "collection_tileset", + collectionId=collection.id, + tileMatrixSetId=tms, + ), + "rel": "self", + "type": "application/json", + "title": f"'{collection.id}' tileset tiled using {tms} TileMatrixSet", + }, + { + "href": self.url_for( + request, + "tilematrixset", + tileMatrixSetId=tms, + ), + "rel": "http://www.opengis.net/def/rel/ogc/1.0/tiling-scheme", + "type": "application/json", + "title": f"Definition of '{tms}' tileMatrixSet", + }, + { + "href": self.url_for( + request, + "collection_get_tile", + tileMatrixSetId=tms, + collectionId=collection.id, + z="{z}", + x="{x}", + y="{y}", + ), + "rel": "tile", + "type": "application/vnd.mapbox-vector-tile", + "title": "Templated link for retrieving Vector tiles", + }, + ], + } + for tms in self.supported_tms.list() + ] + } + ) + + if output_type == MediaType.html: + return self._create_html_response( + request, + data.model_dump(exclude_none=True, mode="json"), + template_name="tilesets", + title=f"{collection.id} tilesets", + ) + + return data + + @self.router.get( + "/collections/{collectionId}/tiles/{tileMatrixSetId}", + response_model=model.TileSet, + response_class=ORJSONResponse, + response_model_exclude_none=True, + responses={200: {"content": {MediaType.json.value: {}}}}, + summary="Retrieve the vector tileset metadata for the specified collection and tiling scheme (tile matrix set).", + operation_id=".collection.vector.getTileSet", + tags=["OGC Tiles API"], + ) + async def collection_tileset( + request: Request, + collection: Annotated[Collection, Depends(self.collection_dependency)], + tileMatrixSetId: Annotated[ + Literal[tuple(self.supported_tms.list())], + Path(description="Identifier for a supported TileMatrixSet."), + ], + output_type: Annotated[Optional[MediaType], Depends(OutputType)] = None, + ): + """Retrieve the vector tileset metadata for the specified collection and tiling scheme (tile matrix set).""" + tms = self.supported_tms.get(tileMatrixSetId) + + if bounds := collection.bounds: + collection_bbox = { + "lowerLeft": [bounds[0], bounds[1]], + "upperRight": [bounds[2], bounds[3]], + "crs": "http://www.opengis.net/def/crs/OGC/1.3/CRS84", + } + + tilematrix_limit = [] + for matrix in tms: + ulTile = tms.tile(bounds[0], bounds[3], int(matrix.id)) + lrTile = tms.tile(bounds[2], bounds[1], int(matrix.id)) + minx, maxx = (min(ulTile.x, lrTile.x), max(ulTile.x, lrTile.x)) + miny, maxy = (min(ulTile.y, lrTile.y), max(ulTile.y, lrTile.y)) + tilematrix_limit.append( + { + "tileMatrix": matrix.id, + "minTileRow": max(miny, 0), + "maxTileRow": min(maxy, matrix.matrixHeight), + "minTileCol": max(minx, 0), + "maxTileCol": min(maxx, matrix.matrixWidth), + } + ) + + else: + collection_bbox = None + tilematrix_limit = [ + { + "tileMatrix": matrix.id, + "minTileRow": 0, + "maxTileRow": matrix.matrixHeight, + "minTileCol": 0, + "maxTileCol": matrix.matrixWidth, + } + for matrix in tms + ] + + links = [ + { + "href": self.url_for( + request, + "collection_tileset", + collectionId=collection.id, + tileMatrixSetId=tileMatrixSetId, + ), + "rel": "self", + "type": "application/json", + "title": f"'{collection.id}' tileset tiled using {tileMatrixSetId} TileMatrixSet", + }, + { + "href": self.url_for( + request, + "tilematrixset", + tileMatrixSetId=tileMatrixSetId, + ), + "rel": "http://www.opengis.net/def/rel/ogc/1.0/tiling-scheme", + "type": "application/json", + "title": f"Definition of '{tileMatrixSetId}' tileMatrixSet", + }, + { + "href": self.url_for( + request, + "collection_get_tile", + tileMatrixSetId=tileMatrixSetId, + collectionId=collection.id, + z="{z}", + x="{x}", + y="{y}", + ), + "rel": "tile", + "type": "application/vnd.mapbox-vector-tile", + "title": "Templated link for retrieving Vector tiles", + "templated": True, + }, + ] + + if self.with_viewer: + links.append( + { + "href": self.url_for( + request, + "map_viewer", + tileMatrixSetId=tileMatrixSetId, + collectionId=collection.id, + ), + "type": "text/html", + "rel": "data", + "title": f"Map viewer for '{tileMatrixSetId}' tileMatrixSet", + } + ) + + data = model.TileSet.model_validate( + { + "title": f"'{collection.id}' tileset tiled using {tileMatrixSetId} TileMatrixSet", + "dataType": "vector", + "crs": tms.crs, + "boundingBox": collection_bbox, + "links": links, + "tileMatrixSetLimits": tilematrix_limit, + } + ) + + if output_type == MediaType.html: + return self._create_html_response( + request, + data.model_dump(exclude_none=True, mode="json"), + template_name="tileset", + title=f"{collection.id} {tileMatrixSetId} tileset", + ) + + return data + + def _tile_routes(self): + @self.router.get( + "/collections/{collectionId}/tiles/{tileMatrixSetId}/{z}/{x}/{y}", + response_class=Response, + responses={200: {"content": {MediaType.mvt.value: {}}}}, + operation_id=".collection.vector.getTileTms", + tags=["OGC Tiles API"], + ) + async def collection_get_tile( + request: Request, + collection: Annotated[Collection, Depends(self.collection_dependency)], + tileMatrixSetId: Annotated[ + Literal[tuple(self.supported_tms.list())], + Path( + description="Identifier selecting one of the TileMatrixSetId supported." + ), + ], + tile: Annotated[Tile, Depends(TileParams)], + ids_filter: Annotated[Optional[List[str]], Depends(ids_query)] = None, + bbox_filter: Annotated[Optional[List[float]], Depends(bbox_query)] = None, + datetime_filter: Annotated[ + Optional[List[str]], Depends(datetime_query) + ] = None, + properties: Annotated[ + Optional[List[str]], Depends(properties_query) + ] = None, + cql_filter: Annotated[Optional[Expr], Depends(filter_query)] = None, + sortby: Annotated[Optional[str], Depends(sortby_query)] = None, + geom_column: Annotated[ + Optional[str], + Query( + description="Select geometry column.", + alias="geom-column", + ), + ] = None, + datetime_column: Annotated[ + Optional[str], + Query( + description="Select datetime column.", + alias="datetime-column", + ), + ] = None, + limit: Annotated[ + Optional[int], + Query( + description="Limits the number of features in the response. Defaults to 10000 or TIPG_MAX_FEATURES_PER_TILE environment variable." + ), + ] = None, + ): + """Return Vector Tile.""" + tms = self.supported_tms.get(tileMatrixSetId) + + async with request.app.state.pool.acquire() as conn: + tile = await collection.get_tile( + conn, + tms=tms, + tile=tile, + ids_filter=ids_filter, + bbox_filter=bbox_filter, + datetime_filter=datetime_filter, + properties_filter=properties_filter_query(request, collection), + function_parameters=function_parameters_query(request, collection), + cql_filter=cql_filter, + sortby=sortby, + properties=properties, + limit=limit, + geom=geom_column, + dt=datetime_column, + ) + + return Response(tile, media_type=MediaType.mvt.value) + + def _tilejson_routes(self): + ############################################################################ + # ADDITIONAL ENDPOINTS NOT IN OGC Tiles API (tilejson, style.json, viewer) # + ############################################################################ + @self.router.get( + "/collections/{collectionId}/tiles/{tileMatrixSetId}/tilejson.json", + response_model=model.TileJSON, + responses={200: {"description": "Return a tilejson"}}, + response_model_exclude_none=True, + response_class=ORJSONResponse, + operation_id=".collection.vector.getTileJSONTms", + tags=["OGC Tiles API"], + ) + async def collection_tilejson( + request: Request, + collection: Annotated[Collection, Depends(self.collection_dependency)], + tileMatrixSetId: Annotated[ + Literal[tuple(self.supported_tms.list())], + Path( + description="Identifier selecting one of the TileMatrixSetId supported." + ), + ], + minzoom: Annotated[ + Optional[int], + Query(description="Overwrite default minzoom."), + ] = None, + maxzoom: Annotated[ + Optional[int], + Query(description="Overwrite default maxzoom."), + ] = None, + geom_column: Annotated[ + Optional[str], + Query( + description="Select geometry column.", + alias="geom-column", + ), + ] = None, + ): + """Return TileJSON document.""" + tms = self.supported_tms.get(tileMatrixSetId) + + geom = collection.get_geometry_column(geom_column) + if not geom: + raise MissingGeometryColumn + + path_params: Dict[str, Any] = { + "tileMatrixSetId": tileMatrixSetId, + "collectionId": collection.id, + "z": "{z}", + "x": "{x}", + "y": "{y}", + } + tile_endpoint = self.url_for(request, "collection_get_tile", **path_params) + + qs_key_to_remove = ["tilematrixsetid", "minzoom", "maxzoom"] + query_params = [ + (key, value) + for (key, value) in request.query_params._list + if key.lower() not in qs_key_to_remove + ] + + if query_params: + tile_endpoint += f"?{urlencode(query_params)}" + + # Get Min/Max zoom from layer settings if tms is the default tms + if tileMatrixSetId == tms_settings.default_tms: + minzoom = minzoom or tms_settings.default_minzoom + maxzoom = maxzoom or tms_settings.default_maxzoom + + minzoom = minzoom if minzoom is not None else tms.minzoom + maxzoom = maxzoom if maxzoom is not None else tms.maxzoom + + tile_json = { + "minzoom": minzoom, + "maxzoom": maxzoom, + "name": collection.id, + "tiles": [tile_endpoint], + } + if collection.bounds: + tile_json["bounds"] = collection.bounds + + layername = collection.id if mvt_settings.set_mvt_layername else "default" + tile_json["vector_layers"] = [ + { + "id": layername, + "fields": { + col.name: col.json_type + for col in collection.properties + if col.type not in ["geometry", "geography"] + }, + "minzoom": minzoom, + "maxzoom": maxzoom, + } + ] + + return tile_json + + def _stylejson_routes(self): + @self.router.get( + "/collections/{collectionId}/tiles/{tileMatrixSetId}/style.json", + response_model=model.StyleJSON, + responses={200: {"description": "Return a tilejson"}}, + response_model_exclude_none=True, + response_class=ORJSONResponse, + operation_id=".collection.vector.getStyleJSONTms", + tags=["OGC Tiles API"], + ) + async def collection_stylejson( + request: Request, + collection: Annotated[Collection, Depends(self.collection_dependency)], + tileMatrixSetId: Annotated[ + Literal[tuple(self.supported_tms.list())], + Path( + description="Identifier selecting one of the TileMatrixSetId supported." + ), + ], + geom_column: Annotated[ + Optional[str], + Query( + description="Select geometry column.", + alias="geom-column", + ), + ] = None, + minzoom: Annotated[ + Optional[int], + Query(description="Overwrite default minzoom."), + ] = None, + maxzoom: Annotated[ + Optional[int], + Query(description="Overwrite default maxzoom."), + ] = None, + ): + """Return Mapbox/Maplibre StyleJSON document.""" + tms = self.supported_tms.get(tileMatrixSetId) + + geom = collection.get_geometry_column(geom_column) + if not geom: + raise MissingGeometryColumn + + path_params: Dict[str, Any] = { + "collectionId": collection.id, + "tileMatrixSetId": tileMatrixSetId, + "z": "{z}", + "x": "{x}", + "y": "{y}", + } + tiles_endpoint = self.url_for(request, "collection_get_tile", **path_params) + + qs_key_to_remove = ["tilematrixsetid", "minzoom", "maxzoom"] + query_params = [ + (key, value) + for (key, value) in request.query_params._list + if key.lower() not in qs_key_to_remove + ] + if query_params: + tiles_endpoint += f"?{urlencode(query_params)}" + + # Get Min/Max zoom from layer settings if tms is the default tms + if tileMatrixSetId == tms_settings.default_tms: + minzoom = minzoom or tms_settings.default_minzoom + maxzoom = maxzoom or tms_settings.default_maxzoom + + minzoom = minzoom if minzoom is not None else tms.minzoom + maxzoom = maxzoom if maxzoom is not None else tms.maxzoom + + bounds = list(collection.bounds) or list(tms.bbox) + + style_json = { + "name": "TiPg", + "sources": { + collection.id: { + "type": "vector", + "scheme": "xyz", + "tiles": [tiles_endpoint], + "bounds": bounds, + "minzoom": minzoom, + "maxzoom": maxzoom, + } + }, + "layers": [], + "center": [ + (bounds[0] + bounds[2]) / 2, + (bounds[1] + bounds[3]) / 2, + ], + "zoom": minzoom, + } + + layername = collection.id if mvt_settings.set_mvt_layername else "default" + style_json["layers"] = [ + { + "id": f"{collection.id}_fill", + "source": collection.id, + "source-layer": layername, + "type": "fill", + "filter": ["==", ["geometry-type"], "Polygon"], + "paint": { + "fill-color": "rgba(200, 100, 240, 0.4)", + "fill-outline-color": "#000", + }, + }, + { + "id": f"{collection.id}_stroke", + "source": collection.id, + "source-layer": layername, + "type": "line", + "filter": ["==", ["geometry-type"], "LineString"], + "paint": { + "line-color": "#000", + "line-width": 1, + "line-opacity": 0.75, + }, + }, + { + "id": f"{collection.id}_point", + "source": collection.id, + "source-layer": layername, + "type": "circle", + "filter": ["==", ["geometry-type"], "Point"], + "paint": { + "circle-color": "#000", + "circle-radius": 2.5, + "circle-opacity": 0.75, + }, + }, + ] + + return style_json + + if self.with_viewer: + + @self.router.get( + "/collections/{collectionId}/tiles/{tileMatrixSetId}/map.html", + response_class=HTMLResponse, + operation_id=".collection.vector.map", + tags=["Map Viewer"], + ) + def map_viewer( + request: Request, + collection: Annotated[Collection, Depends(self.collection_dependency)], + tileMatrixSetId: Annotated[ + Literal[tuple(self.supported_tms.list())], + Path( + description="Identifier selecting one of the TileMatrixSetId supported." + ), + ], + minzoom: Annotated[ + Optional[int], + Query(description="Overwrite default minzoom."), + ] = None, + maxzoom: Annotated[ + Optional[int], + Query(description="Overwrite default maxzoom."), + ] = None, + geom_column: Annotated[ + Optional[str], + Query( + description="Select geometry column.", + alias="geom-column", + ), + ] = None, + ): + """Return Simple HTML Viewer for a collection.""" + tms = self.supported_tms.get(tileMatrixSetId) + + tilejson_url = self.url_for( + request, + "collection_tilejson", + collectionId=collection.id, + tileMatrixSetId=tileMatrixSetId, + ) + if request.query_params._list: + tilejson_url += f"?{urlencode(request.query_params._list)}" + + return self._create_html_response( + request, + { + "title": collection.id, + "tilejson_endpoint": tilejson_url, + "tms": tms, + "resolutions": [matrix.cellSize for matrix in tms], + }, + template_name="map", + title=f"{collection.id} viewer", + ) diff --git a/tipg/factories/utils.py b/tipg/factories/utils.py new file mode 100644 index 00000000..a74cd0f5 --- /dev/null +++ b/tipg/factories/utils.py @@ -0,0 +1,97 @@ +"""tipg.factories.utils: shared rendering helpers for the factories.""" + +import csv +import re +from typing import Any, Dict, Generator, Iterable, Optional + +import jinja2 + +from starlette.requests import Request +from starlette.templating import Jinja2Templates, _TemplateResponse + +# NOTE: templates live in ``tipg/templates`` (the top-level package), so the +# loader is anchored to ``tipg`` explicitly rather than to this module's +# package (``tipg.factories``). +jinja2_env = jinja2.Environment( + loader=jinja2.ChoiceLoader([jinja2.PackageLoader("tipg", "templates")]) +) +DEFAULT_TEMPLATES = Jinja2Templates(env=jinja2_env) + + +def create_csv_rows(data: Iterable[Dict]) -> Generator[str, None, None]: + """Creates an iterator that returns lines of csv from an iterable of dicts.""" + + class DummyWriter: + """Dummy writer that implements write for use with csv.writer.""" + + def write(self, line: str): + """Return line.""" + return line + + # Get the first row and construct the column names + row = next(data) # type: ignore + fieldnames = row.keys() + writer = csv.DictWriter(DummyWriter(), fieldnames=fieldnames) + + # Write header + yield writer.writerow(dict(zip(fieldnames, fieldnames))) + + # Write first row + yield writer.writerow(row) + + # Write all remaining rows + for row in data: + yield writer.writerow(row) + + +def create_html_response( + request: Request, + data: Any, + templates: Jinja2Templates, + template_name: str, + title: Optional[str] = None, + router_prefix: Optional[str] = None, + **kwargs: Any, +) -> _TemplateResponse: + """Create Template response.""" + urlpath = request.url.path + if root_path := request.scope.get("root_path"): + urlpath = re.sub(r"^" + root_path, "", urlpath) + + if router_prefix: + urlpath = re.sub(r"^" + router_prefix, "", urlpath) + + crumbs = [] + baseurl = str(request.base_url).rstrip("/") + + if router_prefix: + baseurl += router_prefix + + crumbpath = str(baseurl) + if urlpath == "/": + urlpath = "" + + for crumb in urlpath.split("/"): + crumbpath = crumbpath.rstrip("/") + part = crumb + if part is None or part == "": + part = "Home" + crumbpath += f"/{crumb}" + crumbs.append({"url": crumbpath.rstrip("/"), "part": part.capitalize()}) + + return templates.TemplateResponse( + request, + name=f"{template_name}.html", + context={ + "response": data, + "template": { + "api_root": baseurl, + "params": request.query_params, + "title": title or template_name, + }, + "crumbs": crumbs, + "url": baseurl + urlpath, + "params": str(request.url.query), + **kwargs, + }, + ) diff --git a/tipg/factory.py b/tipg/factory.py index bb7879e2..18c5c0d4 100644 --- a/tipg/factory.py +++ b/tipg/factory.py @@ -1,2031 +1,11 @@ -"""tipg.factory: router factories.""" +"""Backwards-compat shim for the factory package. -import abc -import csv -import re -from dataclasses import dataclass, field -from typing import ( - Annotated, - Any, - Callable, - Dict, - Generator, - Iterable, - List, - Literal, - Optional, -) -from urllib.parse import urlencode +The router factories were split into the ``tipg.factories`` package +(``tipg.factories.base`` / ``features`` / ``tiles`` / ``endpoints`` / ``utils``). +This module re-exports them so existing ``from tipg.factory import ...`` and +``from tipg import factory`` imports keep working. Prefer importing from +``tipg.factories`` directly. +""" -import jinja2 -import orjson -from cql2 import Expr -from morecantile import Tile, TileMatrixSet -from morecantile import tms as default_tms -from morecantile.defaults import TileMatrixSets - -from tipg import model -from tipg.collections import Collection, CollectionList -from tipg.dependencies import ( - CollectionParams, - CollectionsParams, - ItemsOutputType, - OutputType, - QueryablesOutputType, - TileParams, - bbox_query, - datetime_query, - filter_query, - function_parameters_query, - ids_query, - properties_filter_query, - properties_query, - sortby_query, -) -from tipg.errors import MissingGeometryColumn, NoPrimaryKey, NotFound -from tipg.resources.enums import MediaType -from tipg.resources.response import ( - GeoJSONResponse, - ORJSONResponse, - SchemaJSONResponse, - orjsonDumps, -) -from tipg.settings import FeaturesSettings, MVTSettings, TMSSettings - -from fastapi import APIRouter, Depends, Path, Query - -from starlette.datastructures import QueryParams -from starlette.requests import Request -from starlette.responses import HTMLResponse, Response, StreamingResponse -from starlette.routing import NoMatchFound, compile_path, replace_params -from starlette.templating import Jinja2Templates, _TemplateResponse - -tms_settings = TMSSettings() -mvt_settings = MVTSettings() -features_settings = FeaturesSettings() - - -jinja2_env = jinja2.Environment( - loader=jinja2.ChoiceLoader([jinja2.PackageLoader(__package__, "templates")]) -) -DEFAULT_TEMPLATES = Jinja2Templates(env=jinja2_env) - - -COMMON_CONFORMS = [ - "http://www.opengis.net/spec/ogcapi-common-1/1.0/conf/core", - "http://www.opengis.net/spec/ogcapi-common-1/1.0/conf/landingPage", - "http://www.opengis.net/spec/ogcapi-common-2/1.0/conf/collections", - "http://www.opengis.net/spec/ogcapi-common-2/1.0/conf/simple-query", - "http://www.opengis.net/spec/ogcapi-common-1/1.0/conf/json", - "http://www.opengis.net/spec/ogcapi-common-1/1.0/conf/html", - "http://www.opengis.net/spec/ogcapi-common-1/1.0/conf/oas30", -] -FEATURES_CONFORMS = [ - "http://www.opengis.net/spec/ogcapi-features-1/1.0/conf/core", - "http://www.opengis.net/spec/ogcapi-features-1/1.0/conf/html", - "http://www.opengis.net/spec/ogcapi-features-1/1.0/conf/oas30", - "http://www.opengis.net/spec/ogcapi-features-1/1.0/conf/geojson", - "http://www.opengis.net/spec/ogcapi-features-3/1.0/conf/filter", - "http://www.opengis.net/spec/ogcapi-features-3/1.0/conf/features-filter", -] -TILES_CONFORMS = [ - "http://www.opengis.net/spec/ogcapi-tiles-1/1.0/conf/core", - "http://www.opengis.net/spec/ogcapi-tiles-1/1.0/conf/oas30", - "http://www.opengis.net/spec/ogcapi-tiles-1/1.0/conf/mvt", - "http://www.opengis.net/spec/ogcapi-tiles-1/1.0/conf/tileset", - "http://www.opengis.net/spec/ogcapi-tiles-1/1.0/conf/tilesets-list", -] - - -def create_csv_rows(data: Iterable[Dict]) -> Generator[str, None, None]: - """Creates an iterator that returns lines of csv from an iterable of dicts.""" - - class DummyWriter: - """Dummy writer that implements write for use with csv.writer.""" - - def write(self, line: str): - """Return line.""" - return line - - # Get the first row and construct the column names - row = next(data) # type: ignore - fieldnames = row.keys() - writer = csv.DictWriter(DummyWriter(), fieldnames=fieldnames) - - # Write header - yield writer.writerow(dict(zip(fieldnames, fieldnames))) - - # Write first row - yield writer.writerow(row) - - # Write all remaining rows - for row in data: - yield writer.writerow(row) - - -def create_html_response( - request: Request, - data: Any, - templates: Jinja2Templates, - template_name: str, - title: Optional[str] = None, - router_prefix: Optional[str] = None, - **kwargs: Any, -) -> _TemplateResponse: - """Create Template response.""" - urlpath = request.url.path - if root_path := request.scope.get("root_path"): - urlpath = re.sub(r"^" + root_path, "", urlpath) - - if router_prefix: - urlpath = re.sub(r"^" + router_prefix, "", urlpath) - - crumbs = [] - baseurl = str(request.base_url).rstrip("/") - - if router_prefix: - baseurl += router_prefix - - crumbpath = str(baseurl) - if urlpath == "/": - urlpath = "" - - for crumb in urlpath.split("/"): - crumbpath = crumbpath.rstrip("/") - part = crumb - if part is None or part == "": - part = "Home" - crumbpath += f"/{crumb}" - crumbs.append({"url": crumbpath.rstrip("/"), "part": part.capitalize()}) - - return templates.TemplateResponse( - request, - name=f"{template_name}.html", - context={ - "response": data, - "template": { - "api_root": baseurl, - "params": request.query_params, - "title": title or template_name, - }, - "crumbs": crumbs, - "url": baseurl + urlpath, - "params": str(request.url.query), - **kwargs, - }, - ) - - -@dataclass -class FactoryExtension(metaclass=abc.ABCMeta): - """Factory Extension.""" - - @abc.abstractmethod - def register(self, factory: "EndpointsFactory"): - """Register extension to the factory.""" - ... - - -# ref: https://github.com/python/mypy/issues/5374 -@dataclass # type: ignore -class EndpointsFactory(metaclass=abc.ABCMeta): - """Endpoints Factory.""" - - # FastAPI router - router: APIRouter = field(default_factory=APIRouter) - - # collection dependency - collection_dependency: Callable[..., Collection] = CollectionParams - - # Router Prefix is needed to find the path for routes when prefixed - # e.g if you mount the route with `/foo` prefix, set router_prefix to foo - router_prefix: str = "" - - extensions: List[FactoryExtension] = field(default_factory=list) - - templates: Jinja2Templates = DEFAULT_TEMPLATES - - # Full application with Landing and Conformance - with_common: bool = True - - title: str = "OGC API" - - def __post_init__(self): - """Post Init: register route and configure specific options.""" - if self.with_common: - self._landing_route() - self._conformance_route() - - self.register_routes() - - # Register Extensions - for ext in self.extensions: - ext.register(self) - - def url_for(self, request: Request, name: str, **path_params: Any) -> str: - """Return full url (with prefix) for a specific handler.""" - url_path = self.router.url_path_for(name, **path_params) - - base_url = str(request.base_url) - if self.router_prefix: - prefix = self.router_prefix.lstrip("/") - # If we have prefix with custom path param we check and replace them with - # the path params provided - if "{" in prefix: - _, path_format, param_convertors = compile_path(prefix) - prefix, _ = replace_params( - path_format, param_convertors, request.path_params.copy() - ) - base_url += prefix - - return str(url_path.make_absolute_url(base_url=base_url)) - - def _create_html_response( - self, - request: Request, - data: Any, - template_name: str, - title: Optional[str] = None, - **kwargs: Any, - ) -> _TemplateResponse: - return create_html_response( - request, - data, - templates=self.templates, - template_name=template_name, - title=title, - router_prefix=self.router_prefix, - **kwargs, - ) - - @abc.abstractmethod - def register_routes(self): - """Register factory Routes.""" - ... - - @property - @abc.abstractmethod - def conforms_to(self) -> List[str]: - """Endpoints conformances.""" - ... - - @abc.abstractmethod - def links(self, request: Request) -> List[model.Link]: - """Register factory Routes.""" - ... - - def _conformance_route(self): - """Register Conformance (/conformance) route.""" - - @self.router.get( - "/conformance", - response_model=model.Conformance, - response_model_exclude_none=True, - response_class=ORJSONResponse, - responses={ - 200: { - "content": { - MediaType.json.value: {}, - MediaType.html.value: {}, - } - }, - }, - tags=["OGC Common"], - ) - def conformance( - request: Request, - output_type: Annotated[Optional[MediaType], Depends(OutputType)] = None, - ): - """Get conformance.""" - data = model.Conformance(conformsTo=[*COMMON_CONFORMS, *self.conforms_to]) - - if output_type == MediaType.html: - return self._create_html_response( - request, - data.model_dump(exclude_none=True, mode="json"), - template_name="conformance", - ) - - return data - - def _landing_route(self): - """Register Landing (/) and Conformance (/conformance) routes.""" - - @self.router.get( - "/", - response_model=model.Landing, - response_model_exclude_none=True, - response_class=ORJSONResponse, - responses={ - 200: { - "content": { - MediaType.json.value: {}, - MediaType.html.value: {}, - } - }, - }, - tags=["OGC Common"], - ) - def landing( - request: Request, - output_type: Annotated[Optional[MediaType], Depends(OutputType)] = None, - ): - """Get landing page.""" - data = model.Landing( - title=self.title, - links=[ - model.Link( - title="Landing Page", - href=self.url_for(request, "landing"), - type=MediaType.json, - rel="self", - ), - model.Link( - title="the API definition (JSON)", - href=str(request.url_for("openapi")), - type=MediaType.openapi30_json, - rel="service-desc", - ), - model.Link( - title="the API documentation", - href=str(request.url_for("swagger_ui_html")), - type=MediaType.html, - rel="service-doc", - ), - model.Link( - title="Conformance", - href=self.url_for(request, "conformance"), - type=MediaType.json, - rel="conformance", - ), - *self.links(request), - ], - ) - - if output_type == MediaType.html: - return self._create_html_response( - request, - data.model_dump(exclude_none=True, mode="json"), - template_name="landing", - title=self.title, - ) - - return data - - -@dataclass -class OGCFeaturesFactory(EndpointsFactory): - """OGC Features Endpoints Factory.""" - - # collections dependency - collections_dependency: Callable[..., CollectionList] = CollectionsParams - - @property - def conforms_to(self) -> List[str]: - """Factory conformances.""" - return FEATURES_CONFORMS - - def links(self, request: Request) -> List[model.Link]: - """OGC Features API links.""" - return [ - model.Link( - title="List of Collections", - href=self.url_for(request, "collections"), - type=MediaType.json, - rel="data", - ), - model.Link( - title="Collection metadata (Template URL)", - href=self.url_for( - request, - "collection", - collectionId="{collectionId}", - ), - type=MediaType.json, - rel="data", - templated=True, - ), - model.Link( - title="Collection queryables (Template URL)", - href=self.url_for( - request, - "queryables", - collectionId="{collectionId}", - ), - type=MediaType.schemajson, - rel="queryables", - templated=True, - ), - model.Link( - title="Collection Features (Template URL)", - href=self.url_for(request, "items", collectionId="{collectionId}"), - type=MediaType.geojson, - rel="data", - templated=True, - ), - model.Link( - title="Collection Feature (Template URL)", - href=self.url_for( - request, - "item", - collectionId="{collectionId}", - itemId="{itemId}", - ), - type=MediaType.geojson, - rel="data", - templated=True, - ), - ] - - def _additional_collection_tiles_links( - self, request: Request, collection: Collection - ) -> List[model.Link]: - links = [] - base_url = str(request.base_url) - try: - links.append( - model.Link( - rel="data", - title="Collection TileSets", - type=MediaType.json, - href=str( - request.app.url_path_for( - "collection_tileset_list", - collectionId=collection.id, - ).make_absolute_url(base_url=base_url) - ), - ), - ) - links.append( - model.Link( - rel="data", - title="Collection TileSet (Template URL)", - type=MediaType.json, - templated=True, - href=str( - request.app.url_path_for( - "collection_tileset", - collectionId=collection.id, - tileMatrixSetId="{tileMatrixSetId}", - ).make_absolute_url(base_url=base_url) - ), - ), - ) - except NoMatchFound: - pass - - try: - links.append( - model.Link( - title="Collection Map viewer (Template URL)", - href=str( - request.app.url_path_for( - "map_viewer", - collectionId=collection.id, - tileMatrixSetId="{tileMatrixSetId}", - ).make_absolute_url(base_url=base_url) - ), - type=MediaType.html, - rel="data", - templated=True, - ) - ) - - except NoMatchFound: - pass - - return links - - def register_routes(self): - """Register OGC Features endpoints.""" - self._collections_route() - self._collection_route() - self._queryables_route() - self._items_route() - self._item_route() - - def _collections_route(self): # noqa: C901 - @self.router.get( - "/collections", - response_model=model.Collections, - response_model_exclude_none=True, - response_class=ORJSONResponse, - responses={ - 200: { - "content": { - MediaType.json.value: {}, - MediaType.html.value: {}, - } - }, - }, - tags=["OGC Features API"], - ) - def collections( - request: Request, - collection_list: Annotated[ - CollectionList, - Depends(self.collections_dependency), - ], - output_type: Annotated[ - Optional[MediaType], - Depends(OutputType), - ] = None, - ): - """List of collections.""" - links: list = [ - model.Link( - href=self.url_for(request, "collections"), - rel="self", - type=MediaType.json, - ), - ] - - if next_token := collection_list["next"]: - query_params = QueryParams( - {**request.query_params, "offset": next_token} - ) - url = self.url_for(request, "collections") + f"?{query_params}" - links.append( - model.Link( - href=url, - rel="next", - type=MediaType.json, - title="Next page", - ), - ) - - if collection_list["prev"] is not None: - prev_token = collection_list["prev"] - qp = dict(request.query_params) - qp.pop("offset", None) - query_params = QueryParams({**qp, "offset": prev_token}) - url = self.url_for(request, "collections") - if query_params: - url += f"?{query_params}" - - links.append( - model.Link( - href=url, - rel="prev", - type=MediaType.json, - title="Previous page", - ), - ) - - data = model.Collections( - links=links, - numberMatched=collection_list["matched"], - numberReturned=len(collection_list["collections"]), - collections=[ - model.Collection( - id=collection.id, - title=collection.id, - description=collection.description, - extent=collection.extent, - links=[ - model.Link( - href=self.url_for( - request, - "collection", - collectionId=collection.id, - ), - rel="collection", - type=MediaType.json, - ), - model.Link( - href=self.url_for( - request, - "items", - collectionId=collection.id, - ), - rel="items", - type=MediaType.geojson, - ), - model.Link( - href=self.url_for( - request, - "queryables", - collectionId=collection.id, - ), - rel="queryables", - type=MediaType.schemajson, - ), - *self._additional_collection_tiles_links( - request, collection - ), - ], - ) - for collection in collection_list["collections"] - ], - ).model_dump(exclude_none=True, mode="json") - - if output_type == MediaType.html: - return self._create_html_response( - request, - data, - template_name="collections", - title="Collections list", - ) - - return ORJSONResponse(data) - - def _collection_route(self): - @self.router.get( - "/collections/{collectionId}", - response_model=model.Collection, - response_model_exclude_none=True, - response_class=ORJSONResponse, - responses={ - 200: { - "content": { - MediaType.json.value: {}, - MediaType.html.value: {}, - } - }, - }, - tags=["OGC Features API"], - ) - def collection( - request: Request, - collection: Annotated[Collection, Depends(self.collection_dependency)], - output_type: Annotated[Optional[MediaType], Depends(OutputType)] = None, - ): - """Metadata for a feature collection.""" - data = model.Collection( - id=collection.id, - title=collection.title, - description=collection.description, - extent=collection.extent, - links=[ - model.Link( - title="Collection", - href=self.url_for( - request, - "collection", - collectionId=collection.id, - ), - rel="self", - type=MediaType.json, - ), - model.Link( - title="Items", - href=self.url_for( - request, - "items", - collectionId=collection.id, - ), - rel="items", - type=MediaType.geojson, - ), - model.Link( - title="Items (CSV)", - href=self.url_for( - request, - "items", - collectionId=collection.id, - ) - + "?f=csv", - rel="alternate", - type=MediaType.csv, - ), - model.Link( - title="Items (GeoJSONSeq)", - href=self.url_for( - request, - "items", - collectionId=collection.id, - ) - + "?f=geojsonseq", - rel="alternate", - type=MediaType.geojsonseq, - ), - model.Link( - href=self.url_for( - request, - "queryables", - collectionId=collection.id, - ), - rel="queryables", - type=MediaType.schemajson, - ), - *self._additional_collection_tiles_links(request, collection), - ], - ).model_dump(exclude_none=True, mode="json") - - if output_type == MediaType.html: - return self._create_html_response( - request, - data, - template_name="collection", - title=f"{collection.id} collection", - ) - - return ORJSONResponse(data) - - def _queryables_route(self): - @self.router.get( - "/collections/{collectionId}/queryables", - response_model=model.Queryables, - response_model_exclude_none=True, - response_model_by_alias=True, - response_class=SchemaJSONResponse, - responses={ - 200: { - "content": { - MediaType.schemajson.value: {}, - MediaType.html.value: {}, - } - }, - }, - tags=["OGC Features API"], - ) - def queryables( - request: Request, - collection: Annotated[Collection, Depends(self.collection_dependency)], - output_type: Annotated[ - Optional[MediaType], Depends(QueryablesOutputType) - ] = None, - ): - """Queryables for a feature collection. - - ref: http://docs.ogc.org/DRAFTS/19-079r1.html#filter-queryables - """ - qs = "?" + str(request.query_params) if request.query_params else "" - self_url = self.url_for(request, "queryables", collectionId=collection.id) - data = model.Queryables( - title=collection.id, - link=self_url + qs, - properties=collection.queryables, - ).model_dump(exclude_none=True, mode="json", by_alias=True) - - if output_type == MediaType.html: - return self._create_html_response( - request, - data, - template_name="queryables", - title=f"{collection.id} queryables", - ) - - return SchemaJSONResponse(data) - - def _items_route(self): # noqa: C901 - @self.router.get( - "/collections/{collectionId}/items", - response_class=GeoJSONResponse, - responses={ - 200: { - "content": { - MediaType.geojson.value: {}, - MediaType.html.value: {}, - MediaType.csv.value: {}, - MediaType.json.value: {}, - MediaType.geojsonseq.value: {}, - MediaType.ndjson.value: {}, - }, - "model": model.Items, - }, - }, - tags=["OGC Features API"], - ) - async def items( # noqa: C901 - request: Request, - collection: Annotated[Collection, Depends(self.collection_dependency)], - ids_filter: Annotated[Optional[List[str]], Depends(ids_query)], - bbox_filter: Annotated[Optional[List[float]], Depends(bbox_query)], - datetime_filter: Annotated[Optional[List[str]], Depends(datetime_query)], - properties: Annotated[Optional[List[str]], Depends(properties_query)], - cql_filter: Annotated[Optional[Expr], Depends(filter_query)], - sortby: Annotated[Optional[str], Depends(sortby_query)], - geom_column: Annotated[ - Optional[str], - Query( - description="Select geometry column.", - alias="geom-column", - ), - ] = None, - datetime_column: Annotated[ - Optional[str], - Query( - description="Select datetime column.", - alias="datetime-column", - ), - ] = None, - limit: Annotated[ - int, - Query( - ge=0, - le=features_settings.max_features_per_query, - description="Limits the number of features in the response.", - ), - ] = features_settings.default_features_limit, - offset: Annotated[ - Optional[int], - Query( - ge=0, - description="Starts the response at an offset.", - ), - ] = None, - bbox_only: Annotated[ - Optional[bool], - Query( - description="Only return the bounding box of the feature.", - alias="bbox-only", - ), - ] = None, - simplify: Annotated[ - Optional[float], - Query( - description="Simplify the output geometry to given threshold in decimal degrees.", - ), - ] = None, - output_type: Annotated[ - Optional[MediaType], Depends(ItemsOutputType) - ] = None, - ): - output_type = output_type or MediaType.geojson - geom_as_wkt = output_type not in [ - MediaType.geojson, - MediaType.geojsonseq, - MediaType.html, - ] - - async with request.app.state.pool.acquire() as conn: - item_list = await collection.features( - conn, - ids_filter=ids_filter, - bbox_filter=bbox_filter, - datetime_filter=datetime_filter, - properties_filter=properties_filter_query(request, collection), - function_parameters=function_parameters_query(request, collection), - cql_filter=cql_filter, - sortby=sortby, - properties=properties, - limit=limit, - offset=offset, - geom=geom_column, - dt=datetime_column, - bbox_only=bbox_only, - simplify=simplify, - geom_as_wkt=geom_as_wkt, - ) - - if output_type in ( - MediaType.csv, - MediaType.json, - MediaType.ndjson, - ): - if any(f.get("geometry", None) is not None for f in item_list["items"]): - rows = ( - { - "collectionId": collection.id, - "itemId": f.get("id"), - **f.get("properties", {}), - "geometry": f.get("geometry", None), - } - for f in item_list["items"] - ) - else: - rows = ( - { - "collectionId": collection.id, - "itemId": f.get("id"), - **f.get("properties", {}), - } - for f in item_list["items"] - ) - - # CSV Response - if output_type == MediaType.csv: - return StreamingResponse( - create_csv_rows(rows), - media_type=MediaType.csv, - headers={ - "Content-Disposition": "attachment;filename=items.csv" - }, - ) - - # JSON Response - if output_type == MediaType.json: - return ORJSONResponse(list(rows)) - - # NDJSON Response - if output_type == MediaType.ndjson: - return StreamingResponse( - (orjsonDumps(row) + b"\n" for row in rows), - media_type=MediaType.ndjson, - headers={ - "Content-Disposition": "attachment;filename=items.ndjson" - }, - ) - - qs = "?" + str(request.query_params) if request.query_params else "" - links: List[Dict] = [ - { - "title": "Collection", - "href": self.url_for( - request, "collection", collectionId=collection.id - ), - "rel": "collection", - "type": "application/json", - }, - { - "title": "Items", - "href": self.url_for(request, "items", collectionId=collection.id) - + qs, - "rel": "self", - "type": "application/geo+json", - }, - ] - - if next_token := item_list["next"]: - query_params = QueryParams( - {**request.query_params, "offset": next_token} - ) - url = ( - self.url_for(request, "items", collectionId=collection.id) - + f"?{query_params}" - ) - links.append( - { - "href": url, - "rel": "next", - "type": "application/geo+json", - "title": "Next page", - }, - ) - - if item_list["prev"] is not None: - prev_token = item_list["prev"] - qp = dict(request.query_params) - qp.pop("offset") - query_params = QueryParams({**qp, "offset": prev_token}) - url = self.url_for(request, "items", collectionId=collection.id) - if query_params: - url += f"?{query_params}" - - links.append( - { - "href": url, - "rel": "prev", - "type": "application/geo+json", - "title": "Previous page", - }, - ) - - data = { - "type": "FeatureCollection", - "id": collection.id, - "title": collection.title or collection.id, - "description": collection.description - or collection.title - or collection.id, - "numberMatched": item_list["matched"], - "numberReturned": len(item_list["items"]), - "links": links, - "features": [ - { - **feature, # type: ignore - "links": [ - { - "title": "Collection", - "href": self.url_for( - request, - "collection", - collectionId=collection.id, - ), - "rel": "collection", - "type": "application/json", - }, - { - "title": "Item", - "href": self.url_for( - request, - "item", - collectionId=collection.id, - itemId=feature.get("id"), - ), - "rel": "item", - "type": "application/geo+json", - }, - ], - } - for feature in item_list["items"] - ], - } - - # HTML Response - if output_type == MediaType.html: - return self._create_html_response( - request, - orjson.loads(orjsonDumps(data).decode()), - template_name="items", - title=f"{collection.id} items", - ) - - # GeoJSONSeq Response - elif output_type == MediaType.geojsonseq: - return StreamingResponse( - (orjsonDumps(f) + b"\n" for f in data["features"]), # type: ignore - media_type=MediaType.geojsonseq, - headers={ - "Content-Disposition": "attachment;filename=items.geojson" - }, - ) - - # Default to GeoJSON Response - return GeoJSONResponse(data) - - def _item_route(self): - @self.router.get( - "/collections/{collectionId}/items/{itemId}", - response_class=GeoJSONResponse, - responses={ - 200: { - "content": { - MediaType.geojson.value: {}, - MediaType.html.value: {}, - MediaType.csv.value: {}, - MediaType.json.value: {}, - MediaType.geojsonseq.value: {}, - MediaType.ndjson.value: {}, - }, - "model": model.Item, - }, - }, - tags=["OGC Features API"], - ) - async def item( - request: Request, - collection: Annotated[Collection, Depends(self.collection_dependency)], - itemId: Annotated[str, Path(description="Item identifier")], - bbox_only: Annotated[ - Optional[bool], - Query( - description="Only return the bounding box of the feature.", - alias="bbox-only", - ), - ] = None, - simplify: Annotated[ - Optional[float], - Query( - description="Simplify the output geometry to given threshold in decimal degrees.", - ), - ] = None, - geom_column: Annotated[ - Optional[str], - Query( - description="Select geometry column.", - alias="geom-column", - ), - ] = None, - datetime_column: Annotated[ - Optional[str], - Query( - description="Select datetime column.", - alias="datetime-column", - ), - ] = None, - properties: Optional[List[str]] = Depends(properties_query), - output_type: Annotated[ - Optional[MediaType], Depends(ItemsOutputType) - ] = None, - ): - if collection.id_column is None: - raise NoPrimaryKey("No primary key is set on this table") - - output_type = output_type or MediaType.geojson - geom_as_wkt = output_type not in [ - MediaType.geojson, - MediaType.geojsonseq, - MediaType.html, - ] - - async with request.app.state.pool.acquire() as conn: - item_list = await collection.features( - conn, - ids_filter=[itemId], - bbox_only=bbox_only, - simplify=simplify, - properties=properties, - function_parameters=function_parameters_query(request, collection), - geom=geom_column, - dt=datetime_column, - geom_as_wkt=geom_as_wkt, - ) - - if not item_list["items"]: - raise NotFound( - f"Item {itemId} in Collection {collection.id} does not exist." - ) - - feature = item_list["items"][0] - - if output_type in ( - MediaType.csv, - MediaType.json, - MediaType.ndjson, - ): - row = { - "collectionId": collection.id, - "itemId": feature.get("id"), - **feature.get("properties", {}), - } - if feature.get("geometry") is not None: - row["geometry"] = (feature["geometry"],) - rows = iter([row]) - - # CSV Response - if output_type == MediaType.csv: - return StreamingResponse( - create_csv_rows(rows), - media_type=MediaType.csv, - headers={ - "Content-Disposition": "attachment;filename=items.csv" - }, - ) - - # JSON Response - if output_type == MediaType.json: - return ORJSONResponse(rows.__next__()) - - # NDJSON Response - if output_type == MediaType.ndjson: - return StreamingResponse( - (orjsonDumps(row) + b"\n" for row in rows), - media_type=MediaType.ndjson, - headers={ - "Content-Disposition": "attachment;filename=items.ndjson" - }, - ) - - data = { - **feature, # type: ignore - "links": [ - { - "href": self.url_for( - request, "collection", collectionId=collection.id - ), - "rel": "collection", - "type": "application/json", - }, - { - "href": self.url_for( - request, - "item", - collectionId=collection.id, - itemId=itemId, - ), - "rel": "self", - "type": "application/geo+json", - }, - ], - } - - # HTML Response - if output_type == MediaType.html: - return self._create_html_response( - request, - orjson.loads(orjsonDumps(data).decode()), - template_name="item", - title=f"{collection.id}/{itemId} item", - ) - - # Default to GeoJSON Response - return GeoJSONResponse(data) - - -@dataclass -class OGCTilesFactory(EndpointsFactory): - """OGC Tiles Endpoints Factory.""" - - supported_tms: TileMatrixSets = default_tms - with_viewer: bool = True - - @property - def conforms_to(self) -> List[str]: - """Factory conformances.""" - return TILES_CONFORMS - - def links(self, request: Request) -> List[model.Link]: - """OGC Tiles API links.""" - links = [ - model.Link( - title="Collection Vector Tiles (Template URL)", - href=self.url_for( - request, - "collection_get_tile", - collectionId="{collectionId}", - tileMatrixSetId="{tileMatrixSetId}", - z="{z}", - x="{x}", - y="{y}", - ), - type=MediaType.mvt, - rel="data", - templated=True, - ), - model.Link( - title="Collection TileSets (Template URL)", - href=self.url_for( - request, - "collection_tileset_list", - collectionId="{collectionId}", - ), - type=MediaType.json, - rel="data", - templated=True, - ), - model.Link( - title="Collection TileSet (Template URL)", - href=self.url_for( - request, - "collection_tileset", - collectionId="{collectionId}", - tileMatrixSetId="{tileMatrixSetId}", - ), - type=MediaType.json, - rel="data", - templated=True, - ), - ] - - if self.with_viewer: - links.append( - model.Link( - title="Collection Map viewer (Template URL)", - href=self.url_for( - request, - "map_viewer", - collectionId="{collectionId}", - tileMatrixSetId="{tileMatrixSetId}", - ), - type=MediaType.html, - rel="data", - templated=True, - ) - ) - - links += [ - model.Link( - title="TileMatrixSets", - href=self.url_for( - request, - "tilematrixsets", - ), - type=MediaType.json, - rel="http://www.opengis.net/def/rel/ogc/1.0/tiling-schemes", - ), - model.Link( - title="TileMatrixSet (Template URL)", - href=self.url_for( - request, - "tilematrixset", - tileMatrixSetId="{tileMatrixSetId}", - ), - type=MediaType.json, - rel="data", - templated=True, - ), - ] - - return links - - def register_routes(self): # noqa: C901 - """Register OGC Tiles endpoints.""" - self._tilematrixsets_routes() - self._tilesets_routes() - self._tile_routes() - self._tilejson_routes() - self._stylejson_routes() - - def _tilematrixsets_routes(self): - @self.router.get( - r"/tileMatrixSets", - response_model=model.TileMatrixSetList, - response_model_exclude_none=True, - summary="Retrieve the list of available tiling schemes (tile matrix sets).", - operation_id="getTileMatrixSetsList", - responses={ - 200: { - "content": { - MediaType.html.value: {}, - MediaType.json.value: {}, - }, - }, - }, - tags=["OGC Tiles API"], - ) - async def tilematrixsets( - request: Request, - output_type: Annotated[Optional[MediaType], Depends(OutputType)] = None, - ): - """ - OGC Specification: http://docs.opengeospatial.org/per/19-069.html#_tilematrixsets - """ - data = model.TileMatrixSetList( - tileMatrixSets=[ - model.TileMatrixSetRef( - id=tms_id, - title=f"Definition of {tms_id} tileMatrixSets", - links=[ - model.TileMatrixSetLink( - href=self.url_for( - request, - "tilematrixset", - tileMatrixSetId=tms_id, - ), - ) - ], - ) - for tms_id in self.supported_tms.list() - ] - ) - - if output_type == MediaType.html: - return self._create_html_response( - request, - data.model_dump(exclude_none=True, mode="json"), - template_name="tilematrixsets", - title="TileMatrixSets list", - ) - - return data - - @self.router.get( - "/tileMatrixSets/{tileMatrixSetId}", - response_model=TileMatrixSet, - response_model_exclude_none=True, - summary="Retrieve the definition of the specified tiling scheme (tile matrix set).", - operation_id="getTileMatrixSet", - responses={ - 200: { - "content": { - MediaType.html.value: {}, - MediaType.json.value: {}, - }, - }, - }, - tags=["OGC Tiles API"], - ) - async def tilematrixset( - request: Request, - tileMatrixSetId: Annotated[ - Literal[tuple(self.supported_tms.list())], - Path(description="Identifier for a supported TileMatrixSet."), - ], - output_type: Annotated[Optional[MediaType], Depends(OutputType)] = None, - ): - """ - OGC Specification: http://docs.opengeospatial.org/per/19-069.html#_tilematrixset - """ - tms = self.supported_tms.get(tileMatrixSetId) - - if output_type == MediaType.html: - return self._create_html_response( - request, - { - **tms.model_dump(exclude_none=True, mode="json"), - # For visualization purpose we add the tms bbox - "bbox": tms.bbox, - }, - template_name="tilematrixset", - title=f"{tileMatrixSetId} TileMatrixSet", - ) - - return tms - - def _tilesets_routes(self): - @self.router.get( - "/collections/{collectionId}/tiles", - response_model=model.TileSetList, - response_class=ORJSONResponse, - response_model_exclude_none=True, - responses={ - 200: { - "content": { - MediaType.json.value: {}, - MediaType.html.value: {}, - } - } - }, - summary="Retrieve a list of available vector tilesets for the specified collection.", - operation_id=".collection.vector.getTileSetsList", - tags=["OGC Tiles API"], - ) - async def collection_tileset_list( - request: Request, - collection: Annotated[Collection, Depends(self.collection_dependency)], - output_type: Annotated[Optional[MediaType], Depends(OutputType)] = None, - ): - """Retrieve a list of available vector tilesets for the specified collection.""" - collection_bbox = None - if bounds := collection.bounds: - collection_bbox = { - "lowerLeft": [bounds[0], bounds[1]], - "upperRight": [bounds[2], bounds[3]], - "crs": "http://www.opengis.net/def/crs/OGC/1.3/CRS84", - } - - data = model.TileSetList.model_validate( - { - "tilesets": [ - { - "title": f"'{collection.id}' tileset tiled using {tms} TileMatrixSet", - "dataType": "vector", - "crs": self.supported_tms.get(tms).crs, - "boundingBox": collection_bbox, - "links": [ - { - "href": self.url_for( - request, - "collection_tileset", - collectionId=collection.id, - tileMatrixSetId=tms, - ), - "rel": "self", - "type": "application/json", - "title": f"'{collection.id}' tileset tiled using {tms} TileMatrixSet", - }, - { - "href": self.url_for( - request, - "tilematrixset", - tileMatrixSetId=tms, - ), - "rel": "http://www.opengis.net/def/rel/ogc/1.0/tiling-scheme", - "type": "application/json", - "title": f"Definition of '{tms}' tileMatrixSet", - }, - { - "href": self.url_for( - request, - "collection_get_tile", - tileMatrixSetId=tms, - collectionId=collection.id, - z="{z}", - x="{x}", - y="{y}", - ), - "rel": "tile", - "type": "application/vnd.mapbox-vector-tile", - "title": "Templated link for retrieving Vector tiles", - }, - ], - } - for tms in self.supported_tms.list() - ] - } - ) - - if output_type == MediaType.html: - return self._create_html_response( - request, - data.model_dump(exclude_none=True, mode="json"), - template_name="tilesets", - title=f"{collection.id} tilesets", - ) - - return data - - @self.router.get( - "/collections/{collectionId}/tiles/{tileMatrixSetId}", - response_model=model.TileSet, - response_class=ORJSONResponse, - response_model_exclude_none=True, - responses={200: {"content": {MediaType.json.value: {}}}}, - summary="Retrieve the vector tileset metadata for the specified collection and tiling scheme (tile matrix set).", - operation_id=".collection.vector.getTileSet", - tags=["OGC Tiles API"], - ) - async def collection_tileset( - request: Request, - collection: Annotated[Collection, Depends(self.collection_dependency)], - tileMatrixSetId: Annotated[ - Literal[tuple(self.supported_tms.list())], - Path(description="Identifier for a supported TileMatrixSet."), - ], - output_type: Annotated[Optional[MediaType], Depends(OutputType)] = None, - ): - """Retrieve the vector tileset metadata for the specified collection and tiling scheme (tile matrix set).""" - tms = self.supported_tms.get(tileMatrixSetId) - - if bounds := collection.bounds: - collection_bbox = { - "lowerLeft": [bounds[0], bounds[1]], - "upperRight": [bounds[2], bounds[3]], - "crs": "http://www.opengis.net/def/crs/OGC/1.3/CRS84", - } - - tilematrix_limit = [] - for matrix in tms: - ulTile = tms.tile(bounds[0], bounds[3], int(matrix.id)) - lrTile = tms.tile(bounds[2], bounds[1], int(matrix.id)) - minx, maxx = (min(ulTile.x, lrTile.x), max(ulTile.x, lrTile.x)) - miny, maxy = (min(ulTile.y, lrTile.y), max(ulTile.y, lrTile.y)) - tilematrix_limit.append( - { - "tileMatrix": matrix.id, - "minTileRow": max(miny, 0), - "maxTileRow": min(maxy, matrix.matrixHeight), - "minTileCol": max(minx, 0), - "maxTileCol": min(maxx, matrix.matrixWidth), - } - ) - - else: - collection_bbox = None - tilematrix_limit = [ - { - "tileMatrix": matrix.id, - "minTileRow": 0, - "maxTileRow": matrix.matrixHeight, - "minTileCol": 0, - "maxTileCol": matrix.matrixWidth, - } - for matrix in tms - ] - - links = [ - { - "href": self.url_for( - request, - "collection_tileset", - collectionId=collection.id, - tileMatrixSetId=tileMatrixSetId, - ), - "rel": "self", - "type": "application/json", - "title": f"'{collection.id}' tileset tiled using {tileMatrixSetId} TileMatrixSet", - }, - { - "href": self.url_for( - request, - "tilematrixset", - tileMatrixSetId=tileMatrixSetId, - ), - "rel": "http://www.opengis.net/def/rel/ogc/1.0/tiling-scheme", - "type": "application/json", - "title": f"Definition of '{tileMatrixSetId}' tileMatrixSet", - }, - { - "href": self.url_for( - request, - "collection_get_tile", - tileMatrixSetId=tileMatrixSetId, - collectionId=collection.id, - z="{z}", - x="{x}", - y="{y}", - ), - "rel": "tile", - "type": "application/vnd.mapbox-vector-tile", - "title": "Templated link for retrieving Vector tiles", - "templated": True, - }, - ] - - if self.with_viewer: - links.append( - { - "href": self.url_for( - request, - "map_viewer", - tileMatrixSetId=tileMatrixSetId, - collectionId=collection.id, - ), - "type": "text/html", - "rel": "data", - "title": f"Map viewer for '{tileMatrixSetId}' tileMatrixSet", - } - ) - - data = model.TileSet.model_validate( - { - "title": f"'{collection.id}' tileset tiled using {tileMatrixSetId} TileMatrixSet", - "dataType": "vector", - "crs": tms.crs, - "boundingBox": collection_bbox, - "links": links, - "tileMatrixSetLimits": tilematrix_limit, - } - ) - - if output_type == MediaType.html: - return self._create_html_response( - request, - data.model_dump(exclude_none=True, mode="json"), - template_name="tileset", - title=f"{collection.id} {tileMatrixSetId} tileset", - ) - - return data - - def _tile_routes(self): - @self.router.get( - "/collections/{collectionId}/tiles/{tileMatrixSetId}/{z}/{x}/{y}", - response_class=Response, - responses={200: {"content": {MediaType.mvt.value: {}}}}, - operation_id=".collection.vector.getTileTms", - tags=["OGC Tiles API"], - ) - async def collection_get_tile( - request: Request, - collection: Annotated[Collection, Depends(self.collection_dependency)], - tileMatrixSetId: Annotated[ - Literal[tuple(self.supported_tms.list())], - Path( - description="Identifier selecting one of the TileMatrixSetId supported." - ), - ], - tile: Annotated[Tile, Depends(TileParams)], - ids_filter: Annotated[Optional[List[str]], Depends(ids_query)] = None, - bbox_filter: Annotated[Optional[List[float]], Depends(bbox_query)] = None, - datetime_filter: Annotated[ - Optional[List[str]], Depends(datetime_query) - ] = None, - properties: Annotated[ - Optional[List[str]], Depends(properties_query) - ] = None, - cql_filter: Annotated[Optional[Expr], Depends(filter_query)] = None, - sortby: Annotated[Optional[str], Depends(sortby_query)] = None, - geom_column: Annotated[ - Optional[str], - Query( - description="Select geometry column.", - alias="geom-column", - ), - ] = None, - datetime_column: Annotated[ - Optional[str], - Query( - description="Select datetime column.", - alias="datetime-column", - ), - ] = None, - limit: Annotated[ - Optional[int], - Query( - description="Limits the number of features in the response. Defaults to 10000 or TIPG_MAX_FEATURES_PER_TILE environment variable." - ), - ] = None, - ): - """Return Vector Tile.""" - tms = self.supported_tms.get(tileMatrixSetId) - - async with request.app.state.pool.acquire() as conn: - tile = await collection.get_tile( - conn, - tms=tms, - tile=tile, - ids_filter=ids_filter, - bbox_filter=bbox_filter, - datetime_filter=datetime_filter, - properties_filter=properties_filter_query(request, collection), - function_parameters=function_parameters_query(request, collection), - cql_filter=cql_filter, - sortby=sortby, - properties=properties, - limit=limit, - geom=geom_column, - dt=datetime_column, - ) - - return Response(tile, media_type=MediaType.mvt.value) - - def _tilejson_routes(self): - ############################################################################ - # ADDITIONAL ENDPOINTS NOT IN OGC Tiles API (tilejson, style.json, viewer) # - ############################################################################ - @self.router.get( - "/collections/{collectionId}/tiles/{tileMatrixSetId}/tilejson.json", - response_model=model.TileJSON, - responses={200: {"description": "Return a tilejson"}}, - response_model_exclude_none=True, - response_class=ORJSONResponse, - operation_id=".collection.vector.getTileJSONTms", - tags=["OGC Tiles API"], - ) - async def collection_tilejson( - request: Request, - collection: Annotated[Collection, Depends(self.collection_dependency)], - tileMatrixSetId: Annotated[ - Literal[tuple(self.supported_tms.list())], - Path( - description="Identifier selecting one of the TileMatrixSetId supported." - ), - ], - minzoom: Annotated[ - Optional[int], - Query(description="Overwrite default minzoom."), - ] = None, - maxzoom: Annotated[ - Optional[int], - Query(description="Overwrite default maxzoom."), - ] = None, - geom_column: Annotated[ - Optional[str], - Query( - description="Select geometry column.", - alias="geom-column", - ), - ] = None, - ): - """Return TileJSON document.""" - tms = self.supported_tms.get(tileMatrixSetId) - - geom = collection.get_geometry_column(geom_column) - if not geom: - raise MissingGeometryColumn - - path_params: Dict[str, Any] = { - "tileMatrixSetId": tileMatrixSetId, - "collectionId": collection.id, - "z": "{z}", - "x": "{x}", - "y": "{y}", - } - tile_endpoint = self.url_for(request, "collection_get_tile", **path_params) - - qs_key_to_remove = ["tilematrixsetid", "minzoom", "maxzoom"] - query_params = [ - (key, value) - for (key, value) in request.query_params._list - if key.lower() not in qs_key_to_remove - ] - - if query_params: - tile_endpoint += f"?{urlencode(query_params)}" - - # Get Min/Max zoom from layer settings if tms is the default tms - if tileMatrixSetId == tms_settings.default_tms: - minzoom = minzoom or tms_settings.default_minzoom - maxzoom = maxzoom or tms_settings.default_maxzoom - - minzoom = minzoom if minzoom is not None else tms.minzoom - maxzoom = maxzoom if maxzoom is not None else tms.maxzoom - - tile_json = { - "minzoom": minzoom, - "maxzoom": maxzoom, - "name": collection.id, - "tiles": [tile_endpoint], - } - if collection.bounds: - tile_json["bounds"] = collection.bounds - - layername = collection.id if mvt_settings.set_mvt_layername else "default" - tile_json["vector_layers"] = [ - { - "id": layername, - "fields": { - col.name: col.json_type - for col in collection.properties - if col.type not in ["geometry", "geography"] - }, - "minzoom": minzoom, - "maxzoom": maxzoom, - } - ] - - return tile_json - - def _stylejson_routes(self): - @self.router.get( - "/collections/{collectionId}/tiles/{tileMatrixSetId}/style.json", - response_model=model.StyleJSON, - responses={200: {"description": "Return a tilejson"}}, - response_model_exclude_none=True, - response_class=ORJSONResponse, - operation_id=".collection.vector.getStyleJSONTms", - tags=["OGC Tiles API"], - ) - async def collection_stylejson( - request: Request, - collection: Annotated[Collection, Depends(self.collection_dependency)], - tileMatrixSetId: Annotated[ - Literal[tuple(self.supported_tms.list())], - Path( - description="Identifier selecting one of the TileMatrixSetId supported." - ), - ], - geom_column: Annotated[ - Optional[str], - Query( - description="Select geometry column.", - alias="geom-column", - ), - ] = None, - minzoom: Annotated[ - Optional[int], - Query(description="Overwrite default minzoom."), - ] = None, - maxzoom: Annotated[ - Optional[int], - Query(description="Overwrite default maxzoom."), - ] = None, - ): - """Return Mapbox/Maplibre StyleJSON document.""" - tms = self.supported_tms.get(tileMatrixSetId) - - geom = collection.get_geometry_column(geom_column) - if not geom: - raise MissingGeometryColumn - - path_params: Dict[str, Any] = { - "collectionId": collection.id, - "tileMatrixSetId": tileMatrixSetId, - "z": "{z}", - "x": "{x}", - "y": "{y}", - } - tiles_endpoint = self.url_for(request, "collection_get_tile", **path_params) - - qs_key_to_remove = ["tilematrixsetid", "minzoom", "maxzoom"] - query_params = [ - (key, value) - for (key, value) in request.query_params._list - if key.lower() not in qs_key_to_remove - ] - if query_params: - tiles_endpoint += f"?{urlencode(query_params)}" - - # Get Min/Max zoom from layer settings if tms is the default tms - if tileMatrixSetId == tms_settings.default_tms: - minzoom = minzoom or tms_settings.default_minzoom - maxzoom = maxzoom or tms_settings.default_maxzoom - - minzoom = minzoom if minzoom is not None else tms.minzoom - maxzoom = maxzoom if maxzoom is not None else tms.maxzoom - - bounds = list(collection.bounds) or list(tms.bbox) - - style_json = { - "name": "TiPg", - "sources": { - collection.id: { - "type": "vector", - "scheme": "xyz", - "tiles": [tiles_endpoint], - "bounds": bounds, - "minzoom": minzoom, - "maxzoom": maxzoom, - } - }, - "layers": [], - "center": [ - (bounds[0] + bounds[2]) / 2, - (bounds[1] + bounds[3]) / 2, - ], - "zoom": minzoom, - } - - layername = collection.id if mvt_settings.set_mvt_layername else "default" - style_json["layers"] = [ - { - "id": f"{collection.id}_fill", - "source": collection.id, - "source-layer": layername, - "type": "fill", - "filter": ["==", ["geometry-type"], "Polygon"], - "paint": { - "fill-color": "rgba(200, 100, 240, 0.4)", - "fill-outline-color": "#000", - }, - }, - { - "id": f"{collection.id}_stroke", - "source": collection.id, - "source-layer": layername, - "type": "line", - "filter": ["==", ["geometry-type"], "LineString"], - "paint": { - "line-color": "#000", - "line-width": 1, - "line-opacity": 0.75, - }, - }, - { - "id": f"{collection.id}_point", - "source": collection.id, - "source-layer": layername, - "type": "circle", - "filter": ["==", ["geometry-type"], "Point"], - "paint": { - "circle-color": "#000", - "circle-radius": 2.5, - "circle-opacity": 0.75, - }, - }, - ] - - return style_json - - if self.with_viewer: - - @self.router.get( - "/collections/{collectionId}/tiles/{tileMatrixSetId}/map.html", - response_class=HTMLResponse, - operation_id=".collection.vector.map", - tags=["Map Viewer"], - ) - def map_viewer( - request: Request, - collection: Annotated[Collection, Depends(self.collection_dependency)], - tileMatrixSetId: Annotated[ - Literal[tuple(self.supported_tms.list())], - Path( - description="Identifier selecting one of the TileMatrixSetId supported." - ), - ], - minzoom: Annotated[ - Optional[int], - Query(description="Overwrite default minzoom."), - ] = None, - maxzoom: Annotated[ - Optional[int], - Query(description="Overwrite default maxzoom."), - ] = None, - geom_column: Annotated[ - Optional[str], - Query( - description="Select geometry column.", - alias="geom-column", - ), - ] = None, - ): - """Return Simple HTML Viewer for a collection.""" - tms = self.supported_tms.get(tileMatrixSetId) - - tilejson_url = self.url_for( - request, - "collection_tilejson", - collectionId=collection.id, - tileMatrixSetId=tileMatrixSetId, - ) - if request.query_params._list: - tilejson_url += f"?{urlencode(request.query_params._list)}" - - return self._create_html_response( - request, - { - "title": collection.id, - "tilejson_endpoint": tilejson_url, - "tms": tms, - "resolutions": [matrix.cellSize for matrix in tms], - }, - template_name="map", - title=f"{collection.id} viewer", - ) - - -@dataclass -class Endpoints(EndpointsFactory): - """OGC Features and Tiles Endpoints Factory.""" - - # OGC Features dependency - collections_dependency: Callable[..., CollectionList] = CollectionsParams - - # OGC Tiles dependency - supported_tms: TileMatrixSets = default_tms - with_tiles_viewer: bool = True - - ogc_features: OGCFeaturesFactory = field(init=False) - ogc_tiles: OGCTilesFactory = field(init=False) - - @property - def conforms_to(self) -> List[str]: - """Endpoints conformances.""" - return [ - *self.ogc_features.conforms_to, - *self.ogc_tiles.conforms_to, - ] - - def links(self, request: Request) -> List[model.Link]: - """List of available links.""" - return [ - *self.ogc_features.links(request), - *self.ogc_tiles.links(request), - ] - - def register_routes(self): - """Register factory Routes.""" - self.ogc_features = OGCFeaturesFactory( - collections_dependency=self.collections_dependency, - collection_dependency=self.collection_dependency, - router_prefix=self.router_prefix, - templates=self.templates, - # We do not want `/` and `/conformance` from the factory - with_common=False, - ) - self.router.include_router(self.ogc_features.router) - - self.ogc_tiles = OGCTilesFactory( - collection_dependency=self.collection_dependency, - router_prefix=self.router_prefix, - templates=self.templates, - supported_tms=self.supported_tms, - with_viewer=self.with_tiles_viewer, - # We do not want `/` and `/conformance` from the factory - with_common=False, - ) - self.router.include_router(self.ogc_tiles.router) +from tipg.factories import * # noqa: F401,F403 +from tipg.factories import __all__ as __all__ # noqa: F401 diff --git a/tipg/filter.py b/tipg/filter.py new file mode 100644 index 00000000..e3bf8355 --- /dev/null +++ b/tipg/filter.py @@ -0,0 +1,296 @@ +"""tipg.filter: build cql2 filter expressions for the WHERE clause. + +Everything here turns tipg's query parameters (ids / bbox / datetime / +properties / tile envelope) and user-supplied CQL into a single ``cql2.Expr`` +that the caller renders to SQL via ``Expr.to_sql()``. +""" + +from functools import reduce +from typing import TYPE_CHECKING, Any, List, Optional, Tuple + +from ciso8601 import parse_rfc3339 +from cql2 import Expr +from morecantile import Tile, TileMatrixSet + +from tipg.errors import ( + InvalidDatetime, + InvalidDatetimeColumnName, + InvalidPropertyName, + MissingDatetimeColumn, + NotFound, +) +from tipg.sqlhelpers import TransformerFromCRS + +if TYPE_CHECKING: + from tipg.dbmodel import Collection + + +_INT_PG_TYPES = frozenset( + { + "smallint", + "integer", + "bigint", + "smallserial", + "serial", + "bigserial", + } +) + + +def _coerce_id(val: str, pg_type: str) -> Any: + """Parse a URL-supplied id into the primary-key column's Python type. + + IDs always arrive as strings (URL path or query parameters); the + underlying column is either text or integer. cql2 treats Python ints + and strs as different literal kinds (and PostgreSQL ``= ANY(text[])`` + does not coerce array elements at runtime), so we normalize here. + + A non-integer string against an integer column means the id can't + exist, so map the parse failure onto a 404 rather than letting the + eventual PostgreSQL ``invalid input syntax`` error bubble up as a 500. + """ + if pg_type not in _INT_PG_TYPES: + return val + try: + return int(val) + except ValueError as exc: + raise NotFound(f"Invalid id {val!r} for {pg_type} column.") from exc + + +def _s_intersects_bbox( + prop: str, + west: float, + south: float, + east: float, + north: float, + srid: Optional[int] = None, +) -> Expr: + """Build a cql2 S_INTERSECTS expression against a polygon envelope. + + Coordinates are taken to be in ``srid``; if ``srid`` is non-4326, the + polygon literal is wrapped in ``ST_SetSRID`` so PostGIS does not pick + up ``ST_GeomFromGeoJSON``'s 4326 default. Pass ``srid=None`` or + ``srid=4326`` for the no-op case. + """ + polygon: Any = { + "type": "Polygon", + "coordinates": [ + [ + [west, south], + [east, south], + [east, north], + [west, north], + [west, south], + ] + ], + } + if srid is not None and srid != 4326: + polygon = {"op": "st_setsrid", "args": [polygon, srid]} + return Expr({"op": "s_intersects", "args": [{"property": prop}, polygon]}) + + +_GEOJSON_GEOMETRY_TYPES = frozenset( + { + "Point", + "MultiPoint", + "LineString", + "MultiLineString", + "Polygon", + "MultiPolygon", + "GeometryCollection", + } +) + + +def _transform_cql_geom_literals(node: Any, target_srid: int) -> Any: + """Walk a user-supplied cql2-json tree and wrap every GeoJSON geometry + literal in ``ST_Transform(, target_srid)``. + + Per OGC API Features Part 3 Req 7, geometries in a CQL filter are + CRS84 (≈ EPSG:4326). Wrapping the literal side (rather than the + column side) keeps PostGIS' spatial index on the column usable and + lets the planner fold the transform on the constant once. + """ + if isinstance(node, dict): + if node.get("type") in _GEOJSON_GEOMETRY_TYPES and ( + "coordinates" in node or "geometries" in node + ): + return {"op": "st_transform", "args": [node, target_srid]} + return { + k: _transform_cql_geom_literals(v, target_srid) for k, v in node.items() + } + if isinstance(node, list): + return [_transform_cql_geom_literals(item, target_srid) for item in node] + return node + + +def cql_where( # noqa: C901 + collection: "Collection", + ids: Optional[List[str]] = None, + datetime: Optional[List[str]] = None, + bbox: Optional[List[float]] = None, + properties: Optional[List[Tuple[str, Any]]] = None, + cql: Optional[Expr] = None, + geom: Optional[str] = None, + dt: Optional[str] = None, + tile: Optional[Tile] = None, + tms: Optional[TileMatrixSet] = None, +) -> Optional[Expr]: + """Construct WHERE query as a cql2 Expr.""" + exprs: List[Expr] = [] + geometry_column = collection.get_geometry_column(geom) + + # Per OGC API Features Part 3 Req 7, geometries in `cql` and `bbox` + # are CRS84 (≈ EPSG:4326). When the stored column is in another + # SRID, wrap their literals in `ST_Transform(literal, )` + # so the comparison happens in the column's native CRS — PostGIS + # folds the transform on the constant side and the spatial index + # on the column stays usable. + col_srid = geometry_column.srid if geometry_column is not None else None + needs_srid_wrap = col_srid is not None and col_srid != 4326 + + if cql is not None: + if needs_srid_wrap: + cql = Expr(_transform_cql_geom_literals(cql.to_json(), col_srid)) + exprs.append(cql) + + # `ids` filter + if ids: + id_prop = {"property": collection.id_column.name} + typed_ids = [_coerce_id(i, collection.id_column.type) for i in ids] + if len(typed_ids) == 1: + exprs.append(Expr({"op": "=", "args": [id_prop, typed_ids[0]]})) + else: + exprs.append(Expr({"op": "in", "args": [id_prop, typed_ids]})) + + # `properties` filter + if properties is not None: + for prop, val in properties: + if not collection.get_column(prop): + raise InvalidPropertyName(f"Invalid property name: {prop}") + exprs.append(Expr({"op": "=", "args": [{"property": prop}, val]})) + + # `bbox` filter — bbox is CRS84 (4326) per OGC API Features. + # Reproject the four corner coords to the column's CRS in Python + # so the polygon literal already carries target-CRS coordinates; + # ST_SetSRID then just tags it, no PostGIS-side transform needed. + if bbox is not None and geometry_column is not None: + if len(bbox) == 6: + west, south, _, east, north, _ = bbox + else: + west, south, east, north = bbox + if needs_srid_wrap: + transformer = TransformerFromCRS(4326, col_srid, always_xy=True) + west, south, east, north = transformer.transform_bounds( + west, south, east, north + ) + exprs.append( + _s_intersects_bbox( + geometry_column.name, west, south, east, north, srid=col_srid + ) + ) + + # `datetime` filter + if datetime: + if not collection.datetime_columns: + raise MissingDatetimeColumn( + "Must have timestamp/timestamptz/date typed column to filter with datetime." + ) + + datetime_column = collection.get_datetime_column(dt) + if not datetime_column: + raise InvalidDatetimeColumnName(f"Invalid Datetime Column: {dt}.") + + dt_prop = {"property": datetime_column.name} + + if len(datetime) == 1: + parse_rfc3339(datetime[0]) + exprs.append( + Expr( + { + "op": "=", + "args": [dt_prop, {"timestamp": datetime[0]}], + } + ) + ) + else: + start_str, end_str = datetime[0], datetime[1] + start = parse_rfc3339(start_str) if start_str not in ["..", ""] else None + end = parse_rfc3339(end_str) if end_str not in ["..", ""] else None + + if start is None and end is None: + raise InvalidDatetime( + "Double open-ended datetime intervals are not allowed." + ) + + if start is not None and end is not None and start > end: + raise InvalidDatetime("Start datetime cannot be before end datetime.") + + if start is not None: + exprs.append( + Expr( + { + "op": ">=", + "args": [dt_prop, {"timestamp": start_str}], + } + ) + ) + if end is not None: + # TODO: Understand the correct way to handle inclusive/exclusive end + # Closed interval uses exclusive upper bound to keep + # the half-open `[start, end)` semantics tipg has always + # used; the open-ended `../` form remains inclusive + # (`<=`) + op = "<" if start is not None else "<=" + exprs.append( + Expr( + { + "op": op, + "args": [dt_prop, {"timestamp": end_str}], + } + ) + ) + + # `tile` envelope filter — reproject tile bounds directly from the + # TMS CRS to the stored column's CRS (skipping the 4326 round-trip) + # and tag the resulting polygon literal with `ST_SetSRID` so it + # carries the column's SRID rather than the 4326 default that + # `ST_GeomFromGeoJSON` would otherwise apply. + if tile and tms and geometry_column: + bounds = tms.xy_bounds(tile) + west, south, east, north = ( + bounds.left, + bounds.bottom, + bounds.right, + bounds.top, + ) + # Bounds are now in TMS CRS + tms_srid = tms.crs.to_epsg() or 4326 + tile_target_srid = col_srid if col_srid is not None else 4326 + + # If the TMS CRS is different from the column's SRID, + # transform the bounds to the column's SRID. + if tms_srid != tile_target_srid: + transformer = TransformerFromCRS(tms_srid, tile_target_srid, always_xy=True) + west, south, east, north = transformer.transform_bounds( + west, south, east, north + ) + + exprs.append( + _s_intersects_bbox( + geometry_column.name, + west, + south, + east, + north, + srid=tile_target_srid, + ) + ) + + if exprs: + # NOTE: do not call .reduce() — cql2-rs constant-folds some + # predicates (e.g. `"numeric" IS NULL`) incorrectly. + # TODO: Open a bug in cql2-rs for this. + return reduce(lambda x, y: x + y, exprs) + + return None diff --git a/tipg/middleware.py b/tipg/middleware.py index 50b6a44b..f7c931ed 100644 --- a/tipg/middleware.py +++ b/tipg/middleware.py @@ -4,7 +4,7 @@ from datetime import datetime, timedelta from typing import Any, Optional, Protocol, Set -from tipg.collections import Catalog +from tipg.dbmodel import Catalog from tipg.errors import MissingCollectionCatalog from tipg.logger import logger diff --git a/tipg/model.py b/tipg/model.py index 94742751..2816f786 100644 --- a/tipg/model.py +++ b/tipg/model.py @@ -1,955 +1,11 @@ -"""tipg models.""" +"""Backwards-compat shim for the model package. -from datetime import datetime -from typing import Annotated, Dict, List, Literal, Optional, Set, Tuple, Union +The response models were split into the ``tipg.models`` package +(``tipg.models.common`` / ``tipg.models.features`` / ``tipg.models.tiles``). +This module re-exports them so existing ``from tipg.model import ...`` and +``from tipg import model`` imports keep working. Prefer importing from +``tipg.models`` directly. +""" -from geojson_pydantic.features import Feature, FeatureCollection -from morecantile.models import CRSType -from pydantic import AnyUrl, BaseModel, Field, RootModel, model_validator - -from tipg.resources.enums import MediaType - - -class Link(BaseModel): - """Link model. - - Ref: https://github.com/opengeospatial/ogcapi-tiles/blob/master/openapi/schemas/common-core/link.yaml - - Code generated using https://github.com/koxudaxi/datamodel-code-generator/ - """ - - href: Annotated[ - str, - Field( - json_schema_extra={ - "description": "Supplies the URI to a remote resource (or resource fragment).", - "examples": ["http://data.example.com/buildings/123"], - } - ), - ] - rel: Annotated[ - str, - Field( - json_schema_extra={ - "description": "The type or semantics of the relation.", - "examples": ["alternate"], - } - ), - ] - type: Annotated[ - Optional[MediaType], - Field( - json_schema_extra={ - "description": "A hint indicating what the media type of the result of dereferencing the link should be.", - "examples": ["application/geo+json"], - } - ), - ] = None - templated: Annotated[ - Optional[bool], - Field( - json_schema_extra={ - "description": "This flag set to true if the link is a URL template.", - } - ), - ] = None - varBase: Annotated[ - Optional[str], - Field( - json_schema_extra={ - "description": "A base path to retrieve semantic information about the variables used in URL template.", - "examples": ["/ogcapi/vars/"], - } - ), - ] = None - hreflang: Annotated[ - Optional[str], - Field( - json_schema_extra={ - "description": "A hint indicating what the language of the result of dereferencing the link should be.", - "examples": ["en"], - } - ), - ] = None - title: Annotated[ - Optional[str], - Field( - json_schema_extra={ - "description": "Used to label the destination of a link such that it can be used as a human-readable identifier.", - "examples": ["Trierer Strasse 70, 53115 Bonn"], - } - ), - ] = None - length: Optional[int] = None - - model_config = {"use_enum_values": True} - - -class Spatial(BaseModel): - """Spatial Extent model. - - Ref: http://schemas.opengis.net/ogcapi/features/part1/1.0/openapi/schemas/extent.yaml - - """ - - # Bbox - # One or more bounding boxes that describe the spatial extent of the dataset. - # The first bounding box describes the overall spatial - # extent of the data. All subsequent bounding boxes describe - # more precise bounding boxes, e.g., to identify clusters of data. - bbox: List[List[float]] - crs: str = "http://www.opengis.net/def/crs/OGC/1.3/CRS84" - - -class Temporal(BaseModel): - """Temporal Extent model. - - Ref: http://schemas.opengis.net/ogcapi/features/part1/1.0/openapi/schemas/extent.yaml - - """ - - # The first time interval describes the overall - # temporal extent of the data. All subsequent time intervals describe - # more precise time intervals, e.g., to identify clusters of data. - # Clients only interested in the overall temporal extent will only need - # to access the first time interval in the array (a pair of lower and upper - # bound instants). - interval: List[List[Optional[str]]] - trs: str = "http://www.opengis.net/def/uom/ISO-8601/0/Gregorian" - - -class Extent(BaseModel): - """Extent model. - - Ref: http://schemas.opengis.net/ogcapi/features/part1/1.0/openapi/schemas/extent.yaml - - """ - - spatial: Optional[Spatial] = None - temporal: Optional[Temporal] = None - - -class Collection(BaseModel): - """Collection model. - - Note: `CRS` is the list of CRS supported by the service not the CRS of the collection - - Ref: http://schemas.opengis.net/ogcapi/features/part1/1.0/openapi/schemas/collection.yaml - - """ - - id: str - title: Optional[str] = None - description: Optional[str] = None - links: List[Link] - extent: Optional[Extent] = None - itemType: str = "feature" - crs: List[str] = ["http://www.opengis.net/def/crs/OGC/1.3/CRS84"] - - model_config = {"extra": "ignore"} - - -class Collections(BaseModel): - """ - Collections model. - - Ref: http://beta.schemas.opengis.net/ogcapi/common/part2/0.1/collections/openapi/schemas/collections.yaml - - """ - - links: List[Link] - timeStamp: Optional[str] = None - numberMatched: Optional[int] = None - numberReturned: Optional[int] = None - collections: List[Collection] - - model_config = {"extra": "allow"} - - -class Item(Feature): - """Item model - - Ref: http://schemas.opengis.net/ogcapi/features/part1/1.0/openapi/schemas/featureGeoJSON.yaml - - """ - - links: Optional[List[Link]] = None - - model_config = {"arbitrary_types_allowed": True} - - -class Items(FeatureCollection): - """Items model - - Ref: http://schemas.opengis.net/ogcapi/features/part1/1.0/openapi/schemas/featureCollectionGeoJSON.yaml - - """ - - id: str - title: Optional[str] = None - description: Optional[str] = None - keywords: Optional[List[str]] = None - features: List[Item] - links: Optional[List[Link]] = None - timeStamp: Optional[str] = None - numberMatched: Optional[int] = None - numberReturned: Optional[int] = None - - model_config = {"arbitrary_types_allowed": True} - - def json_seq(self, **kwargs): - """return a GeoJSON sequence representation.""" - for f in self.features: - yield f.json(**kwargs) + "\n" - - -class Conformance(BaseModel): - """Conformance model. - - Ref: http://schemas.opengis.net/ogcapi/features/part1/1.0/openapi/schemas/confClasses.yaml - - """ - - conformsTo: List[str] - - -class Landing(BaseModel): - """Landing page model. - - Ref: http://schemas.opengis.net/ogcapi/features/part1/1.0/openapi/schemas/landingPage.yaml - - """ - - title: Optional[str] = None - description: Optional[str] = None - links: List[Link] - - -class Queryables(BaseModel): - """Queryables model. - - Ref: https://docs.ogc.org/DRAFTS/19-079r1.html#filter-queryables - - """ - - title: str - properties: Dict[str, Dict[str, str]] - type: str = "object" - schema_name: Annotated[str, Field(alias="$schema")] = ( - "https://json-schema.org/draft/2019-09/schema" - ) - link: Annotated[str, Field(alias="$id")] - - model_config = {"populate_by_name": True} - - -class TileMatrixSetLink(BaseModel): - """ - TileMatrixSetLink model. - Based on http://docs.opengeospatial.org/per/19-069.html#_tilematrixsets - """ - - href: str - rel: str = "http://www.opengis.net/def/rel/ogc/1.0/tiling-scheme" - type: MediaType = MediaType.json - - model_config = {"use_enum_values": True} - - -class TileMatrixSetRef(BaseModel): - """ - TileMatrixSetRef model. - Based on http://docs.opengeospatial.org/per/19-069.html#_tilematrixsets - """ - - id: str - title: Optional[str] = None - links: List[TileMatrixSetLink] - - -class TileMatrixSetList(BaseModel): - """ - TileMatrixSetList model. - Based on http://docs.opengeospatial.org/per/19-069.html#_tilematrixsets - """ - - tileMatrixSets: List[TileMatrixSetRef] - - -class LayerJSON(BaseModel): - """ - https://github.com/mapbox/tilejson-spec/tree/master/3.0.0#33-vector_layers - """ - - id: str - fields: Annotated[Dict, Field(default_factory=dict)] - description: Optional[str] = None - minzoom: Optional[int] = None - maxzoom: Optional[int] = None - - -class TileJSON(BaseModel): - """ - TileJSON model. - - Based on https://github.com/mapbox/tilejson-spec/tree/master/3.0.0 - """ - - tilejson: str = "3.0.0" - name: Optional[str] = None - description: Optional[str] = None - version: str = "1.0.0" - attribution: Optional[str] = None - template: Optional[str] = None - legend: Optional[str] = None - scheme: Literal["xyz", "tms"] = "xyz" - tiles: List[str] - vector_layers: Optional[List[LayerJSON]] = None - grids: Optional[List[str]] = None - data: Optional[List[str]] = None - minzoom: int = Field(0) - maxzoom: int = Field(30) - fillzoom: Optional[int] = None - bounds: List[float] = [180, -85.05112877980659, 180, 85.0511287798066] - center: Optional[Tuple[float, float, int]] = None - - @model_validator(mode="after") - def compute_center(self): - """Compute center if it does not exist.""" - bounds = self.bounds - if not self.center: - self.center = ( - (bounds[0] + bounds[2]) / 2, - (bounds[1] + bounds[3]) / 2, - self.minzoom, - ) - return self - - -class StyleJSON(BaseModel): - """ - Simple Mapbox/Maplibre Style JSON model. - - Based on https://docs.mapbox.com/help/glossary/style/ - - """ - - version: int = 8 - name: Optional[str] = None - metadata: Optional[Dict] = None - layers: List[Dict] - sources: Dict - center: List[float] = [0, 0] - zoom: int = 1 - - -class TimeStamp(RootModel): - """TimeStamp model. - - Ref: https://github.com/opengeospatial/ogcapi-tiles/blob/master/openapi/schemas/common-geodata/timeStamp.yaml - - Code generated using https://github.com/koxudaxi/datamodel-code-generator/ - """ - - root: Annotated[ - datetime, - Field( - json_schema_extra={ - "description": "This property indicates the time and date when the response was generated using RFC 3339 notation.", - "examples": ["2017-08-17T08:05:32Z"], - } - ), - ] - - -class BoundingBox(BaseModel): - """BoundingBox model. - - Ref: https://github.com/opengeospatial/ogcapi-tiles/blob/master/openapi/schemas/tms/2DBoundingBox.yaml - - Code generated using https://github.com/koxudaxi/datamodel-code-generator/ - """ - - lowerLeft: Annotated[ - List[float], - Field( - max_length=2, - min_length=2, - json_schema_extra={ - "description": "A 2D Point in the CRS indicated elsewhere", - }, - ), - ] - upperRight: Annotated[ - List[float], - Field( - max_length=2, - min_length=2, - json_schema_extra={ - "description": "A 2D Point in the CRS indicated elsewhere", - }, - ), - ] - crs: Annotated[Optional[CRSType], Field(json_schema_extra={"title": "CRS"})] = None - orderedAxes: Annotated[Optional[List[str]], Field(max_length=2, min_length=2)] = ( - None - ) - - -# Ref: https://github.com/opengeospatial/ogcapi-tiles/blob/master/openapi/schemas/tms/propertiesSchema.yaml -Type = Literal["array", "boolean", "integer", "null", "number", "object", "string"] - -# Ref: https://github.com/opengeospatial/ogcapi-tiles/blob/master/openapi/schemas/tms/propertiesSchema.yaml -AccessConstraints = Literal[ - "unclassified", "restricted", "confidential", "secret", "topSecret" -] - - -class Properties(BaseModel): - """Properties model. - - Ref: https://github.com/opengeospatial/ogcapi-tiles/blob/master/openapi/schemas/tms/propertiesSchema.yaml - - Code generated using https://github.com/koxudaxi/datamodel-code-generator/ - """ - - title: Optional[str] = None - description: Annotated[ - Optional[str], - Field( - json_schema_extra={ - "description": "Implements 'description'", - } - ), - ] = None - type: Optional[Type] = None - enum: Annotated[ - Optional[Set], - Field( - min_length=1, - json_schema_extra={ - "description": "Implements 'acceptedValues'", - }, - ), - ] = None - format: Annotated[ - Optional[str], - Field( - json_schema_extra={ - "description": "Complements implementation of 'type'", - } - ), - ] = None - contentMediaType: Annotated[ - Optional[str], - Field( - json_schema_extra={ - "description": "Implements 'mediaType'", - } - ), - ] = None - maximum: Annotated[ - Optional[float], - Field( - json_schema_extra={ - "description": "Implements 'range'", - } - ), - ] = None - exclusiveMaximum: Annotated[ - Optional[float], - Field( - json_schema_extra={ - "description": "Implements 'range'", - } - ), - ] = None - minimum: Annotated[ - Optional[float], - Field( - json_schema_extra={ - "description": "Implements 'range'", - } - ), - ] = None - exclusiveMinimum: Annotated[ - Optional[float], - Field( - json_schema_extra={ - "description": "Implements 'range'", - } - ), - ] = None - pattern: Optional[str] = None - maxItems: Annotated[ - Optional[int], - Field( - ge=0, - json_schema_extra={ - "description": "Implements 'upperMultiplicity'", - }, - ), - ] = None - minItems: Annotated[ - Optional[int], - Field( - ge=0, - json_schema_extra={ - "description": "Implements 'lowerMultiplicity'", - }, - ), - ] = 0 - observedProperty: Optional[str] = None - observedPropertyURI: Optional[AnyUrl] = None - uom: Optional[str] = None - uomURI: Optional[AnyUrl] = None - - -class PropertiesSchema(BaseModel): - """PropertiesSchema model. - - Ref: https://github.com/opengeospatial/ogcapi-tiles/blob/master/openapi/schemas/tms/propertiesSchema.yaml - - Code generated using https://github.com/koxudaxi/datamodel-code-generator/ - """ - - type: Literal["object"] - required: Annotated[ - Optional[List[str]], - Field( - min_length=1, - json_schema_extra={ - "description": "Implements 'multiplicity' by citing property 'name' defined as 'additionalProperties'", - }, - ), - ] = None - properties: Dict[str, Properties] - - -class Style(BaseModel): - """Style model. - - Ref: https://github.com/opengeospatial/ogcapi-tiles/blob/master/openapi/schemas/tms/style.yaml - - Code generated using https://github.com/koxudaxi/datamodel-code-generator/ - """ - - id: Annotated[ - str, - Field( - json_schema_extra={ - "description": "An identifier for this style. Implementation of 'identifier'", - } - ), - ] - title: Annotated[ - Optional[str], - Field( - json_schema_extra={ - "description": "A title for this style", - } - ), - ] = None - description: Annotated[ - Optional[str], - Field( - json_schema_extra={ - "description": "Brief narrative description of this style", - } - ), - ] = None - keywords: Annotated[ - Optional[List[str]], - Field( - json_schema_extra={ - "description": "keywords about this style", - } - ), - ] = None - links: Annotated[ - Optional[List[Link]], - Field( - min_length=1, - json_schema_extra={ - "description": "Links to style related resources. Possible link 'rel' values are: 'style' for a URL pointing to the style description, 'styleSpec' for a URL pointing to the specification or standard used to define the style.", - }, - ), - ] = None - - -class GeospatialData(BaseModel): - """Geospatial model. - - Ref: https://github.com/opengeospatial/ogcapi-tiles/blob/master/openapi/schemas/tms/geospatialData.yaml - - Code generated using https://github.com/koxudaxi/datamodel-code-generator/ - """ - - title: Annotated[ - Optional[str], - Field( - json_schema_extra={ - "description": "Title of this tile matrix set, normally used for display to a human", - } - ), - ] = None - description: Annotated[ - Optional[str], - Field( - json_schema_extra={ - "description": "Brief narrative description of this tile matrix set, normally available for display to a human", - } - ), - ] = None - keywords: Annotated[ - Optional[str], - Field( - json_schema_extra={ - "description": "Unordered list of one or more commonly used or formalized word(s) or phrase(s) used to describe this layer", - } - ), - ] = None - id: Annotated[ - str, - Field( - json_schema_extra={ - "description": "Unique identifier of the Layer. Implementation of 'identifier'", - } - ), - ] - dataType: Annotated[ - Literal["map", "vector", "coverage"], - Field( - json_schema_extra={ - "description": "Type of data represented in the tileset", - } - ), - ] - geometryDimension: Annotated[ - Optional[int], - Field( # type: ignore - ge=0, - le=3, - json_schema_extra={ - "description": "The geometry dimension of the features shown in this layer (0: points, 1: curves, 2: surfaces, 3: solids), unspecified: mixed or unknown", - }, - ), - ] = None - featureType: Annotated[ - Optional[str], - Field( - json_schema_extra={ - "description": "Feature type identifier. Only applicable to layers of datatype 'geometries'", - } - ), - ] = None - attribution: Annotated[ - Optional[str], - Field( - json_schema_extra={ - "description": "Short reference to recognize the author or provider", - } - ), - ] = None - license: Annotated[ - Optional[str], - Field( - json_schema_extra={ - "description": "License applicable to the tiles", - } - ), - ] = None - pointOfContact: Annotated[ - Optional[str], - Field( - json_schema_extra={ - "description": "Useful information to contact the authors or custodians for the layer (e.g. e-mail address, a physical address, phone numbers, etc)", - } - ), - ] = None - publisher: Annotated[ - Optional[str], - Field( - json_schema_extra={ - "description": "Organization or individual responsible for making the layer available", - } - ), - ] = None - theme: Annotated[ - Optional[str], - Field( - json_schema_extra={ - "description": "Category where the layer can be grouped", - } - ), - ] = None - crs: Annotated[Optional[CRSType], Field(json_schema_extra={"title": "CRS"})] = None - epoch: Annotated[ - Optional[float], - Field( - json_schema_extra={ - "description": "Epoch of the Coordinate Reference System (CRS)", - } - ), - ] = None - minScaleDenominator: Annotated[ - Optional[float], - Field( - json_schema_extra={ - "description": "Minimum scale denominator for usage of the layer", - } - ), - ] = None - maxScaleDenominator: Annotated[ - Optional[float], - Field( - json_schema_extra={ - "description": "Maximum scale denominator for usage of the layer", - } - ), - ] = None - minCellSize: Annotated[ - Optional[float], - Field( - json_schema_extra={ - "description": "Minimum cell size for usage of the layer", - } - ), - ] = None - maxCellSize: Annotated[ - Optional[float], - Field( - json_schema_extra={ - "description": "Maximum cell size for usage of the layer", - } - ), - ] = None - maxTileMatrix: Annotated[ - Optional[str], - Field( - json_schema_extra={ - "description": "TileMatrix identifier associated with the minScaleDenominator", - } - ), - ] = None - minTileMatrix: Annotated[ - Optional[str], - Field( - json_schema_extra={ - "description": "TileMatrix identifier associated with the maxScaleDenominator", - } - ), - ] = None - boundingBox: Optional[BoundingBox] = None - created: Optional[TimeStamp] = None - updated: Optional[TimeStamp] = None - style: Optional[Style] = None - geoDataClasses: Annotated[ - Optional[List[str]], - Field( - json_schema_extra={ - "description": "URI identifying a class of data contained in this layer (useful to determine compatibility with styles or processes)", - } - ), - ] = None - propertiesSchema: Optional[PropertiesSchema] = None - links: Annotated[ - Optional[List[Link]], - Field( - min_length=1, - json_schema_extra={ - "description": "Links related to this layer. Possible link 'rel' values are: 'geodata' for a URL pointing to the collection of geospatial data.", - }, - ), - ] = None - - -class TilePoint(BaseModel): - """TilePoint model. - - Ref: https://github.com/opengeospatial/ogcapi-tiles/blob/master/openapi/schemas/tms/tilePoint.yaml - - Code generated using https://github.com/koxudaxi/datamodel-code-generator/ - """ - - coordinates: Annotated[List[float], Field(max_length=2, min_length=2)] - crs: Annotated[Optional[CRSType], Field(json_schema_extra={"title": "CRS"})] - tileMatrix: Annotated[ - Optional[str], - Field( - json_schema_extra={ - "description": "TileMatrix identifier associated with the scaleDenominator", - } - ), - ] = None - scaleDenominator: Annotated[ - Optional[float], - Field( - json_schema_extra={ - "description": "Scale denominator of the tile matrix selected", - } - ), - ] = None - cellSize: Annotated[ - Optional[float], - Field( - json_schema_extra={ - "description": "Cell size of the tile matrix selected", - } - ), - ] = None - - -class TileMatrixLimits(BaseModel): - """ - The limits for an individual tile matrix of a TileSet's TileMatrixSet, as defined in the OGC 2D TileMatrixSet and TileSet Metadata Standard - - Based on https://github.com/opengeospatial/ogcapi-tiles/blob/master/openapi/schemas/tms/tileMatrixLimits.yaml - """ - - tileMatrix: str - minTileRow: Annotated[int, Field(ge=0)] - maxTileRow: Annotated[int, Field(ge=0)] - minTileCol: Annotated[int, Field(ge=0)] - maxTileCol: Annotated[int, Field(ge=0)] - - -class TileSet(BaseModel): - """ - TileSet model. - - Based on https://github.com/opengeospatial/ogcapi-tiles/blob/master/openapi/schemas/tms/tileSet.yaml - """ - - title: Annotated[ - Optional[str], - Field( - json_schema_extra={ - "description": "A title for this tileset", - } - ), - ] = None - description: Annotated[ - Optional[str], - Field( - json_schema_extra={ - "description": "Brief narrative description of this tile set", - } - ), - ] = None - dataType: Annotated[ - Literal["map", "vector", "coverage"], - Field( - json_schema_extra={ - "description": "Type of data represented in the tileset", - } - ), - ] - crs: Annotated[CRSType, Field(json_schema_extra={"title": "CRS"})] - tileMatrixSetURI: Annotated[ - Optional[AnyUrl], - Field( - json_schema_extra={ - "description": "Reference to a Tile Matrix Set on an official source for Tile Matrix Sets", - } - ), - ] = None - links: Annotated[ - List[Link], - Field( - json_schema_extra={ - "description": "Links to related resources", - } - ), - ] - tileMatrixSetLimits: Annotated[ - Optional[List[TileMatrixLimits]], - Field( - json_schema_extra={ - "description": "Limits for the TileRow and TileCol values for each TileMatrix in the tileMatrixSet. If missing, there are no limits other that the ones imposed by the TileMatrixSet. If present the TileMatrices listed are limited and the rest not available at all", - } - ), - ] = None - epoch: Annotated[ - Optional[Union[float, int]], - Field( - json_schema_extra={ - "description": "Epoch of the Coordinate Reference System (CRS)", - } - ), - ] = None - layers: Annotated[ - Optional[List[GeospatialData]], - Field(min_length=1), - ] = None - boundingBox: Optional[BoundingBox] = None - centerPoint: Optional[TilePoint] = None - style: Optional[Style] = None - attribution: Annotated[ - Optional[str], - Field( - json_schema_extra={ - "description": "Short reference to recognize the author or provider", - } - ), - ] = None - license: Annotated[ - Optional[str], - Field( - json_schema_extra={ - "description": "License applicable to the tiles", - } - ), - ] = None - accessConstraints: Annotated[ - Optional[AccessConstraints], - Field( - json_schema_extra={ - "description": "Restrictions on the availability of the Tile Set that the user needs to be aware of before using or redistributing the Tile Set", - } - ), - ] = "unclassified" - keywords: Annotated[ - Optional[List[str]], - Field( - json_schema_extra={ - "description": "keywords about this tileset", - } - ), - ] = None - version: Annotated[ - Optional[str], - Field( - json_schema_extra={ - "description": "Version of the Tile Set. Changes if the data behind the tiles has been changed", - } - ), - ] = None - created: Optional[TimeStamp] = None - updated: Optional[TimeStamp] = None - pointOfContact: Annotated[ - Optional[str], - Field( - json_schema_extra={ - "description": "Useful information to contact the authors or custodians for the Tile Set", - } - ), - ] = None - mediaTypes: Annotated[ - Optional[List[str]], - Field( - json_schema_extra={ - "description": "Media types available for the tiles", - } - ), - ] = None - - -class TileSetList(BaseModel): - """ - TileSetList model. - - Based on https://docs.ogc.org/is/20-057/20-057.html#toc34 - """ - - tilesets: List[TileSet] +from tipg.models import * # noqa: F401,F403 +from tipg.models import __all__ as __all__ # noqa: F401 diff --git a/tipg/models/__init__.py b/tipg/models/__init__.py new file mode 100644 index 00000000..1a8e685f --- /dev/null +++ b/tipg/models/__init__.py @@ -0,0 +1,79 @@ +"""tipg.models: OGC API response/serialization models. + +Models are organized into submodules: + +* ``common`` — models shared by the features and tiles APIs +* ``features`` — OGC API - Features / Common response models +* ``tiles`` — OGC API - Tiles response models + +All public names are re-exported here, so ``from tipg.models import Item`` works +regardless of which submodule defines the model. +""" + +from tipg.models.common import Link +from tipg.models.features import ( + Collection, + Collections, + Conformance, + Extent, + Item, + Items, + Landing, + Queryables, + Spatial, + Temporal, +) +from tipg.models.tiles import ( + AccessConstraints, + BoundingBox, + GeospatialData, + LayerJSON, + Properties, + PropertiesSchema, + Style, + StyleJSON, + TileJSON, + TileMatrixLimits, + TileMatrixSetLink, + TileMatrixSetList, + TileMatrixSetRef, + TilePoint, + TileSet, + TileSetList, + TimeStamp, + Type, +) +from tipg.resources.enums import MediaType + +__all__ = [ + "AccessConstraints", + "BoundingBox", + "Collection", + "Collections", + "Conformance", + "Extent", + "GeospatialData", + "Item", + "Items", + "Landing", + "LayerJSON", + "Link", + "MediaType", + "Properties", + "PropertiesSchema", + "Queryables", + "Spatial", + "Style", + "StyleJSON", + "Temporal", + "TileJSON", + "TileMatrixLimits", + "TileMatrixSetLink", + "TileMatrixSetList", + "TileMatrixSetRef", + "TilePoint", + "TileSet", + "TileSetList", + "TimeStamp", + "Type", +] diff --git a/tipg/models/common.py b/tipg/models/common.py new file mode 100644 index 00000000..79126255 --- /dev/null +++ b/tipg/models/common.py @@ -0,0 +1,82 @@ +"""tipg.models.common: models shared by the features and tiles APIs.""" + +from typing import Annotated, Optional + +from pydantic import BaseModel, Field + +from tipg.resources.enums import MediaType + + +class Link(BaseModel): + """Link model. + + Ref: https://github.com/opengeospatial/ogcapi-tiles/blob/master/openapi/schemas/common-core/link.yaml + + Code generated using https://github.com/koxudaxi/datamodel-code-generator/ + """ + + href: Annotated[ + str, + Field( + json_schema_extra={ + "description": "Supplies the URI to a remote resource (or resource fragment).", + "examples": ["http://data.example.com/buildings/123"], + } + ), + ] + rel: Annotated[ + str, + Field( + json_schema_extra={ + "description": "The type or semantics of the relation.", + "examples": ["alternate"], + } + ), + ] + type: Annotated[ + Optional[MediaType], + Field( + json_schema_extra={ + "description": "A hint indicating what the media type of the result of dereferencing the link should be.", + "examples": ["application/geo+json"], + } + ), + ] = None + templated: Annotated[ + Optional[bool], + Field( + json_schema_extra={ + "description": "This flag set to true if the link is a URL template.", + } + ), + ] = None + varBase: Annotated[ + Optional[str], + Field( + json_schema_extra={ + "description": "A base path to retrieve semantic information about the variables used in URL template.", + "examples": ["/ogcapi/vars/"], + } + ), + ] = None + hreflang: Annotated[ + Optional[str], + Field( + json_schema_extra={ + "description": "A hint indicating what the language of the result of dereferencing the link should be.", + "examples": ["en"], + } + ), + ] = None + title: Annotated[ + Optional[str], + Field( + json_schema_extra={ + "description": "Used to label the destination of a link such that it can be used as a human-readable identifier.", + "examples": ["Trierer Strasse 70, 53115 Bonn"], + } + ), + ] = None + length: Optional[int] = None + + model_config = {"use_enum_values": True} diff --git a/tipg/models/features.py b/tipg/models/features.py new file mode 100644 index 00000000..0b6d50bd --- /dev/null +++ b/tipg/models/features.py @@ -0,0 +1,166 @@ +"""tipg.models.features: OGC API - Features / Common response models.""" + +from typing import Annotated, Dict, List, Optional + +from geojson_pydantic.features import Feature, FeatureCollection +from pydantic import BaseModel, Field + +from tipg.models.common import Link + + +class Spatial(BaseModel): + """Spatial Extent model. + + Ref: http://schemas.opengis.net/ogcapi/features/part1/1.0/openapi/schemas/extent.yaml + + """ + + # Bbox + # One or more bounding boxes that describe the spatial extent of the dataset. + # The first bounding box describes the overall spatial + # extent of the data. All subsequent bounding boxes describe + # more precise bounding boxes, e.g., to identify clusters of data. + bbox: List[List[float]] + crs: str = "http://www.opengis.net/def/crs/OGC/1.3/CRS84" + + +class Temporal(BaseModel): + """Temporal Extent model. + + Ref: http://schemas.opengis.net/ogcapi/features/part1/1.0/openapi/schemas/extent.yaml + + """ + + # The first time interval describes the overall + # temporal extent of the data. All subsequent time intervals describe + # more precise time intervals, e.g., to identify clusters of data. + # Clients only interested in the overall temporal extent will only need + # to access the first time interval in the array (a pair of lower and upper + # bound instants). + interval: List[List[Optional[str]]] + trs: str = "http://www.opengis.net/def/uom/ISO-8601/0/Gregorian" + + +class Extent(BaseModel): + """Extent model. + + Ref: http://schemas.opengis.net/ogcapi/features/part1/1.0/openapi/schemas/extent.yaml + + """ + + spatial: Optional[Spatial] = None + temporal: Optional[Temporal] = None + + +class Collection(BaseModel): + """Collection model. + + Note: `CRS` is the list of CRS supported by the service not the CRS of the collection + + Ref: http://schemas.opengis.net/ogcapi/features/part1/1.0/openapi/schemas/collection.yaml + + """ + + id: str + title: Optional[str] = None + description: Optional[str] = None + links: List[Link] + extent: Optional[Extent] = None + itemType: str = "feature" + crs: List[str] = ["http://www.opengis.net/def/crs/OGC/1.3/CRS84"] + + model_config = {"extra": "ignore"} + + +class Collections(BaseModel): + """ + Collections model. + + Ref: http://beta.schemas.opengis.net/ogcapi/common/part2/0.1/collections/openapi/schemas/collections.yaml + + """ + + links: List[Link] + timeStamp: Optional[str] = None + numberMatched: Optional[int] = None + numberReturned: Optional[int] = None + collections: List[Collection] + + model_config = {"extra": "allow"} + + +class Item(Feature): + """Item model + + Ref: http://schemas.opengis.net/ogcapi/features/part1/1.0/openapi/schemas/featureGeoJSON.yaml + + """ + + links: Optional[List[Link]] = None + + model_config = {"arbitrary_types_allowed": True} + + +class Items(FeatureCollection): + """Items model + + Ref: http://schemas.opengis.net/ogcapi/features/part1/1.0/openapi/schemas/featureCollectionGeoJSON.yaml + + """ + + id: str + title: Optional[str] = None + description: Optional[str] = None + keywords: Optional[List[str]] = None + features: List[Item] + links: Optional[List[Link]] = None + timeStamp: Optional[str] = None + numberMatched: Optional[int] = None + numberReturned: Optional[int] = None + + model_config = {"arbitrary_types_allowed": True} + + def json_seq(self, **kwargs): + """return a GeoJSON sequence representation.""" + for f in self.features: + yield f.json(**kwargs) + "\n" + + +class Conformance(BaseModel): + """Conformance model. + + Ref: http://schemas.opengis.net/ogcapi/features/part1/1.0/openapi/schemas/confClasses.yaml + + """ + + conformsTo: List[str] + + +class Landing(BaseModel): + """Landing page model. + + Ref: http://schemas.opengis.net/ogcapi/features/part1/1.0/openapi/schemas/landingPage.yaml + + """ + + title: Optional[str] = None + description: Optional[str] = None + links: List[Link] + + +class Queryables(BaseModel): + """Queryables model. + + Ref: https://docs.ogc.org/DRAFTS/19-079r1.html#filter-queryables + + """ + + title: str + properties: Dict[str, Dict[str, str]] + type: str = "object" + schema_name: Annotated[str, Field(alias="$schema")] = ( + "https://json-schema.org/draft/2019-09/schema" + ) + link: Annotated[str, Field(alias="$id")] + + model_config = {"populate_by_name": True} diff --git a/tipg/models/tiles.py b/tipg/models/tiles.py new file mode 100644 index 00000000..870b21d8 --- /dev/null +++ b/tipg/models/tiles.py @@ -0,0 +1,722 @@ +"""tipg.models.tiles: OGC API - Tiles response models.""" + +from datetime import datetime +from typing import Annotated, Dict, List, Literal, Optional, Set, Tuple, Union + +from morecantile.models import CRSType +from pydantic import AnyUrl, BaseModel, Field, RootModel, model_validator + +from tipg.models.common import Link +from tipg.resources.enums import MediaType + + +class TileMatrixSetLink(BaseModel): + """ + TileMatrixSetLink model. + Based on http://docs.opengeospatial.org/per/19-069.html#_tilematrixsets + """ + + href: str + rel: str = "http://www.opengis.net/def/rel/ogc/1.0/tiling-scheme" + type: MediaType = MediaType.json + + model_config = {"use_enum_values": True} + + +class TileMatrixSetRef(BaseModel): + """ + TileMatrixSetRef model. + Based on http://docs.opengeospatial.org/per/19-069.html#_tilematrixsets + """ + + id: str + title: Optional[str] = None + links: List[TileMatrixSetLink] + + +class TileMatrixSetList(BaseModel): + """ + TileMatrixSetList model. + Based on http://docs.opengeospatial.org/per/19-069.html#_tilematrixsets + """ + + tileMatrixSets: List[TileMatrixSetRef] + + +class LayerJSON(BaseModel): + """ + https://github.com/mapbox/tilejson-spec/tree/master/3.0.0#33-vector_layers + """ + + id: str + fields: Annotated[Dict, Field(default_factory=dict)] + description: Optional[str] = None + minzoom: Optional[int] = None + maxzoom: Optional[int] = None + + +class TileJSON(BaseModel): + """ + TileJSON model. + + Based on https://github.com/mapbox/tilejson-spec/tree/master/3.0.0 + """ + + tilejson: str = "3.0.0" + name: Optional[str] = None + description: Optional[str] = None + version: str = "1.0.0" + attribution: Optional[str] = None + template: Optional[str] = None + legend: Optional[str] = None + scheme: Literal["xyz", "tms"] = "xyz" + tiles: List[str] + vector_layers: Optional[List[LayerJSON]] = None + grids: Optional[List[str]] = None + data: Optional[List[str]] = None + minzoom: int = Field(0) + maxzoom: int = Field(30) + fillzoom: Optional[int] = None + bounds: List[float] = [180, -85.05112877980659, 180, 85.0511287798066] + center: Optional[Tuple[float, float, int]] = None + + @model_validator(mode="after") + def compute_center(self): + """Compute center if it does not exist.""" + bounds = self.bounds + if not self.center: + self.center = ( + (bounds[0] + bounds[2]) / 2, + (bounds[1] + bounds[3]) / 2, + self.minzoom, + ) + return self + + +class StyleJSON(BaseModel): + """ + Simple Mapbox/Maplibre Style JSON model. + + Based on https://docs.mapbox.com/help/glossary/style/ + + """ + + version: int = 8 + name: Optional[str] = None + metadata: Optional[Dict] = None + layers: List[Dict] + sources: Dict + center: List[float] = [0, 0] + zoom: int = 1 + + +class TimeStamp(RootModel): + """TimeStamp model. + + Ref: https://github.com/opengeospatial/ogcapi-tiles/blob/master/openapi/schemas/common-geodata/timeStamp.yaml + + Code generated using https://github.com/koxudaxi/datamodel-code-generator/ + """ + + root: Annotated[ + datetime, + Field( + json_schema_extra={ + "description": "This property indicates the time and date when the response was generated using RFC 3339 notation.", + "examples": ["2017-08-17T08:05:32Z"], + } + ), + ] + + +class BoundingBox(BaseModel): + """BoundingBox model. + + Ref: https://github.com/opengeospatial/ogcapi-tiles/blob/master/openapi/schemas/tms/2DBoundingBox.yaml + + Code generated using https://github.com/koxudaxi/datamodel-code-generator/ + """ + + lowerLeft: Annotated[ + List[float], + Field( + max_length=2, + min_length=2, + json_schema_extra={ + "description": "A 2D Point in the CRS indicated elsewhere", + }, + ), + ] + upperRight: Annotated[ + List[float], + Field( + max_length=2, + min_length=2, + json_schema_extra={ + "description": "A 2D Point in the CRS indicated elsewhere", + }, + ), + ] + crs: Annotated[Optional[CRSType], Field(json_schema_extra={"title": "CRS"})] = None + orderedAxes: Annotated[Optional[List[str]], Field(max_length=2, min_length=2)] = ( + None + ) + + +# Ref: https://github.com/opengeospatial/ogcapi-tiles/blob/master/openapi/schemas/tms/propertiesSchema.yaml +Type = Literal["array", "boolean", "integer", "null", "number", "object", "string"] + +# Ref: https://github.com/opengeospatial/ogcapi-tiles/blob/master/openapi/schemas/tms/propertiesSchema.yaml +AccessConstraints = Literal[ + "unclassified", "restricted", "confidential", "secret", "topSecret" +] + + +class Properties(BaseModel): + """Properties model. + + Ref: https://github.com/opengeospatial/ogcapi-tiles/blob/master/openapi/schemas/tms/propertiesSchema.yaml + + Code generated using https://github.com/koxudaxi/datamodel-code-generator/ + """ + + title: Optional[str] = None + description: Annotated[ + Optional[str], + Field( + json_schema_extra={ + "description": "Implements 'description'", + } + ), + ] = None + type: Optional[Type] = None + enum: Annotated[ + Optional[Set], + Field( + min_length=1, + json_schema_extra={ + "description": "Implements 'acceptedValues'", + }, + ), + ] = None + format: Annotated[ + Optional[str], + Field( + json_schema_extra={ + "description": "Complements implementation of 'type'", + } + ), + ] = None + contentMediaType: Annotated[ + Optional[str], + Field( + json_schema_extra={ + "description": "Implements 'mediaType'", + } + ), + ] = None + maximum: Annotated[ + Optional[float], + Field( + json_schema_extra={ + "description": "Implements 'range'", + } + ), + ] = None + exclusiveMaximum: Annotated[ + Optional[float], + Field( + json_schema_extra={ + "description": "Implements 'range'", + } + ), + ] = None + minimum: Annotated[ + Optional[float], + Field( + json_schema_extra={ + "description": "Implements 'range'", + } + ), + ] = None + exclusiveMinimum: Annotated[ + Optional[float], + Field( + json_schema_extra={ + "description": "Implements 'range'", + } + ), + ] = None + pattern: Optional[str] = None + maxItems: Annotated[ + Optional[int], + Field( + ge=0, + json_schema_extra={ + "description": "Implements 'upperMultiplicity'", + }, + ), + ] = None + minItems: Annotated[ + Optional[int], + Field( + ge=0, + json_schema_extra={ + "description": "Implements 'lowerMultiplicity'", + }, + ), + ] = 0 + observedProperty: Optional[str] = None + observedPropertyURI: Optional[AnyUrl] = None + uom: Optional[str] = None + uomURI: Optional[AnyUrl] = None + + +class PropertiesSchema(BaseModel): + """PropertiesSchema model. + + Ref: https://github.com/opengeospatial/ogcapi-tiles/blob/master/openapi/schemas/tms/propertiesSchema.yaml + + Code generated using https://github.com/koxudaxi/datamodel-code-generator/ + """ + + type: Literal["object"] + required: Annotated[ + Optional[List[str]], + Field( + min_length=1, + json_schema_extra={ + "description": "Implements 'multiplicity' by citing property 'name' defined as 'additionalProperties'", + }, + ), + ] = None + properties: Dict[str, Properties] + + +class Style(BaseModel): + """Style model. + + Ref: https://github.com/opengeospatial/ogcapi-tiles/blob/master/openapi/schemas/tms/style.yaml + + Code generated using https://github.com/koxudaxi/datamodel-code-generator/ + """ + + id: Annotated[ + str, + Field( + json_schema_extra={ + "description": "An identifier for this style. Implementation of 'identifier'", + } + ), + ] + title: Annotated[ + Optional[str], + Field( + json_schema_extra={ + "description": "A title for this style", + } + ), + ] = None + description: Annotated[ + Optional[str], + Field( + json_schema_extra={ + "description": "Brief narrative description of this style", + } + ), + ] = None + keywords: Annotated[ + Optional[List[str]], + Field( + json_schema_extra={ + "description": "keywords about this style", + } + ), + ] = None + links: Annotated[ + Optional[List[Link]], + Field( + min_length=1, + json_schema_extra={ + "description": "Links to style related resources. Possible link 'rel' values are: 'style' for a URL pointing to the style description, 'styleSpec' for a URL pointing to the specification or standard used to define the style.", + }, + ), + ] = None + + +class GeospatialData(BaseModel): + """Geospatial model. + + Ref: https://github.com/opengeospatial/ogcapi-tiles/blob/master/openapi/schemas/tms/geospatialData.yaml + + Code generated using https://github.com/koxudaxi/datamodel-code-generator/ + """ + + title: Annotated[ + Optional[str], + Field( + json_schema_extra={ + "description": "Title of this tile matrix set, normally used for display to a human", + } + ), + ] = None + description: Annotated[ + Optional[str], + Field( + json_schema_extra={ + "description": "Brief narrative description of this tile matrix set, normally available for display to a human", + } + ), + ] = None + keywords: Annotated[ + Optional[str], + Field( + json_schema_extra={ + "description": "Unordered list of one or more commonly used or formalized word(s) or phrase(s) used to describe this layer", + } + ), + ] = None + id: Annotated[ + str, + Field( + json_schema_extra={ + "description": "Unique identifier of the Layer. Implementation of 'identifier'", + } + ), + ] + dataType: Annotated[ + Literal["map", "vector", "coverage"], + Field( + json_schema_extra={ + "description": "Type of data represented in the tileset", + } + ), + ] + geometryDimension: Annotated[ + Optional[int], + Field( # type: ignore + ge=0, + le=3, + json_schema_extra={ + "description": "The geometry dimension of the features shown in this layer (0: points, 1: curves, 2: surfaces, 3: solids), unspecified: mixed or unknown", + }, + ), + ] = None + featureType: Annotated[ + Optional[str], + Field( + json_schema_extra={ + "description": "Feature type identifier. Only applicable to layers of datatype 'geometries'", + } + ), + ] = None + attribution: Annotated[ + Optional[str], + Field( + json_schema_extra={ + "description": "Short reference to recognize the author or provider", + } + ), + ] = None + license: Annotated[ + Optional[str], + Field( + json_schema_extra={ + "description": "License applicable to the tiles", + } + ), + ] = None + pointOfContact: Annotated[ + Optional[str], + Field( + json_schema_extra={ + "description": "Useful information to contact the authors or custodians for the layer (e.g. e-mail address, a physical address, phone numbers, etc)", + } + ), + ] = None + publisher: Annotated[ + Optional[str], + Field( + json_schema_extra={ + "description": "Organization or individual responsible for making the layer available", + } + ), + ] = None + theme: Annotated[ + Optional[str], + Field( + json_schema_extra={ + "description": "Category where the layer can be grouped", + } + ), + ] = None + crs: Annotated[Optional[CRSType], Field(json_schema_extra={"title": "CRS"})] = None + epoch: Annotated[ + Optional[float], + Field( + json_schema_extra={ + "description": "Epoch of the Coordinate Reference System (CRS)", + } + ), + ] = None + minScaleDenominator: Annotated[ + Optional[float], + Field( + json_schema_extra={ + "description": "Minimum scale denominator for usage of the layer", + } + ), + ] = None + maxScaleDenominator: Annotated[ + Optional[float], + Field( + json_schema_extra={ + "description": "Maximum scale denominator for usage of the layer", + } + ), + ] = None + minCellSize: Annotated[ + Optional[float], + Field( + json_schema_extra={ + "description": "Minimum cell size for usage of the layer", + } + ), + ] = None + maxCellSize: Annotated[ + Optional[float], + Field( + json_schema_extra={ + "description": "Maximum cell size for usage of the layer", + } + ), + ] = None + maxTileMatrix: Annotated[ + Optional[str], + Field( + json_schema_extra={ + "description": "TileMatrix identifier associated with the minScaleDenominator", + } + ), + ] = None + minTileMatrix: Annotated[ + Optional[str], + Field( + json_schema_extra={ + "description": "TileMatrix identifier associated with the maxScaleDenominator", + } + ), + ] = None + boundingBox: Optional[BoundingBox] = None + created: Optional[TimeStamp] = None + updated: Optional[TimeStamp] = None + style: Optional[Style] = None + geoDataClasses: Annotated[ + Optional[List[str]], + Field( + json_schema_extra={ + "description": "URI identifying a class of data contained in this layer (useful to determine compatibility with styles or processes)", + } + ), + ] = None + propertiesSchema: Optional[PropertiesSchema] = None + links: Annotated[ + Optional[List[Link]], + Field( + min_length=1, + json_schema_extra={ + "description": "Links related to this layer. Possible link 'rel' values are: 'geodata' for a URL pointing to the collection of geospatial data.", + }, + ), + ] = None + + +class TilePoint(BaseModel): + """TilePoint model. + + Ref: https://github.com/opengeospatial/ogcapi-tiles/blob/master/openapi/schemas/tms/tilePoint.yaml + + Code generated using https://github.com/koxudaxi/datamodel-code-generator/ + """ + + coordinates: Annotated[List[float], Field(max_length=2, min_length=2)] + crs: Annotated[Optional[CRSType], Field(json_schema_extra={"title": "CRS"})] + tileMatrix: Annotated[ + Optional[str], + Field( + json_schema_extra={ + "description": "TileMatrix identifier associated with the scaleDenominator", + } + ), + ] = None + scaleDenominator: Annotated[ + Optional[float], + Field( + json_schema_extra={ + "description": "Scale denominator of the tile matrix selected", + } + ), + ] = None + cellSize: Annotated[ + Optional[float], + Field( + json_schema_extra={ + "description": "Cell size of the tile matrix selected", + } + ), + ] = None + + +class TileMatrixLimits(BaseModel): + """ + The limits for an individual tile matrix of a TileSet's TileMatrixSet, as defined in the OGC 2D TileMatrixSet and TileSet Metadata Standard + + Based on https://github.com/opengeospatial/ogcapi-tiles/blob/master/openapi/schemas/tms/tileMatrixLimits.yaml + """ + + tileMatrix: str + minTileRow: Annotated[int, Field(ge=0)] + maxTileRow: Annotated[int, Field(ge=0)] + minTileCol: Annotated[int, Field(ge=0)] + maxTileCol: Annotated[int, Field(ge=0)] + + +class TileSet(BaseModel): + """ + TileSet model. + + Based on https://github.com/opengeospatial/ogcapi-tiles/blob/master/openapi/schemas/tms/tileSet.yaml + """ + + title: Annotated[ + Optional[str], + Field( + json_schema_extra={ + "description": "A title for this tileset", + } + ), + ] = None + description: Annotated[ + Optional[str], + Field( + json_schema_extra={ + "description": "Brief narrative description of this tile set", + } + ), + ] = None + dataType: Annotated[ + Literal["map", "vector", "coverage"], + Field( + json_schema_extra={ + "description": "Type of data represented in the tileset", + } + ), + ] + crs: Annotated[CRSType, Field(json_schema_extra={"title": "CRS"})] + tileMatrixSetURI: Annotated[ + Optional[AnyUrl], + Field( + json_schema_extra={ + "description": "Reference to a Tile Matrix Set on an official source for Tile Matrix Sets", + } + ), + ] = None + links: Annotated[ + List[Link], + Field( + json_schema_extra={ + "description": "Links to related resources", + } + ), + ] + tileMatrixSetLimits: Annotated[ + Optional[List[TileMatrixLimits]], + Field( + json_schema_extra={ + "description": "Limits for the TileRow and TileCol values for each TileMatrix in the tileMatrixSet. If missing, there are no limits other that the ones imposed by the TileMatrixSet. If present the TileMatrices listed are limited and the rest not available at all", + } + ), + ] = None + epoch: Annotated[ + Optional[Union[float, int]], + Field( + json_schema_extra={ + "description": "Epoch of the Coordinate Reference System (CRS)", + } + ), + ] = None + layers: Annotated[ + Optional[List[GeospatialData]], + Field(min_length=1), + ] = None + boundingBox: Optional[BoundingBox] = None + centerPoint: Optional[TilePoint] = None + style: Optional[Style] = None + attribution: Annotated[ + Optional[str], + Field( + json_schema_extra={ + "description": "Short reference to recognize the author or provider", + } + ), + ] = None + license: Annotated[ + Optional[str], + Field( + json_schema_extra={ + "description": "License applicable to the tiles", + } + ), + ] = None + accessConstraints: Annotated[ + Optional[AccessConstraints], + Field( + json_schema_extra={ + "description": "Restrictions on the availability of the Tile Set that the user needs to be aware of before using or redistributing the Tile Set", + } + ), + ] = "unclassified" + keywords: Annotated[ + Optional[List[str]], + Field( + json_schema_extra={ + "description": "keywords about this tileset", + } + ), + ] = None + version: Annotated[ + Optional[str], + Field( + json_schema_extra={ + "description": "Version of the Tile Set. Changes if the data behind the tiles has been changed", + } + ), + ] = None + created: Optional[TimeStamp] = None + updated: Optional[TimeStamp] = None + pointOfContact: Annotated[ + Optional[str], + Field( + json_schema_extra={ + "description": "Useful information to contact the authors or custodians for the Tile Set", + } + ), + ] = None + mediaTypes: Annotated[ + Optional[List[str]], + Field( + json_schema_extra={ + "description": "Media types available for the tiles", + } + ), + ] = None + + +class TileSetList(BaseModel): + """ + TileSetList model. + + Based on https://docs.ogc.org/is/20-057/20-057.html#toc34 + """ + + tilesets: List[TileSet] diff --git a/tipg/pgcollection.py b/tipg/pgcollection.py new file mode 100644 index 00000000..7009945c --- /dev/null +++ b/tipg/pgcollection.py @@ -0,0 +1,409 @@ +"""tipg.pgcollection: PgCollection implementation.""" + +import re +from typing import Any, Dict, List, Optional, Tuple + +import asyncpg +from cql2 import Expr +from morecantile import Tile, TileMatrixSet +from pydantic import Field + +from tipg.dbmodel import Collection, Column, Feature, ItemList +from tipg.errors import InvalidGeometryColumnName, InvalidLimit, InvalidPropertyName +from tipg.filter import cql_where +from tipg.settings import FeaturesSettings, MVTSettings +from tipg.sqlhelpers import _quote_ident, debug_query + +mvt_settings = MVTSettings() +features_settings = FeaturesSettings() + + +class PgCollection(Collection): + """Model for DB Table and Function.""" + + dbschema: str = Field(alias="schema") + + @property + def _qualified_name(self) -> str: + """SQL-quoted ``"schema"."table"`` identifier for this collection.""" + return f"{_quote_ident(self.dbschema)}.{_quote_ident(self.table)}" + + def _select_property_columns(self, properties: Optional[List[str]]) -> List[str]: + """Quoted column-name fragments for the property columns to SELECT.""" + return [_quote_ident(c) for c in self.columns(properties)] + + def _select_id(self) -> str: + """SQL fragment that selects the primary-key column as ``tipg_id``. + + Falls back to ``ROW_NUMBER()`` when the collection has no primary key. + """ + if self.id_column: + return f"{_quote_ident(self.id_column.name)} AS tipg_id" + return "ROW_NUMBER() OVER () AS tipg_id" + + def _geom_expr( + self, + geometry_column: Optional[Column], + bbox_only: Optional[bool], + simplify: Optional[float], + ) -> Optional[str]: + """Build the SQL expression for the geometry column (reprojected to + 4326 if needed, optionally bbox-only or simplified). + """ + if geometry_column is None: + return None + + g = f"CAST({_quote_ident(geometry_column.name)} AS geometry)" + + if geometry_column.srid != 4326: + g = f"ST_Transform({g}, 4326)" + + if bbox_only: + g = f"ST_Envelope({g})" + elif simplify: + s = float(simplify) + g = f"ST_SnapToGrid(ST_Simplify({g}, {s}), {s})" + + return g + + def _select( + self, + properties: Optional[List[str]], + geometry_column: Optional[Column], + bbox_only: Optional[bool], + simplify: Optional[float], + geom_as_wkt: bool = False, + ) -> str: + """Build the SELECT clause for the main features query.""" + cols = self._select_property_columns(properties) + cols.append(self._select_id()) + + geom = self._geom_expr(geometry_column, bbox_only, simplify) + if geom_as_wkt: + cols.append( + f"ST_AsEWKT({geom}) AS tipg_geom" + if geom + else "CAST(NULL AS text) AS tipg_geom" + ) + else: + cols.append( + f"CAST(ST_AsGeoJSON({geom}) AS json) AS tipg_geom" + if geom + else "CAST(NULL AS json) AS tipg_geom" + ) + + return "SELECT " + ", ".join(cols) + + def _select_mvt( + self, + properties: Optional[List[str]], + geometry_column: Column, + tms: TileMatrixSet, + tile: Tile, + ) -> str: + """Build the SELECT clause that emits an MVT geometry per row.""" + cols = self._select_property_columns(properties) + + geom = f"CAST({_quote_ident(geometry_column.name)} AS geometry)" + + # For tiles that fall outside the TMS's natural domain (e.g. an + # over-zoomed corner tile), clip the source geometry to the TMS's + # geographic bbox before reprojecting — otherwise ST_AsMVTGeom can + # produce garbage near the antimeridian / poles. ``tms.bbox`` is in + # the TMS's ``geographic_crs``, which is NOT always EPSG:4326 (e.g. + # CanadianNAD83_LCC uses 4269, EuropeanETRS89_LAEAQuad uses 4258, + # NZTM2000Quad uses 4167). When the geographic CRS has no EPSG code + # (WorldCRS84Quad → OGC:CRS84), 4326 is a safe fallback since the + # coordinate values are identical (only axis order differs, and + # PostGIS treats 4326 as lon/lat). + if not tms.is_valid(tile): + west, south, east, north = tms.bbox + geo_srid = tms.geographic_crs.to_epsg() or 4326 + geom = ( + f"ST_Intersection(" + f"ST_MakeEnvelope({west}, {south}, {east}, {north}, {geo_srid}), " + f"ST_Transform({geom}, {geo_srid})" + f")" + ) + + # Reproject to TMS CRS — prefer EPSG code, fall back to PROJ string. + if tms_srid := tms.crs.to_epsg(): + transformed = f"ST_Transform({geom}, {tms_srid})" + else: + tms_proj = tms.crs.to_proj4().replace("'", "''") + transformed = f"ST_Transform({geom}, '{tms_proj}')" + + bbox = tms.xy_bounds(tile) + envelope = ( + f"ST_Segmentize(" + f"ST_MakeEnvelope({bbox.left}, {bbox.bottom}, {bbox.right}, {bbox.top}), " + f"{bbox.right - bbox.left}" + f")" + ) + cols.append( + f"ST_AsMVTGeom(" + f"{transformed}, {envelope}, " + f"{int(mvt_settings.tile_resolution)}, " + f"{int(mvt_settings.tile_buffer)}, " + f"{'TRUE' if mvt_settings.tile_clip else 'FALSE'}" + f") AS geom" + ) + + return "SELECT " + ", ".join(cols) + + def _from( + self, + function_parameters: Optional[Dict[str, str]], + params: List[Any], + ) -> str: + """Build the FROM clause. Function-table parameters are appended to + ``params`` and referenced via ``$N`` placeholders. + """ + name = self._qualified_name + if self.type != "Function": + return f"FROM {name}" + + if not function_parameters: + return f"FROM {name}()" + + args = [] + for p in self.parameters: + if p.name in function_parameters: + params.append(function_parameters[p.name]) + # Double cast (value → text → target type): asyncpg sends + # Python strs as text, and the explicit cast to ``p.type`` + # then coerces to whatever the function signature expects. + args.append(f"CAST(CAST(${len(params)} AS text) AS {p.type})") + return f"FROM {name}({', '.join(args)})" + + def _sortby(self, sortby: Optional[str]) -> str: + """Build the ORDER BY clause.""" + sorts = [] + if sortby: + for s in sortby.strip().split(","): + parts = re.match("^(?P[+-]?)(?P.*)$", s).groupdict() # type:ignore + + direction = parts["direction"] + column = parts["column"].strip() + if not self.get_column(column): + raise InvalidPropertyName(f"Property {column} does not exist.") + col_sql = _quote_ident(column) + sorts.append(f"{col_sql} DESC" if direction == "-" else col_sql) + elif self.id_column is not None: + sorts.append(_quote_ident(self.id_column.name)) + else: + sorts.append(_quote_ident(self.properties[0].name)) + + return "ORDER BY " + ", ".join(sorts) + + @staticmethod + def _where_clause(where: Optional[Expr]) -> str: + """Render a cql2 ``Expr`` (or ``None``) as a WHERE clause.""" + if where is None: + return "WHERE TRUE" + return f"WHERE {where.to_sql()}" + + async def _features_query( + self, + conn: asyncpg.Connection, + *, + where: Optional[Expr] = None, + sortby: Optional[str] = None, + properties: Optional[List[str]] = None, + geom: Optional[str] = None, + limit: Optional[int] = None, + offset: Optional[int] = None, + bbox_only: Optional[bool] = None, + simplify: Optional[float] = None, + geom_as_wkt: bool = False, + function_parameters: Optional[Dict[str, str]], + ): + """Build and run the Features query, yielding each row as a Feature.""" + limit = limit or features_settings.default_features_limit + offset = offset or 0 + params: List[Any] = [] + + sql = " ".join( + [ + self._select( + properties=properties, + geometry_column=self.get_geometry_column(geom), + bbox_only=bbox_only, + simplify=simplify, + geom_as_wkt=geom_as_wkt, + ), + self._from(function_parameters, params), + self._where_clause(where), + self._sortby(sortby), + f"LIMIT {int(limit)}", + f"OFFSET {int(offset)}", + ] + ) + + for r in await conn.fetch(sql, *params): + props = dict(r) + g = props.pop("tipg_geom") + id = props.pop("tipg_id") + feature = Feature(type="Feature", geometry=g, id=id, properties=props) + yield feature + + async def _features_count_query( + self, + conn: asyncpg.Connection, + *, + where: Optional[Expr] = None, + function_parameters: Optional[Dict[str, str]], + ) -> int: + """Run the COUNT(*) query for the current filter.""" + params: List[Any] = [] + sql = " ".join( + [ + "SELECT COUNT(*)", + self._from(function_parameters, params), + self._where_clause(where), + ] + ) + return await conn.fetchval(sql, *params) + + async def features( + self, + conn: asyncpg.Connection, + *, + ids_filter: Optional[List[str]] = None, + bbox_filter: Optional[List[float]] = None, + datetime_filter: Optional[List[str]] = None, + properties_filter: Optional[List[Tuple[str, str]]] = None, + cql_filter: Optional[Expr] = None, + sortby: Optional[str] = None, + properties: Optional[List[str]] = None, + geom: Optional[str] = None, + dt: Optional[str] = None, + limit: Optional[int] = None, + offset: Optional[int] = None, + bbox_only: Optional[bool] = None, + simplify: Optional[float] = None, + geom_as_wkt: bool = False, + function_parameters: Optional[Dict[str, str]] = None, + ) -> ItemList: + """Build and run Pg query.""" + limit = limit or features_settings.default_features_limit + offset = offset or 0 + + function_parameters = function_parameters or {} + + if geom and geom.lower() != "none" and not self.get_geometry_column(geom): + raise InvalidGeometryColumnName(f"Invalid Geometry Column: {geom}.") + + if limit and limit > features_settings.max_features_per_query: + raise InvalidLimit( + f"Limit can not be set higher than the `tipg_max_features_per_query` setting of {features_settings.max_features_per_query}" + ) + + where_filter = cql_where( + self, + ids=ids_filter, + datetime=datetime_filter, + bbox=bbox_filter, + properties=properties_filter, + cql=cql_filter, + geom=geom, + dt=dt, + ) + + matched = await self._features_count_query( + conn, + where=where_filter, + function_parameters=function_parameters, + ) + + features = [ + f + async for f in self._features_query( + conn, + where=where_filter, + sortby=sortby, + properties=properties, + geom=geom, + limit=limit, + offset=offset, + bbox_only=bbox_only, + simplify=simplify, + geom_as_wkt=geom_as_wkt, + function_parameters=function_parameters, + ) + ] + returned = len(features) + + return ItemList( + items=features, + matched=matched, + next=offset + returned if matched - returned > offset else None, + prev=max(offset - limit, 0) if offset else None, + ) + + async def get_tile( + self, + conn: asyncpg.Connection, + *, + tms: TileMatrixSet, + tile: Tile, + ids_filter: Optional[List[str]] = None, + bbox_filter: Optional[List[float]] = None, + datetime_filter: Optional[List[str]] = None, + properties_filter: Optional[List[Tuple[str, str]]] = None, + function_parameters: Optional[Dict[str, str]] = None, + cql_filter: Optional[Expr] = None, + sortby: Optional[str] = None, + properties: Optional[List[str]] = None, + geom: Optional[str] = None, + dt: Optional[str] = None, + limit: Optional[int] = None, + ): + """Build query to get Vector Tile.""" + limit = limit or mvt_settings.max_features_per_tile + + geometry_column = self.get_geometry_column(geom) + if not geometry_column: + raise InvalidGeometryColumnName(f"Invalid Geometry Column Name {geom}") + + if limit > mvt_settings.max_features_per_tile: + raise InvalidLimit( + f"Limit can not be set higher than the `tipg_max_features_per_tile` setting of {mvt_settings.max_features_per_tile}" + ) + + where_filter = cql_where( + self, + ids=ids_filter, + datetime=datetime_filter, + bbox=bbox_filter, + properties=properties_filter, + cql=cql_filter, + geom=geom, + dt=dt, + tms=tms, + tile=tile, + ) + + params: List[Any] = [] + inner_sql = " ".join( + [ + self._select_mvt( + properties=properties, + geometry_column=geometry_column, + tms=tms, + tile=tile, + ), + self._from(function_parameters, params), + self._where_clause(where_filter), + f"LIMIT {int(limit)}", + ] + ) + + layer = self.table if mvt_settings.set_mvt_layername is True else "default" + params.append(layer) + sql = f"WITH t AS ({inner_sql}) " f"SELECT ST_AsMVT(t.*, ${len(params)}) FROM t" + debug_query(sql, *params) + + tile = await conn.fetchval(sql, *params) + + return bytes(tile) diff --git a/tipg/sqlhelpers.py b/tipg/sqlhelpers.py new file mode 100644 index 00000000..d7cabf05 --- /dev/null +++ b/tipg/sqlhelpers.py @@ -0,0 +1,38 @@ +"""tipg.sqlhelpers: shared low-level SQL/query helpers.""" + +import re +from functools import lru_cache + +from pyproj import Transformer + +from tipg.logger import logger + +# Cache CRS transformers — building a pyproj Transformer is expensive and the +# same (source, target) pairs recur across every request. +TransformerFromCRS = lru_cache(Transformer.from_crs) + + +def _quote_ident(name: str) -> str: + """Quote a PostgreSQL identifier (column/table/schema) safely.""" + return '"' + name.replace('"', '""') + '"' + + +def debug_query(q, *p): + """Utility to print raw statement to use for debugging.""" + + # Escape literal `{` and `}` (cql2 emits GeoJSON literals containing them) + # then turn `$N` placeholders into `{N}` format slots. + qsub = re.sub(r"\$([0-9]+)", r"{\1}", q.replace("{", "{{").replace("}", "}}")) + + def quote_str(s): + """Quote strings.""" + + if s is None: + return "null" + elif isinstance(s, str): + return f"'{s}'" + else: + return s + + p = [quote_str(s) for s in p] + logger.debug(qsub.format(None, *p))