From 769da6626fe8d124f08ef43c720f2fac3fae03f1 Mon Sep 17 00:00:00 2001 From: Christian Beilschmidt Date: Wed, 12 Feb 2025 11:26:42 +0100 Subject: [PATCH 1/7] adjust tests --- geoengine/colorizer.py | 2 +- geoengine/datasets.py | 4 +-- geoengine/workflow.py | 13 ++++++++-- tests/ge_test.py | 1 + tests/test_workflow_storage.py | 45 +++++++++++++++++----------------- 5 files changed, 38 insertions(+), 27 deletions(-) diff --git a/geoengine/colorizer.py b/geoengine/colorizer.py index 1ff96ccc..e353833c 100644 --- a/geoengine/colorizer.py +++ b/geoengine/colorizer.py @@ -309,7 +309,7 @@ def to_api_dict(self) -> geoengine_openapi_client.Colorizer: """Return the colorizer as a dictionary.""" return geoengine_openapi_client.Colorizer(geoengine_openapi_client.PaletteColorizer( type='palette', - colors=self.colors, + colors={str(k): v for k,v in self.colors.items()}, default_color=self.default_color, no_data_color=self.no_data_color, )) diff --git a/geoengine/datasets.py b/geoengine/datasets.py index 036cd8d5..dd3e36e2 100644 --- a/geoengine/datasets.py +++ b/geoengine/datasets.py @@ -494,11 +494,11 @@ def upload_dataframe( ), on_error=on_error.to_api_enum(), ), - result_descriptor=VectorResultDescriptor( + result_descriptor=geoengine_openapi_client.VectorResultDescriptor.from_dict(VectorResultDescriptor( data_type=vector_type, spatial_reference=df.crs.to_string(), columns=columns, - ).to_api_dict().actual_instance + ).to_api_dict().actual_instance.to_dict()) ) ) ) diff --git a/geoengine/workflow.py b/geoengine/workflow.py index 59c39c68..9b017987 100644 --- a/geoengine/workflow.py +++ b/geoengine/workflow.py @@ -535,6 +535,13 @@ def save_as_dataset( session = get_session() + print(geoengine_openapi_client.RasterDatasetFromWorkflow( + name=name, + display_name=display_name, + description=description, + query=query_rectangle + ).to_json()) + with geoengine_openapi_client.ApiClient(session.configuration) as api_client: workflows_api = geoengine_openapi_client.WorkflowsApi(api_client) response = workflows_api.dataset_from_workflow_handler( @@ -984,8 +991,9 @@ def data_usage(offset: int = 0, limit: int = 10) -> List[geoengine_openapi_clien response = user_api.data_usage_handler(offset=offset, limit=limit) # create dataframe from response - usage_dicts = [data_usage.dict(by_alias=True) for data_usage in response] + usage_dicts = [data_usage.model_dump(by_alias=True) for data_usage in response] df = pd.DataFrame(usage_dicts) + df['timestamp'] = pd.to_datetime(df['timestamp'], utc=True) return df @@ -1005,7 +1013,8 @@ def data_usage_summary(granularity: geoengine_openapi_client.UsageSummaryGranula offset=offset, limit=limit) # create dataframe from response - usage_dicts = [data_usage.dict(by_alias=True) for data_usage in response] + usage_dicts = [data_usage.model_dump(by_alias=True) for data_usage in response] df = pd.DataFrame(usage_dicts) + df['timestamp'] = pd.to_datetime(df['timestamp'], utc=True) return df diff --git a/tests/ge_test.py b/tests/ge_test.py index f532d0a0..a9f8d3fd 100644 --- a/tests/ge_test.py +++ b/tests/ge_test.py @@ -201,6 +201,7 @@ def _start(self) -> None: 'GEOENGINE__POSTGRES__PASSWORD': POSTGRES_PASSWORD, 'GEOENGINE__POSTGRES__SCHEMA': self.db_schema, 'GEOENGINE__LOGGING__LOG_SPEC': GE_LOG_SPEC, + 'GEOENGINE__POSTGRES__CLEAR_DATABASE_ON_START': 'true', 'PATH': os.environ['PATH'], }, stderr=subprocess.PIPE, diff --git a/tests/test_workflow_storage.py b/tests/test_workflow_storage.py index 170b39a1..5bd46103 100644 --- a/tests/test_workflow_storage.py +++ b/tests/test_workflow_storage.py @@ -17,28 +17,29 @@ def setUp(self) -> None: def test_storing_workflow(self): expected_request_text = { - 'name': None, - 'displayName': 'Foo', - 'description': 'Bar', - 'query': { - 'spatialBounds': { - 'upperLeftCoordinate': { - 'x': -180.0, - 'y': 90.0 - }, - 'lowerRightCoordinate': { - 'x': 180.0, - 'y': -90.0 - } - }, - 'timeInterval': { - 'start': 1396353600000, - 'end': 1396353600000, - }, - 'spatialResolution': { - 'x': 1.8, - 'y': 1.8 - } + "asCog": True, + "description": "Bar", + "displayName": "Foo", + "name": None, + "query": { + "spatialBounds": { + "lowerRightCoordinate": { + "x": 180, + "y": -90 + }, + "upperLeftCoordinate": { + "x": -180, + "y": 90 + } + }, + "spatialResolution": { + "x": 1.8, + "y": 1.8 + }, + "timeInterval": { + "end": 1396353600000, + "start": 1396353600000 + } } } From 5ec4475d6cfa74cd879c4ec952130d835785c84b Mon Sep 17 00:00:00 2001 From: Christian Beilschmidt Date: Wed, 12 Feb 2025 11:27:49 +0100 Subject: [PATCH 2/7] WIP setup.cfg --- setup.cfg | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.cfg b/setup.cfg index 327ddedf..661bdefd 100644 --- a/setup.cfg +++ b/setup.cfg @@ -18,7 +18,7 @@ package_dir = packages = find: python_requires = >=3.9 install_requires = - geoengine-openapi-client == 0.0.19 + geoengine-openapi-client @ git+https://github.com/geo-engine/openapi-client@update-openapi-generator#subdirectory=python # file:///home/beilschmidt/git/geo-engine/openapi-client/python/ # TODO update when merged geopandas >=0.9,<0.15 matplotlib >=3.5,<3.8 numpy >=1.21,<2.1 @@ -34,7 +34,7 @@ install_requires = websockets >= 10.0,<11 xarray >=0.19,<2024.12 urllib3 >= 2.0, < 2.3 - pydantic >= 1.10.5, < 2 + pydantic >= 2, < 2.11 skl2onnx >=1.17,<2 [options.extras_require] From afe0e2f0df287146de6accb0057e8ff6eec60530 Mon Sep 17 00:00:00 2001 From: Christian Beilschmidt Date: Wed, 12 Feb 2025 16:13:08 +0100 Subject: [PATCH 3/7] fix formatting and typing --- geoengine/colorizer.py | 27 ++++++++++++++++--------- geoengine/datasets.py | 16 ++++++++++----- geoengine/error.py | 2 +- geoengine/types.py | 25 ++++++++++++++++++++++- geoengine/workflow.py | 7 ------- geoengine/workflow_builder/operators.py | 9 +++++---- 6 files changed, 58 insertions(+), 28 deletions(-) diff --git a/geoengine/colorizer.py b/geoengine/colorizer.py index e353833c..df23466b 100644 --- a/geoengine/colorizer.py +++ b/geoengine/colorizer.py @@ -230,6 +230,13 @@ def from_response(response: geoengine_openapi_client.Colorizer) -> Colorizer: raise TypeError("Unknown colorizer type") +def rgba_from_list(values: list[int]) -> Rgba: + """Convert a list of integers to an RGBA tuple.""" + if len(values) != 4: + raise ValueError(f"Expected a list of 4 integers, got {len(values)} instead.") + return (values[0], values[1], values[2], values[3]) + + @dataclass class LinearGradientColorizer(Colorizer): '''A linear gradient colorizer.''' @@ -242,10 +249,10 @@ def from_response_linear(response: geoengine_openapi_client.LinearGradient) -> L """Create a colorizer from a response.""" breakpoints = [ColorBreakpoint.from_response(breakpoint) for breakpoint in response.breakpoints] return LinearGradientColorizer( - no_data_color=response.no_data_color, + no_data_color=rgba_from_list(response.no_data_color), breakpoints=breakpoints, - over_color=response.over_color, - under_color=response.under_color, + over_color=rgba_from_list(response.over_color), + under_color=rgba_from_list(response.under_color), ) def to_api_dict(self) -> geoengine_openapi_client.Colorizer: @@ -273,9 +280,9 @@ def from_response_logarithmic( breakpoints = [ColorBreakpoint.from_response(breakpoint) for breakpoint in response.breakpoints] return LogarithmicGradientColorizer( breakpoints=breakpoints, - no_data_color=response.no_data_color, - over_color=response.over_color, - under_color=response.under_color, + no_data_color=rgba_from_list(response.no_data_color), + over_color=rgba_from_list(response.over_color), + under_color=rgba_from_list(response.under_color), ) def to_api_dict(self) -> geoengine_openapi_client.Colorizer: @@ -300,16 +307,16 @@ def from_response_palette(response: geoengine_openapi_client.PaletteColorizer) - """Create a colorizer from a response.""" return PaletteColorizer( - colors={float(k): v for k, v in response.colors.items()}, - no_data_color=response.no_data_color, - default_color=response.default_color, + colors={float(k): rgba_from_list(v) for k, v in response.colors.items()}, + no_data_color=rgba_from_list(response.no_data_color), + default_color=rgba_from_list(response.default_color), ) def to_api_dict(self) -> geoengine_openapi_client.Colorizer: """Return the colorizer as a dictionary.""" return geoengine_openapi_client.Colorizer(geoengine_openapi_client.PaletteColorizer( type='palette', - colors={str(k): v for k,v in self.colors.items()}, + colors={str(k): v for k, v in self.colors.items()}, default_color=self.default_color, no_data_color=self.no_data_color, )) diff --git a/geoengine/datasets.py b/geoengine/datasets.py index dd3e36e2..1ad9fbc4 100644 --- a/geoengine/datasets.py +++ b/geoengine/datasets.py @@ -466,6 +466,14 @@ def upload_dataframe( ints = [key for (key, value) in columns.items() if value.data_type == 'int'] texts = [key for (key, value) in columns.items() if value.data_type == 'text'] + result_descriptor = VectorResultDescriptor( + data_type=vector_type, + spatial_reference=df.crs.to_string(), + columns=columns, + ).to_api_dict().actual_instance + if not isinstance(result_descriptor, geoengine_openapi_client.TypedVectorResultDescriptor): + raise TypeError('Expected TypedVectorResultDescriptor') + create = geoengine_openapi_client.CreateDataset( data_path=geoengine_openapi_client.DataPath(geoengine_openapi_client.DataPathOneOf1( upload=str(upload_id) @@ -494,11 +502,9 @@ def upload_dataframe( ), on_error=on_error.to_api_enum(), ), - result_descriptor=geoengine_openapi_client.VectorResultDescriptor.from_dict(VectorResultDescriptor( - data_type=vector_type, - spatial_reference=df.crs.to_string(), - columns=columns, - ).to_api_dict().actual_instance.to_dict()) + result_descriptor=geoengine_openapi_client.VectorResultDescriptor.from_dict( + result_descriptor.to_dict() + ) ) ) ) diff --git a/geoengine/error.py b/geoengine/error.py index 3efbcd5f..d19fff85 100644 --- a/geoengine/error.py +++ b/geoengine/error.py @@ -21,7 +21,7 @@ def __init__(self, response: Union[geoengine_openapi_client.ApiException, Dict[s super().__init__() if isinstance(response, geoengine_openapi_client.ApiException): - obj = json.loads(response.body) + obj = json.loads(response.body) if response.body else {'error': 'unknown', 'message': 'unknown'} else: obj = response diff --git a/geoengine/types.py b/geoengine/types.py index fbdc8aab..afff0d40 100644 --- a/geoengine/types.py +++ b/geoengine/types.py @@ -660,6 +660,29 @@ def __repr__(self) -> str: return f'{self.name}: {self.measurement}' +def literal_raster_data_type( + data_type: geoengine_openapi_client.RasterDataType +) -> Literal['U8', 'U16', 'U32', 'U64', 'I8', 'I16', 'I32', 'I64', 'F32', 'F64']: + '''Convert a `RasterDataType` to a literal''' + + data_type_map: dict[ + geoengine_openapi_client.RasterDataType, + Literal['U8', 'U16', 'U32', 'U64', 'I8', 'I16', 'I32', 'I64', 'F32', 'F64'] + ] = { + geoengine_openapi_client.RasterDataType.U8: 'U8', + geoengine_openapi_client.RasterDataType.U16: 'U16', + geoengine_openapi_client.RasterDataType.U32: 'U32', + geoengine_openapi_client.RasterDataType.U64: 'U64', + geoengine_openapi_client.RasterDataType.I8: 'I8', + geoengine_openapi_client.RasterDataType.I16: 'I16', + geoengine_openapi_client.RasterDataType.I32: 'I32', + geoengine_openapi_client.RasterDataType.I64: 'I64', + geoengine_openapi_client.RasterDataType.F32: 'F32', + geoengine_openapi_client.RasterDataType.F64: 'F64', + } + return data_type_map[data_type] + + class RasterResultDescriptor(ResultDescriptor): ''' A raster result descriptor @@ -701,7 +724,7 @@ def from_response_raster( response: geoengine_openapi_client.TypedRasterResultDescriptor) -> RasterResultDescriptor: '''Parse a raster result descriptor from an http response''' spatial_ref = response.spatial_reference - data_type = response.data_type.value + data_type = literal_raster_data_type(response.data_type) bands = [RasterBandDescriptor.from_response(band) for band in response.bands] time_bounds = None diff --git a/geoengine/workflow.py b/geoengine/workflow.py index 9b017987..636ec64f 100644 --- a/geoengine/workflow.py +++ b/geoengine/workflow.py @@ -535,13 +535,6 @@ def save_as_dataset( session = get_session() - print(geoengine_openapi_client.RasterDatasetFromWorkflow( - name=name, - display_name=display_name, - description=description, - query=query_rectangle - ).to_json()) - with geoengine_openapi_client.ApiClient(session.configuration) as api_client: workflows_api = geoengine_openapi_client.WorkflowsApi(api_client) response = workflows_api.dataset_from_workflow_handler( diff --git a/geoengine/workflow_builder/operators.py b/geoengine/workflow_builder/operators.py index 00dcac5a..967361da 100644 --- a/geoengine/workflow_builder/operators.py +++ b/geoengine/workflow_builder/operators.py @@ -694,11 +694,12 @@ def from_operator_dict(cls, operator_dict: Dict[str, Any]) -> 'Expression': output_band = None if "outputBand" in operator_dict["params"] and operator_dict["params"]["outputBand"] is not None: - output_band = RasterBandDescriptor.from_response( - geoengine_openapi_client.RasterBandDescriptor.from_dict( - operator_dict["params"]["outputBand"] - ) + raster_band_descriptor = geoengine_openapi_client.RasterBandDescriptor.from_dict( + operator_dict["params"]["outputBand"] ) + if raster_band_descriptor is None: + raise ValueError("Invalid output band") + output_band = RasterBandDescriptor.from_response(raster_band_descriptor) return Expression( expression=operator_dict["params"]["expression"], From fffeacd1e15676d490640f29e48d14bec9d08b59 Mon Sep 17 00:00:00 2001 From: Christian Beilschmidt Date: Wed, 12 Feb 2025 16:48:01 +0100 Subject: [PATCH 4/7] fix test --- setup.cfg | 2 +- tests/test_ml.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.cfg b/setup.cfg index 661bdefd..4f2c16c3 100644 --- a/setup.cfg +++ b/setup.cfg @@ -34,7 +34,7 @@ install_requires = websockets >= 10.0,<11 xarray >=0.19,<2024.12 urllib3 >= 2.0, < 2.3 - pydantic >= 2, < 2.11 + pydantic >= 2.10.6, < 2.11 skl2onnx >=1.17,<2 [options.extras_require] diff --git a/tests/test_ml.py b/tests/test_ml.py index 0d7e9d21..0bf975f9 100644 --- a/tests/test_ml.py +++ b/tests/test_ml.py @@ -106,7 +106,7 @@ def test_uploading_onnx_model(self): ) self.assertEqual( str(exception.exception), - 'Model input type `TensorProto.FLOAT` does not match the expected type `RasterDataType.F64`' + 'Model input type `TensorProto.FLOAT` does not match the expected type `F64`' ) with self.assertRaises(ge.InputException) as exception: @@ -126,5 +126,5 @@ def test_uploading_onnx_model(self): ) self.assertEqual( str(exception.exception), - 'Model output type `TensorProto.INT64` does not match the expected type `RasterDataType.I32`' + 'Model output type `TensorProto.INT64` does not match the expected type `I32`' ) From 31ad668ff3429d7c81f3c44a8c8e46f1f03ad655 Mon Sep 17 00:00:00 2001 From: Christian Beilschmidt Date: Wed, 12 Feb 2025 17:53:08 +0100 Subject: [PATCH 5/7] fix ml error msg --- geoengine/ml.py | 6 ++++-- tests/test_ml.py | 4 ++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/geoengine/ml.py b/geoengine/ml.py index adcdeda2..dd123b46 100644 --- a/geoengine/ml.py +++ b/geoengine/ml.py @@ -103,10 +103,12 @@ def check_data_type(data_type: TypeProto, expected_type: RasterDataType, prefix: if not data_type.tensor_type: raise InputException('Only tensor input types are supported') elem_type = data_type.tensor_type.elem_type - if elem_type != RASTER_TYPE_TO_ONNX_TYPE[expected_type]: + expected_tensor_type = RASTER_TYPE_TO_ONNX_TYPE[expected_type] + if elem_type != expected_tensor_type: elem_type_str = tensor_dtype_to_string(elem_type) + expected_type_str = tensor_dtype_to_string(expected_tensor_type) raise InputException(f'Model {prefix} type `{elem_type_str}` does not match the ' - f'expected type `{expected_type}`') + f'expected type `{expected_type_str}`') model_inputs = onnx_model.graph.input model_outputs = onnx_model.graph.output diff --git a/tests/test_ml.py b/tests/test_ml.py index 0bf975f9..768dae18 100644 --- a/tests/test_ml.py +++ b/tests/test_ml.py @@ -106,7 +106,7 @@ def test_uploading_onnx_model(self): ) self.assertEqual( str(exception.exception), - 'Model input type `TensorProto.FLOAT` does not match the expected type `F64`' + 'Model input type `TensorProto.FLOAT` does not match the expected type `TensorProto.DOUBLE`' ) with self.assertRaises(ge.InputException) as exception: @@ -126,5 +126,5 @@ def test_uploading_onnx_model(self): ) self.assertEqual( str(exception.exception), - 'Model output type `TensorProto.INT64` does not match the expected type `I32`' + 'Model output type `TensorProto.INT64` does not match the expected type `TensorProto.INT32`' ) From 8da12285c01ec8312be50aa9a7bbd20078fe4c29 Mon Sep 17 00:00:00 2001 From: Christian Beilschmidt Date: Thu, 13 Feb 2025 12:38:35 +0100 Subject: [PATCH 6/7] fix version --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 4f2c16c3..5a79755b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -18,7 +18,7 @@ package_dir = packages = find: python_requires = >=3.9 install_requires = - geoengine-openapi-client @ git+https://github.com/geo-engine/openapi-client@update-openapi-generator#subdirectory=python # file:///home/beilschmidt/git/geo-engine/openapi-client/python/ # TODO update when merged + geoengine-openapi-client == 0.0.21 geopandas >=0.9,<0.15 matplotlib >=3.5,<3.8 numpy >=1.21,<2.1 From f4fbbfd66b27400fd92aa8834bfd72f04cce4fa1 Mon Sep 17 00:00:00 2001 From: Christian Beilschmidt Date: Fri, 14 Feb 2025 07:58:07 +0100 Subject: [PATCH 7/7] empty df fix --- geoengine/workflow.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/geoengine/workflow.py b/geoengine/workflow.py index 636ec64f..cb1fb935 100644 --- a/geoengine/workflow.py +++ b/geoengine/workflow.py @@ -986,7 +986,8 @@ def data_usage(offset: int = 0, limit: int = 10) -> List[geoengine_openapi_clien # create dataframe from response usage_dicts = [data_usage.model_dump(by_alias=True) for data_usage in response] df = pd.DataFrame(usage_dicts) - df['timestamp'] = pd.to_datetime(df['timestamp'], utc=True) + if 'timestamp' in df.columns: + df['timestamp'] = pd.to_datetime(df['timestamp'], utc=True) return df @@ -1008,6 +1009,7 @@ def data_usage_summary(granularity: geoengine_openapi_client.UsageSummaryGranula # create dataframe from response usage_dicts = [data_usage.model_dump(by_alias=True) for data_usage in response] df = pd.DataFrame(usage_dicts) - df['timestamp'] = pd.to_datetime(df['timestamp'], utc=True) + if 'timestamp' in df.columns: + df['timestamp'] = pd.to_datetime(df['timestamp'], utc=True) return df