fix: add fix for slow full count in postgresql (#2174)

* feat: implement optional count for postgresql provider

This work has been done to allow the result count to be enabled or disabled for the PostgreSQL provider. By disabling the count you can get improved performance on large datasets but on smaller datasets this is unlikely to have any affect.

* fix: move count to base provider

This work has been done to move count to the base provider. While doing this work I also added a debug log message to state when the count had been disabled in the SQL provider.

Also, I removed some tests that were no longer needed after the introduction of the str2bool function, when getting the count value from the configuration file.

* fix: convert string true to boolean

Author: mdearos

docs/source/publishing/ogcapi-features.rst

@@ -625,6 +625,7 @@ Must have PostGIS installed.
          id_field: osm_id
          table: hotosm_bdi_waterways
          geom_field: foo_geom
+         count: true # Optional; Default true; Enable/disable count for improved performance.
 
 A number of database connection options can be also configured in the provider in order to adjust properly the sqlalchemy engine client.
 These are optional and if not specified, the default from the engine will be used. Please see also `SQLAlchemy docs <https://docs.sqlalchemy.org/en/14/core/engines.html#custom-dbapi-connect-arguments-on-connect-routines>`_.
@@ -662,6 +663,7 @@ These are optional and if not specified, the default from the engine will be use
          id_field: osm_id
          table: hotosm_bdi_waterways
          geom_field: foo_geom
+         count: true # Optional; Default true; Enable/disable count for improved performance.
 
 The PostgreSQL provider is also able to connect to Cloud SQL databases.
 
@@ -677,6 +679,7 @@ The PostgreSQL provider is also able to connect to Cloud SQL databases.
              password: postgres
          id_field: id
          table: states
+         count: true # Optional; Default true; Enable/disable count for improved performance.
 
 This is what a configuration for `Google Cloud SQL`_ connection looks like. The ``host``
 block contains the necessary socket connection information.

pygeoapi/provider/base.py

@@ -57,6 +57,8 @@ def __init__(self, provider_def):
         :returns: pygeoapi.provider.base.BaseProvider
         """
 
+        from pygeoapi.util import str2bool
+
         try:
             self.name = provider_def['name']
             self.type = provider_def['type']
@@ -65,6 +67,7 @@ def __init__(self, provider_def):
             raise RuntimeError('name/type/data are required')
 
         self.editable = provider_def.get('editable', False)
+        self.count = str2bool(provider_def.get('count', True))
         self.options = provider_def.get('options')
         self.id_field = provider_def.get('id_field')
         self.uri_field = provider_def.get('uri_field')

pygeoapi/provider/sql.py

@@ -91,7 +91,6 @@
     ProviderQueryError,
     ProviderItemNotFoundError
 )
-from pygeoapi.util import str2bool
 
 LOGGER = logging.getLogger(__name__)
 
@@ -128,7 +127,6 @@ def __init__(
         self.id_field = provider_def['id_field']
         self.geom = provider_def.get('geom_field', 'geom')
         self.driver_name = driver_name
-        self.count = str2bool(provider_def.get('count', True))
 
         LOGGER.debug(f'Name: {self.name}')
         LOGGER.debug(f'Table: {self.table}')
@@ -214,18 +212,20 @@ def query(
                 .options(selected_properties)
             )
 
-            matched = results.count()
-
-            LOGGER.debug(f'Found {matched} result(s)')
-
             LOGGER.debug('Preparing response')
             response = {
                 'type': 'FeatureCollection',
                 'features': [],
-                'numberMatched': matched,
                 'numberReturned': 0
             }
 
+            if self.count or resulttype == 'hits':
+                matched = results.count()
+                response['numberMatched'] = matched
+                LOGGER.debug(f'Found {matched} result(s)')
+            else:
+                LOGGER.debug('Count disabled')
+
             if resulttype == 'hits' or not results:
                 return response
 

tests/provider/test_postgresql_provider.py

@@ -908,3 +908,38 @@ def test_transaction_create_handles_invalid_input_data(pg_api_, data):
     headers, code, content = manage_collection_item(
         pg_api_, req, action='create', dataset='hot_osm_waterways')
     assert 'generic error' in content
+
+
+def test_provider_count_default_value(config):
+    # Arrange
+    provider = PostgreSQLProvider(config)
+
+    # Act
+    results = provider.query()
+
+    # Assert
+    assert results['numberMatched'] == 14776
+
+
+def test_provider_count_false(config):
+    # Arrange
+    config['count'] = 'false'
+    provider = PostgreSQLProvider(config)
+
+    # Act
+    results = provider.query()
+
+    # Assert
+    assert 'numberMatched' not in results
+
+
+def test_provider_count_false_with_resulttype_hits(config):
+    # Arrange
+    config['count'] = 'false'
+    provider = PostgreSQLProvider(config)
+
+    # Act
+    results = provider.query(resulttype="hits")
+
+    # Assert
+    assert results['numberMatched'] == 14776