fix: reduce test interference (#779)

tests/conftest.py

@@ -36,13 +36,6 @@
 dotenv.load_dotenv()
 
 
-# TODO: Redundant with Live Database Client. Consolidate and remove this.
-@pytest.fixture
-def dev_db_client() -> Generator[DatabaseClient, Any, None]:
-    db_client = DatabaseClient()
-    yield db_client
-
-
 ClientWithMockDB = namedtuple("ClientWithMockDB", ["client", "mock_db"])
 
 
@@ -115,12 +108,6 @@ def test_table_data(live_database_client: DatabaseClient):
     )
 
 
-@pytest.fixture
-def clear_data_requests(dev_db_client: DatabaseClient):
-    """Clear `data_requests` and associated tables"""
-    dev_db_client.execute_raw_sql("DELETE FROM DATA_REQUESTS;")
-
-
 @pytest.fixture(scope="session")
 def test_data_creator_db_client() -> Generator[TestDataCreatorDBClient, Any, None]:
     yield TestDataCreatorDBClient()

tests/db_client/test_database_client.py

@@ -242,10 +242,11 @@ def test_get_linked_rows(
 
 
 def test_get_unarchived_data_requests_with_issues(
-    test_data_creator_db_client: TestDataCreatorDBClient, clear_data_requests
+    test_data_creator_db_client: TestDataCreatorDBClient,
 ):
     # Add data requests with issues
     tdc = test_data_creator_db_client
+    tdc.clear_test_data()
 
     def create_data_request_with_issue_and_request_status(
         request_status: RequestStatus,

tests/db_client/test_get_data_sources_for_map.py

@@ -2,14 +2,12 @@
 from tests.helpers.helper_classes.test_data_creator.db_client_.core import (
     TestDataCreatorDBClient,
 )
-from tests.helpers.wipe import wipe_database
 
 
 def test_get_data_sources_for_map(
     live_database_client: DatabaseClient,
     test_data_creator_db_client: TestDataCreatorDBClient,
 ):
-    wipe_database(live_database_client)
     tdc = test_data_creator_db_client
     location_id = tdc.locality()
     ds_id = tdc.data_source().id

tests/helpers/helper_classes/MultiLocationSetup.py

@@ -1,3 +1,5 @@
+from sqlalchemy.exc import IntegrityError
+
 from db.db_client_dataclasses import WhereMapping
 from db.enums import LocationType
 from middleware.schema_and_dto.dtos.locations.put import LocationPutDTO
@@ -9,6 +11,13 @@
 class MultiLocationSetup:
     def __init__(self, tdc: TestDataCreatorDBClient) -> None:
         self.tdc = tdc
+        # Ensure Pittsburgh locality exists (may have been cleaned by wipe_database)
+        try:
+            tdc.locality(
+                locality_name="Pittsburgh", state_iso="PA", county_name="Allegheny"
+            )
+        except IntegrityError:
+            pass
         self.pittsburgh_id = self.get_location_id(
             {
                 "locality_name": "Pittsburgh",

tests/helpers/helper_classes/test_data_creator/db_client_/core.py

@@ -1,7 +1,6 @@
 import datetime
 import uuid
 
-from sqlalchemy import delete
 from sqlalchemy.exc import IntegrityError
 
 from db.client.core import DatabaseClient
@@ -11,21 +10,6 @@
     ExternalAccountTypeEnum,
     RequestUrgency,
 )
-from db.models.implementations.core.data_request.core import DataRequest
-from db.models.implementations.core.data_source.core import DataSource
-from db.models.implementations.core.log.notification import NotificationLog
-from db.models.implementations.core.notification.pending.data_request import (
-    DataRequestPendingEventNotification,
-)
-from db.models.implementations.core.notification.pending.data_source import (
-    DataSourcePendingEventNotification,
-)
-from db.models.implementations.core.notification.queue.data_request import (
-    DataRequestUserNotificationQueue,
-)
-from db.models.implementations.core.notification.queue.data_source import (
-    DataSourceUserNotificationQueue,
-)
 from middleware.enums import (
     JurisdictionType,
     Relations,
@@ -84,20 +68,9 @@ def test_url(self, midfix: str = "") -> str:
         return f"TEST_{midfix}_{uuid.uuid4().hex}.com"
 
     def clear_test_data(self) -> None:
-        for model in [
-            DataRequest,
-            # Agency,
-            # Locality
-            DataSource,
-            # User,
-            NotificationLog,
-            DataRequestUserNotificationQueue,
-            DataSourceUserNotificationQueue,
-            DataRequestPendingEventNotification,
-            DataSourcePendingEventNotification,
-        ]:
-            query = delete(model)
-            self.db_client.execute(query)
+        from tests.helpers.wipe import wipe_database
+
+        wipe_database(self.db_client)
 
     def county(
         self,

tests/helpers/wipe.py

@@ -1,5 +1,8 @@
 def wipe_database(db_client):
     for table in [
+        "change_log",
+        "table_count_log",
+        "notification_log",
         "agencies",
         "data_sources",
         "data_requests",

tests/integration/auth/test_reset_password.py

@@ -19,7 +19,7 @@
 
 
 def test_reset_password_post(
-    test_data_creator_flask: TestDataCreatorFlask, dev_db_client, mocker
+    test_data_creator_flask: TestDataCreatorFlask, mocker
 ):
     """
     Test that POST call to /reset-password endpoint successfully resets the user's password, and verifies the new password digest is distinct from the old one in the database
@@ -39,7 +39,7 @@ def login(password: str, expected_response_status: HTTPStatus = HTTPStatus.OK):
     # User should be able to log in with the old password
     login(user_info.password)
 
-    old_password_digest = dev_db_client.get_user_info(user_info.email).password_digest
+    old_password_digest = tdc.db_client.get_user_info(user_info.email).password_digest
 
     token = request_reset_password_api(tdc.flask_client, mocker, user_info)
 
@@ -64,7 +64,7 @@ def login(password: str, expected_response_status: HTTPStatus = HTTPStatus.OK):
         password=new_password,
     )
 
-    new_password_digest = dev_db_client.get_user_info(user_info.email).password_digest
+    new_password_digest = tdc.db_client.get_user_info(user_info.email).password_digest
     assert new_password_digest != old_password_digest, (
         "Old and new password digests should be distinct"
     )

tests/integration/data_sources/get/by_id/test_happy_path.py

@@ -19,7 +19,6 @@ def test_data_sources_by_id_get(test_data_creator_flask: TestDataCreatorFlask):
     retrieves the data source with the correct homepage URL
     """
     tdc = test_data_creator_flask
-    tdc.clear_test_data()
 
     tus = tdc.standard_user()
     cds = tdc.data_source()

tests/integration/github_data_requests_issues/happy_path/manager.py

@@ -20,7 +20,6 @@
 class TestSynchronizeGithubIssueHappyPathManager:
     def __init__(self, tdc: TestDataCreatorFlask, monkeypatch_):
         self.tdc = tdc
-        self.tdc.clear_test_data()
         self.db_client = tdc.db_client
         self.mock_issue_count = 0
         self.mock_repo: dict[int, GIPIInfo] = {}

tests/integration/github_data_requests_issues/test_denied.py

@@ -8,7 +8,7 @@
 
 
 def test_synchronize_github_issue_denied(
-    test_data_creator_flask: TestDataCreatorFlask, monkeypatch, clear_data_requests
+    test_data_creator_flask: TestDataCreatorFlask, monkeypatch
 ):
     # Give a user every permission except github_sync
     tdc = test_data_creator_flask

tests/integration/notifications/pending_to_queue/location/manager.py

@@ -38,24 +38,37 @@
 class NotificationsPendingToQueueLocationTestManager:
     def __init__(self, tdc: TestDataCreatorDBClient):
         self.tdc = tdc
-        self.tdc.clear_test_data()
         self.db_client = tdc.db_client
+        self.dr_checker: DataRequestsEventQueueChecker | None = None
+        self.ds_checker: DataSourcesEventQueueChecker | None = None
+
+    def _create_test_data(self):
+        """Clear and recreate test data.
+
+        Uses targeted cleanup instead of clear_test_data() because this
+        test depends on locality fixtures (pittsburgh_id, allegheny_id)
+        that must survive cleanup.
+        """
+        for model in [
+            DataRequestUserNotificationQueue,
+            DataSourceUserNotificationQueue,
+            DataRequestPendingEventNotification,
+            DataSourcePendingEventNotification,
+        ]:
+            self.db_client.execute(delete(model))
+        for table in ["agencies", "data_sources", "data_requests", "users"]:
+            self.db_client.execute_raw_sql("DELETE FROM " + table)
         self.user_id_1 = self.tdc.user().id
         self.user_id_2 = self.tdc.user().id
         self.data_source_id = self.tdc.data_source().id
         self.data_request_id = self.tdc.data_request().id
-        # Clear pending tables to ensure clean slate
-        self._clear_pending_tables()
-        self.dr_checker: DataRequestsEventQueueChecker | None = None
-        self.ds_checker: DataSourcesEventQueueChecker | None = None
-
-    def _clear_pending_tables(self):
+        # Clear pending tables again after data creation, since DB triggers
+        # may auto-insert pending notifications when data sources are created
         for model in [
             DataRequestPendingEventNotification,
             DataSourcePendingEventNotification,
         ]:
-            query = delete(model)
-            self.db_client.execute(query)
+            self.db_client.execute(delete(model))
 
     def _setup_user_follows(self, location_id: int):
         for user_id in [self.user_id_1, self.user_id_2]:
@@ -89,6 +102,7 @@ def _add_entries_to_pending(self):
         self.tdc.db_client.add_many(entries)
 
     def setup(self, follow_location_id: int, entity_location_id: int) -> None:
+        self._create_test_data()
         self._setup_user_follows(location_id=follow_location_id)
         self._setup_entity_location(location_id=entity_location_id)
         self._add_entries_to_pending()
@@ -99,6 +113,7 @@ def setup(self, follow_location_id: int, entity_location_id: int) -> None:
     def setup_follow_locations(
         self, follow_location_ids: list[int], entity_location_id: int
     ) -> None:
+        self._create_test_data()
         for follow_location_id in follow_location_ids:
             self._setup_user_follows(location_id=follow_location_id)
         self._setup_entity_location(location_id=entity_location_id)

tests/integration/search/conftest.py

@@ -5,13 +5,11 @@
 )
 from tests.integration.search.constants import TEST_LOCALITY, TEST_STATE, TEST_COUNTY
 from tests.integration.search.search_test_setup import SearchTestSetup
-from tests.helpers.wipe import wipe_database
 
 
 @pytest.fixture
 def search_test_setup(test_data_creator_flask: TestDataCreatorFlask):
     tdc = test_data_creator_flask
-    wipe_database(tdc.db_client)
 
     try:
         tdc.locality(TEST_LOCALITY)

tests/integration/search/tests/test_search_federal.py

@@ -8,7 +8,6 @@
 
 def test_search_federal(test_data_creator_flask: TestDataCreatorFlask):
     tdc = test_data_creator_flask
-    tdc.clear_test_data()
     # Create two approved federal agencies
     agency_ids = []
     for i in range(2):

tests/integration/test_data_requests.py

@@ -44,9 +44,6 @@ def test_data_requests_get(
     test_data_creator_flask: TestDataCreatorFlask,
 ):
     tdc = test_data_creator_flask
-    # Delete all data from the data requests table
-    tdc.db_client.execute_raw_sql("""DELETE FROM data_requests""")
-    tdc.clear_test_data()
 
     tus_creator = tdc.standard_user()
 

tests/integration/test_locations.py

@@ -76,7 +76,6 @@ def test_locations_related_data_requests(locations_test_setup: LocationsTestSetu
 
 def test_map_locations(test_data_creator_flask: TestDataCreatorFlask):
     tdc = test_data_creator_flask
-    tdc.clear_test_data()
     mls = MultiLocationSetup(tdc.tdcdb)
 
     data = tdc.request_validator.get_locations_map(

tests/integration/test_match.py

@@ -45,7 +45,6 @@ def match_agency_setup(
     test_data_creator_flask: TestDataCreatorFlask,
 ) -> TestMatchAgencySetup:
     tdc = test_data_creator_flask
-    tdc.clear_test_data()
     loc_info: TestMatchLocationInfo = TestMatchLocationInfo(tdc)
     agency = tdc.agency(location_ids=[loc_info.locality_id])
     return TestMatchAgencySetup(

tests/integration/test_typeahead_suggestions.py

@@ -121,7 +121,6 @@ def test_typeahead_agencies_approved(test_data_creator_flask: TestDataCreatorFla
     Test that GET call to /typeahead/agencies endpoint successfully retrieves data
     """
     tdc = test_data_creator_flask
-    tdc.clear_test_data()
     tdc.locality(locality_name="Qzy")
     agency_id = tdc.agency(agency_name="Qzy").id
     tdc.refresh_typeahead_agencies()

tests/integration/user/patch/test_happy_path.py

@@ -17,7 +17,6 @@ def get_capacities_from_db(tdc, user_id: int):
 
 def test_user_patch(test_data_creator_flask: TestDataCreatorFlask):
     tdc = test_data_creator_flask
-    tdc.clear_test_data()
     tus = tdc.standard_user()
 
     user_id = tus.user_info.user_id

tests/test_database.py

@@ -41,16 +41,16 @@ def setup_fake_locations(live_database_client: DatabaseClient):
         column_value_mappings=FAKE_STATE_INFO,
     )
     # Populate `counties` with data, returning id
-    FAKE_COUNTY_INFO.update({"state_id": state_id})
+    county_info = {**FAKE_COUNTY_INFO, "state_id": state_id}
     county_id = live_database_client.create_or_get(
         table_name=Relations.COUNTIES.value,
-        column_value_mappings=FAKE_COUNTY_INFO,
+        column_value_mappings=county_info,
     )
     # Populate `localities` with data, returning id
-    FAKE_LOCALITY_INFO.update({"county_id": county_id})
+    locality_info = {**FAKE_LOCALITY_INFO, "county_id": county_id}
     locality_id = live_database_client.create_or_get(
         table_name=Relations.LOCALITIES.value,
-        column_value_mappings=FAKE_LOCALITY_INFO,
+        column_value_mappings=locality_info,
     )
 
     yield FakeLocationsInfo(state_id, county_id, locality_id)