fix: 534 related schedule and realtime feeds dont update when changed (#1501)

* changed feed references logic

Author: qcdyx

api/src/scripts/populate_db_gtfs.py

@@ -127,21 +127,33 @@ def process_feed_references(self, session: "Session"):
             if data_type != "gtfs_rt":
                 continue
             gtfs_rt_feed = self.query_feed_by_stable_id(session, stable_id, "gtfs_rt")
+
+            # Parse CSV static_reference only to derive relationships (do not persist the raw value)
             static_reference = self.get_safe_value(row, "static_reference", "")
+            previous = [f.stable_id for f in getattr(gtfs_rt_feed, "gtfs_feeds", [])] if gtfs_rt_feed else []
+            gtfs_rt_feed.gtfs_feeds = []
             if static_reference:
-                try:
-                    gtfs_stable_id = f"mdb-{int(float(static_reference))}"
-                except ValueError:
-                    gtfs_stable_id = static_reference
-                gtfs_feed = self.query_feed_by_stable_id(session, gtfs_stable_id, "gtfs")
-                if not gtfs_feed:
-                    self.logger.warning(f"Could not find static reference feed {gtfs_stable_id} for feed {stable_id}")
-                    continue
-                already_referenced_ids = {ref.id for ref in gtfs_feed.gtfs_rt_feeds}
-                if gtfs_feed and gtfs_rt_feed.id not in already_referenced_ids:
-                    gtfs_feed.gtfs_rt_feeds.append(gtfs_rt_feed)
-                    # Flush to avoid FK violation
-                    session.flush()
+                raw_tokens = [tok.strip() for tok in str(static_reference).split("|") if tok and tok.strip()]
+                matched_feeds = []
+                for token in raw_tokens:
+                    try:
+                        gtfs_stable_id = f"mdb-{int(float(token))}"
+                    except ValueError:
+                        gtfs_stable_id = token
+                    gtfs_feed = self.query_feed_by_stable_id(session, gtfs_stable_id, "gtfs")
+                    if not gtfs_feed:
+                        self.logger.warning(
+                            f"Could not find static reference feed {gtfs_stable_id} for feed {stable_id}"
+                        )
+                        continue
+                    matched_feeds.append(gtfs_feed)
+
+                gtfs_rt_feed.gtfs_feeds = matched_feeds
+                session.add(gtfs_rt_feed)
+                session.flush()
+                self.logger.info(
+                    f"Set feed references for {stable_id}: {previous} -> {[f.stable_id for f in matched_feeds]}"
+                )
 
     def process_redirects(self, session: "Session"):
         """

api/tests/integration/populate_tests/README.md

@@ -0,0 +1,18 @@
+## Adding Tests to populate_tests
+
+To add a test in `populate_tests`, you need to work with 3 `sources_test.csv` files in sequence:
+
+1. **`tests/test_data/sources_test.csv`** - The base test data file. The data from this file is common to all tests in the tests directory (and subdirectories)
+2. **`tests/integration/test_data/sources_test.csv`** - The data from this file is loaded in the DB on top of the previous content. Data from this file is common to all tests in the tests/integration directory (and subdirectories).
+3. **`tests/integration/populate_tests/test_data/sources_test.csv`** - Similarly, the data of this file is loaded in the DB on top of the previous content.
+
+This allows having common data for certain tests according to the directory structure.
+Note also that data in a given sources_test.csv can overwrite data from the previous files (for example, to test an update scenario).
+
+### Example
+
+We want to test that for a realtime feed that has a certain static reference, we can change the reference and it will remove the original one and keep the new one (the current bug is that both feed references are kept in the DB)
+1. `tests/test_data/integration/source_tests.csv` contains the rt feed 1562 with the static reference 40. After the file is loaded, that is what put in the DB.
+2. `tests/test_data/integration/populate_tests/source_tests.csv` has a line for rt feed 1562 that modifies the static reference from 40 to 50. 
+2. Write your test assertions in `test_populate.py` to validate the expected behavior. In our example, we want to check that the static reference for feed 1562 is 50, and not both 40 and 50. 
+3. Run the populate script/tests

api/tests/integration/populate_tests/test_data/sources_test.csv

@@ -1,5 +1,5 @@
 mdb_source_id,data_type,entity_type,location.country_code,location.subdivision_name,location.municipality,provider,is_official,name,note,feed_contact_email,static_reference,urls.direct_download,urls.authentication_type,urls.authentication_info,urls.api_key_parameter_name,urls.latest,urls.license,location.bounding_box.minimum_latitude,location.bounding_box.maximum_latitude,location.bounding_box.minimum_longitude,location.bounding_box.maximum_longitude,location.bounding_box.extracted_on,status,features,redirect.id,redirect.comment
 40,gtfs,,CA,Ontario,London,London Transit Commission,,,,croy@londontransit.ca,,http://www.londontransit.ca/gtfsfeed/google_transit.zip,0,,,https://storage.googleapis.com/storage/v1/b/mdb-latest/o/ca-ontario-london-transit-commission-gtfs-2.zip?alt=media,https://www.londontransit.ca/open-data/ltcs-open-data-terms-of-use/,42.905244,43.051188,-81.36311,-81.137591,2022-02-22T19:51:34+00:00,inactive,,,
 50,gtfs,,CA,Ontario,Barrie,ZBarrie Transit,,,,,,http://www.myridebarrie.ca/gtfs/Google_transit.zip,,,,https://storage.googleapis.com/storage/v1/b/mdb-latest/o/ca-ontario-barrie-transit-gtfs-3.zip?alt=media,https://www.barrie.ca/services-payments/transportation-parking/barrie-transit/barrie-gtfs,44.3218044,44.42020676,-79.74063237,-79.61089569,2022-03-01T22:43:25+00:00,deprecated,,40|mdb-702,Some|Comment
-1562,gtfs-rt,sa,CA,BC,Vancouver,Vancouver-Transit(éèàçíóúČ),TRUE,Realtime(ŘŤÜÎ),,,40,http://foo.org/google_transit.zip,0,,,,,,,,,,active,,10,
+1562,gtfs-rt,sa,CA,BC,Vancouver,Vancouver-Transit(éèàçíóúČ),TRUE,Realtime(ŘŤÜÎ),,,50,http://foo.org/google_transit.zip,0,,,,,,,,,,active,,10,
 1563,gtfs-rt,tu,US,SomeState,SomeCity,SomeCity Bus,FALSE,RT,,,mdb-50,http://bar.com,0,,,,,,,,,,inactive,,10,

api/tests/integration/populate_tests/test_populate.py

@@ -53,3 +53,23 @@ def test_is_official_overwrite(client: TestClient, values):
     assert response.status_code == 200
     json_response = response.json()
     assert json_response["official"] is expected_official, values["assert_fail_message"]
+
+
+def test_is_feed_reference_overwrite(client: TestClient):
+    feed_id = "mdb-1562"
+    response = client.request(
+        "GET",
+        "/v1/gtfs_rt_feeds/{id}".format(id=feed_id),
+        headers=authHeaders,
+    )
+    json_response = response.json()
+    assert json_response["feed_references"] == ["mdb-50"]
+
+    feed_id = "mdb-1563"
+    response = client.request(
+        "GET",
+        "/v1/gtfs_rt_feeds/{id}".format(id=feed_id),
+        headers=authHeaders,
+    )
+    json_response = response.json()
+    assert json_response["feed_references"] == ["mdb-50"]

api/tests/integration/test_data/sources_test.csv

@@ -2,5 +2,5 @@ mdb_source_id,data_type,entity_type,location.country_code,location.subdivision_n
 40,gtfs,,CA,Ontario,London,London Transit Commission,TRUE,,,croy@londontransit.ca,,http://www.londontransit.ca/gtfsfeed/google_transit.zip,0,,,https://storage.googleapis.com/storage/v1/b/mdb-latest/o/ca-ontario-london-transit-commission-gtfs-2.zip?alt=media,https://www.londontransit.ca/open-data/ltcs-open-data-terms-of-use/,42.905244,43.051188,-81.36311,-81.137591,2022-02-22T19:51:34+00:00,inactive,,,
 50,gtfs,,CA,Ontario,Barrie,ZBarrie Transit,FALSE,,,,,http://www.myridebarrie.ca/gtfs/Google_transit.zip,,,,https://storage.googleapis.com/storage/v1/b/mdb-latest/o/ca-ontario-barrie-transit-gtfs-3.zip?alt=media,https://www.barrie.ca/services-payments/transportation-parking/barrie-transit/barrie-gtfs,44.3218044,44.42020676,-79.74063237,-79.61089569,2022-03-01T22:43:25+00:00,deprecated,,40|mdb-702,Some|Comment
 702,gtfs,,CA,[British Columbia,Whistler],BC Transit (Whistler Transit System),,,,,,http://whistler.mapstrat.com/current/google_transit.zip,,,,https://storage.googleapis.com/storage/v1/b/mdb-latest/o/ca-british-columbia-bc-transit-whistler-transit-system-gtfs-702.zip?alt=media,https://www.bctransit.com/open-data/terms-of-use,50.077122,50.159071,-123.043635,-122.926836,2022-03-16T22:05:05+00:00,development,,,
-1562,gtfs-rt,sa,CA,BC,Vancouver,Vancouver-Transit(éèàçíóúČ),True,Realtime(ŘŤÜÎ),,,40,http://foo.org/google_transit.zip,0,,,,,,,,,,active,,10,
+1562,gtfs-rt,sa,CA,BC,Vancouver,Vancouver-Transit(éèàçíóúČ),True,Realtime(ŘŤÜÎ),,,40|60,http://foo.org/google_transit.zip,0,,,,,,,,,,active,,10,
 1563,gtfs-rt,tu,US,SomeState,SomeCity,SomeCity Bus,False,RT,,,mdb-50,http://bar.com,0,,,,,,,,,,inactive,,10,