From cf8a837a72ebf53ac36887bdf9d46f7c54ea3589 Mon Sep 17 00:00:00 2001 From: kraysent Date: Thu, 12 Mar 2026 12:00:14 +0000 Subject: [PATCH 1/9] #414: designations layer 2 catalog --- .../V030__additional_designations.sql | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 postgres/migrations/V030__additional_designations.sql diff --git a/postgres/migrations/V030__additional_designations.sql b/postgres/migrations/V030__additional_designations.sql new file mode 100644 index 00000000..6e6eae32 --- /dev/null +++ b/postgres/migrations/V030__additional_designations.sql @@ -0,0 +1,19 @@ +CREATE VIEW layer2.designations AS +SELECT + r.pgc +, d.design +, t.bib +, b.code +, b.year +, b.author +, b.title +FROM + designation.data AS d + LEFT JOIN layer0.records AS r ON (d.record_id = r.id) + LEFT JOIN layer0.tables AS t ON (r.table_id = t.id) + LEFT JOIN common.bib AS b ON (t.bib = b.id) +WHERE r.pgc IS NOT NULL; + +CREATE INDEX IF NOT EXISTS layer0_records_id_pgc_not_null +ON layer0.records (id) +WHERE pgc IS NOT NULL; From 9f8df10e158c30ca8e3833a8a5c08835bdbd2fed Mon Sep 17 00:00:00 2001 From: kraysent Date: Thu, 12 Mar 2026 12:45:05 +0000 Subject: [PATCH 2/9] rename object --- app/data/model/__init__.py | 3 +- app/data/model/layer2.py | 6 -- tests/integration/layer2_repository_test.py | 102 +++++++++++--------- tests/lib/postgres.py | 7 +- 4 files changed, 66 insertions(+), 52 deletions(-) diff --git a/app/data/model/__init__.py b/app/data/model/__init__.py index be6aba96..91166ff8 100644 --- a/app/data/model/__init__.py +++ b/app/data/model/__init__.py @@ -3,7 +3,7 @@ from app.data.model.helpers import get_catalog_object_type from app.data.model.icrs import ICRSCatalogObject from app.data.model.interface import CatalogObject, MeasuredValue, RawCatalog, get_object -from app.data.model.layer2 import Layer2CatalogObject, Layer2Object +from app.data.model.layer2 import Layer2Object from app.data.model.nature import NatureCatalogObject from app.data.model.records import ( CrossmatchRecordRow, @@ -40,7 +40,6 @@ "RedshiftRecord", "Record", "StructuredData", - "Layer2CatalogObject", "Layer2Object", "TableRecord", "RawCatalog", diff --git a/app/data/model/layer2.py b/app/data/model/layer2.py index afd90d3d..b6eecf2d 100644 --- a/app/data/model/layer2.py +++ b/app/data/model/layer2.py @@ -3,12 +3,6 @@ from app.data.model import interface -@dataclass -class Layer2CatalogObject: - pgc: int - catalog_object: interface.CatalogObject - - @dataclass class Layer2Object: pgc: int diff --git a/tests/integration/layer2_repository_test.py b/tests/integration/layer2_repository_test.py index 3629737d..62f2e2d4 100644 --- a/tests/integration/layer2_repository_test.py +++ b/tests/integration/layer2_repository_test.py @@ -21,19 +21,20 @@ def setUpClass(cls) -> None: def tearDown(self): self.pg_storage.clear() - def _save_layer2_data(self, objects: list[model.Layer2CatalogObject]) -> None: - by_table: dict[str, list[model.Layer2CatalogObject]] = {} + def _save_layer2_data(self, objects: list[model.Layer2Object]) -> None: + by_table: dict[str, list[tuple[int, model.CatalogObject]]] = {} for obj in objects: - table = obj.catalog_object.layer2_table() - if table not in by_table: - by_table[table] = [] - by_table[table].append(obj) - for table, table_objects in by_table.items(): - if not table_objects: + for catalog_obj in obj.data: + table = catalog_obj.layer2_table() + if table not in by_table: + by_table[table] = [] + by_table[table].append((obj.pgc, catalog_obj)) + for table, table_entries in by_table.items(): + if not table_entries: continue - columns = table_objects[0].catalog_object.layer2_keys() - pgcs = [obj.pgc for obj in table_objects] - data = [[obj.catalog_object.layer2_data()[c] for c in columns] for obj in table_objects] + columns = table_entries[0][1].layer2_keys() + pgcs = [pgc for pgc, _ in table_entries] + data = [[catalog_obj.layer2_data()[c] for c in columns] for _, catalog_obj in table_entries] self.layer2_repo.save(table, columns, pgcs, data) def _get_table(self, table_name: str) -> int: @@ -42,9 +43,9 @@ def _get_table(self, table_name: str) -> int: return table_resp.table_id def test_one_object(self): - objects: list[model.Layer2CatalogObject] = [ - model.Layer2CatalogObject(1, model.DesignationCatalogObject(design="test")), - model.Layer2CatalogObject(2, model.DesignationCatalogObject(design="test2")), + objects: list[model.Layer2Object] = [ + model.Layer2Object(1, [model.DesignationCatalogObject(design="test")]), + model.Layer2Object(2, [model.DesignationCatalogObject(design="test2")]), ] self.common_repo.register_pgcs([1, 2]) @@ -62,9 +63,9 @@ def test_one_object(self): self.assertEqual(actual, expected) def test_several_objects(self): - objects: list[model.Layer2CatalogObject] = [ - model.Layer2CatalogObject(1, model.ICRSCatalogObject(ra=10, dec=10, e_ra=0.1, e_dec=0.1)), - model.Layer2CatalogObject(2, model.ICRSCatalogObject(ra=11, dec=11, e_ra=0.1, e_dec=0.1)), + objects: list[model.Layer2Object] = [ + model.Layer2Object(1, [model.ICRSCatalogObject(ra=10, dec=10, e_ra=0.1, e_dec=0.1)]), + model.Layer2Object(2, [model.ICRSCatalogObject(ra=11, dec=11, e_ra=0.1, e_dec=0.1)]), ] self.common_repo.register_pgcs([1, 2]) @@ -86,9 +87,14 @@ def test_several_objects(self): def test_several_catalogs(self): objects = [ - model.Layer2CatalogObject(1, model.ICRSCatalogObject(ra=10, dec=10, e_ra=0.1, e_dec=0.1)), - model.Layer2CatalogObject(2, model.ICRSCatalogObject(ra=11, dec=11, e_ra=0.1, e_dec=0.1)), - model.Layer2CatalogObject(2, model.DesignationCatalogObject(design="test2")), + model.Layer2Object(1, [model.ICRSCatalogObject(ra=10, dec=10, e_ra=0.1, e_dec=0.1)]), + model.Layer2Object( + 2, + [ + model.ICRSCatalogObject(ra=11, dec=11, e_ra=0.1, e_dec=0.1), + model.DesignationCatalogObject(design="test2"), + ], + ), ] self.common_repo.register_pgcs([1, 2]) @@ -115,10 +121,20 @@ def test_several_catalogs(self): def test_several_filters(self): objects = [ - model.Layer2CatalogObject(1, model.ICRSCatalogObject(ra=10, dec=10, e_ra=0.1, e_dec=0.1)), - model.Layer2CatalogObject(2, model.ICRSCatalogObject(ra=11, dec=11, e_ra=0.1, e_dec=0.1)), - model.Layer2CatalogObject(2, model.DesignationCatalogObject(design="test2")), - model.Layer2CatalogObject(1, model.DesignationCatalogObject(design="test")), + model.Layer2Object( + 1, + [ + model.ICRSCatalogObject(ra=10, dec=10, e_ra=0.1, e_dec=0.1), + model.DesignationCatalogObject(design="test"), + ], + ), + model.Layer2Object( + 2, + [ + model.ICRSCatalogObject(ra=11, dec=11, e_ra=0.1, e_dec=0.1), + model.DesignationCatalogObject(design="test2"), + ], + ), ] self.common_repo.register_pgcs([1, 2]) @@ -154,12 +170,12 @@ def test_several_filters(self): self.assertEqual(actual, expected) def test_pagination(self): - objects: list[model.Layer2CatalogObject] = [ - model.Layer2CatalogObject(1, model.ICRSCatalogObject(ra=10, dec=10, e_ra=0.1, e_dec=0.1)), - model.Layer2CatalogObject(2, model.ICRSCatalogObject(ra=11, dec=11, e_ra=0.1, e_dec=0.1)), - model.Layer2CatalogObject(3, model.ICRSCatalogObject(ra=12, dec=12, e_ra=0.1, e_dec=0.1)), - model.Layer2CatalogObject(4, model.ICRSCatalogObject(ra=13, dec=13, e_ra=0.1, e_dec=0.1)), - model.Layer2CatalogObject(5, model.ICRSCatalogObject(ra=14, dec=14, e_ra=0.1, e_dec=0.1)), + objects: list[model.Layer2Object] = [ + model.Layer2Object(1, [model.ICRSCatalogObject(ra=10, dec=10, e_ra=0.1, e_dec=0.1)]), + model.Layer2Object(2, [model.ICRSCatalogObject(ra=11, dec=11, e_ra=0.1, e_dec=0.1)]), + model.Layer2Object(3, [model.ICRSCatalogObject(ra=12, dec=12, e_ra=0.1, e_dec=0.1)]), + model.Layer2Object(4, [model.ICRSCatalogObject(ra=13, dec=13, e_ra=0.1, e_dec=0.1)]), + model.Layer2Object(5, [model.ICRSCatalogObject(ra=14, dec=14, e_ra=0.1, e_dec=0.1)]), ] self.common_repo.register_pgcs([1, 2, 3, 4, 5]) @@ -176,12 +192,12 @@ def test_pagination(self): self.assertEqual(len(actual), 2) def test_batch_query(self): - objects: list[model.Layer2CatalogObject] = [ - model.Layer2CatalogObject(1, model.ICRSCatalogObject(ra=10, dec=10, e_ra=0.1, e_dec=0.1)), - model.Layer2CatalogObject(2, model.ICRSCatalogObject(ra=11, dec=11, e_ra=0.1, e_dec=0.1)), - model.Layer2CatalogObject(3, model.ICRSCatalogObject(ra=12, dec=12, e_ra=0.1, e_dec=0.1)), - model.Layer2CatalogObject(4, model.ICRSCatalogObject(ra=13, dec=13, e_ra=0.1, e_dec=0.1)), - model.Layer2CatalogObject(5, model.ICRSCatalogObject(ra=14, dec=14, e_ra=0.1, e_dec=0.1)), + objects: list[model.Layer2Object] = [ + model.Layer2Object(1, [model.ICRSCatalogObject(ra=10, dec=10, e_ra=0.1, e_dec=0.1)]), + model.Layer2Object(2, [model.ICRSCatalogObject(ra=11, dec=11, e_ra=0.1, e_dec=0.1)]), + model.Layer2Object(3, [model.ICRSCatalogObject(ra=12, dec=12, e_ra=0.1, e_dec=0.1)]), + model.Layer2Object(4, [model.ICRSCatalogObject(ra=13, dec=13, e_ra=0.1, e_dec=0.1)]), + model.Layer2Object(5, [model.ICRSCatalogObject(ra=14, dec=14, e_ra=0.1, e_dec=0.1)]), ] self.common_repo.register_pgcs([1, 2, 3, 4, 5]) @@ -227,8 +243,8 @@ def test_get_orphaned_pgcs_returns_pgcs_without_layer1_data(self) -> None: self.common_repo.register_pgcs([1, 2]) self._save_layer2_data( [ - model.Layer2CatalogObject(1, model.DesignationCatalogObject(design="a")), - model.Layer2CatalogObject(2, model.DesignationCatalogObject(design="b")), + model.Layer2Object(1, [model.DesignationCatalogObject(design="a")]), + model.Layer2Object(2, [model.DesignationCatalogObject(design="b")]), ] ) @@ -243,7 +259,7 @@ def test_get_orphaned_pgcs_returns_empty_when_layer1_present(self) -> None: self.common_repo.register_pgcs([100]) self.layer0_repo.upsert_pgc({"r1": 100}) self.layer1_repo.save_structured_data("designation.data", ["design"], ["r1"], [["x"]]) - self._save_layer2_data([model.Layer2CatalogObject(100, model.DesignationCatalogObject(design="x"))]) + self._save_layer2_data([model.Layer2Object(100, [model.DesignationCatalogObject(design="x")])]) orphaned = self.layer2_repo.get_orphaned_pgcs([model.RawCatalog.DESIGNATION]) @@ -257,8 +273,8 @@ def test_get_orphaned_pgcs_returns_only_pgcs_without_layer1_data(self) -> None: self.layer1_repo.save_structured_data("designation.data", ["design"], ["r1"], [["linked"]]) self._save_layer2_data( [ - model.Layer2CatalogObject(100, model.DesignationCatalogObject(design="linked")), - model.Layer2CatalogObject(200, model.DesignationCatalogObject(design="orphan")), + model.Layer2Object(100, [model.DesignationCatalogObject(design="linked")]), + model.Layer2Object(200, [model.DesignationCatalogObject(design="orphan")]), ] ) @@ -271,8 +287,8 @@ def test_remove_pgcs_removes_specified_pgcs(self) -> None: self.common_repo.register_pgcs([1, 2]) self._save_layer2_data( [ - model.Layer2CatalogObject(1, model.DesignationCatalogObject(design="d1")), - model.Layer2CatalogObject(2, model.DesignationCatalogObject(design="d2")), + model.Layer2Object(1, [model.DesignationCatalogObject(design="d1")]), + model.Layer2Object(2, [model.DesignationCatalogObject(design="d2")]), ] ) diff --git a/tests/lib/postgres.py b/tests/lib/postgres.py index 4b5fefe1..4e9ea222 100644 --- a/tests/lib/postgres.py +++ b/tests/lib/postgres.py @@ -135,7 +135,12 @@ def clear(self): "SELECT table_schema, table_name FROM information_schema.tables " "WHERE table_schema = 'layer2' OR table_schema = 'layer0'" ): - self.storage.exec(f"TRUNCATE {table['table_schema']}.{table['table_name']} CASCADE") + try: + self.storage.exec(f"TRUNCATE {table['table_schema']}.{table['table_name']} CASCADE") + except psycopg.Error as e: + logger.warning( + "truncate skipped", schema=table["table_schema"], table=table["table_name"], error=str(e) + ) self.storage.exec("INSERT INTO layer2.last_update (dt, catalog) VALUES (to_timestamp(0), 'designation')") self.storage.exec("INSERT INTO layer2.last_update (dt, catalog) VALUES (to_timestamp(0), 'icrs')") From c03cdf8f6931a5c677da05a899c90d58ac7c3bf1 Mon Sep 17 00:00:00 2001 From: kraysent Date: Thu, 12 Mar 2026 12:49:56 +0000 Subject: [PATCH 3/9] rename catalog object --- app/data/model/__init__.py | 4 +- app/data/model/layer2.py | 2 +- app/data/repositories/layer2/repository.py | 10 +-- app/domain/responders/fits_responder.py | 6 +- app/domain/responders/interface.py | 2 +- app/domain/responders/json_responder.py | 4 +- app/domain/responders/structured_responder.py | 2 +- tests/integration/layer2_import_test.py | 2 +- tests/integration/layer2_repository_test.py | 72 +++++++++---------- tests/unit/domain/fits_responder_test.py | 8 +-- 10 files changed, 56 insertions(+), 56 deletions(-) diff --git a/app/data/model/__init__.py b/app/data/model/__init__.py index 91166ff8..bec97406 100644 --- a/app/data/model/__init__.py +++ b/app/data/model/__init__.py @@ -3,7 +3,7 @@ from app.data.model.helpers import get_catalog_object_type from app.data.model.icrs import ICRSCatalogObject from app.data.model.interface import CatalogObject, MeasuredValue, RawCatalog, get_object -from app.data.model.layer2 import Layer2Object +from app.data.model.layer2 import Layer2CatalogObject from app.data.model.nature import NatureCatalogObject from app.data.model.records import ( CrossmatchRecordRow, @@ -40,7 +40,7 @@ "RedshiftRecord", "Record", "StructuredData", - "Layer2Object", + "Layer2CatalogObject", "TableRecord", "RawCatalog", "CatalogObject", diff --git a/app/data/model/layer2.py b/app/data/model/layer2.py index b6eecf2d..0e5b69ad 100644 --- a/app/data/model/layer2.py +++ b/app/data/model/layer2.py @@ -4,7 +4,7 @@ @dataclass -class Layer2Object: +class Layer2CatalogObject: pgc: int data: list[interface.CatalogObject] diff --git a/app/data/repositories/layer2/repository.py b/app/data/repositories/layer2/repository.py index 3b6d0571..4b03ecf5 100644 --- a/app/data/repositories/layer2/repository.py +++ b/app/data/repositories/layer2/repository.py @@ -163,14 +163,14 @@ def query_batch( search_params: Mapping[str, params.SearchParams], limit: int, offset: int, - ) -> dict[str, list[model.Layer2Object]]: + ) -> dict[str, list[model.Layer2CatalogObject]]: query, params = self._construct_batch_query(catalogs, search_types, search_params, limit, offset) records = self._storage.query(query, params=params) records_by_id = containers.group_by(records, key_func=lambda obj: str(obj["record_id"])) - result: dict[str, list[model.Layer2Object]] = {} + result: dict[str, list[model.Layer2CatalogObject]] = {} for record_id, records in records_by_id.items(): if record_id not in result: @@ -180,12 +180,12 @@ def query_batch( return result - def _group_by_pgc(self, objects: list[rows.DictRow]) -> list[model.Layer2Object]: + def _group_by_pgc(self, objects: list[rows.DictRow]) -> list[model.Layer2CatalogObject]: objects_by_pgc = containers.group_by(objects, key_func=lambda obj: int(obj["pgc"])) result = [] for pgc, pgc_objects in objects_by_pgc.items(): - layer2_obj = model.Layer2Object(pgc, []) + layer2_obj = model.Layer2CatalogObject(pgc, []) # TODO: what if for each pgc there are multiple rows? For example, if # the catalog does not have a UNIQUE constraint on pgc. @@ -282,7 +282,7 @@ def query( search_params: params.SearchParams, limit: int, offset: int, - ) -> list[model.Layer2Object]: + ) -> list[model.Layer2CatalogObject]: res = self.query_batch(catalogs, {search_params.name(): filters}, {"obj": search_params}, limit, offset) if "obj" not in res: diff --git a/app/domain/responders/fits_responder.py b/app/domain/responders/fits_responder.py index 80737d51..d12de1b1 100644 --- a/app/domain/responders/fits_responder.py +++ b/app/domain/responders/fits_responder.py @@ -7,7 +7,7 @@ from app.domain.responders import interface -def extract_object_data(objects: list[model.Layer2Object]) -> dict[str, np.ndarray]: +def extract_object_data(objects: list[model.Layer2CatalogObject]) -> dict[str, np.ndarray]: data_dict = {} for obj in objects: @@ -61,7 +61,7 @@ def extract_object_data(objects: list[model.Layer2Object]) -> dict[str, np.ndarr return data_dict -def create_fits_hdul(objects: list[model.Layer2Object]) -> fits.HDUList: +def create_fits_hdul(objects: list[model.Layer2CatalogObject]) -> fits.HDUList: data_dict = extract_object_data(objects) columns = [] @@ -87,7 +87,7 @@ def create_fits_hdul(objects: list[model.Layer2Object]) -> fits.HDUList: class FITSResponder(interface.ObjectResponder): - def build_response(self, objects: list[model.Layer2Object]) -> bytes: + def build_response(self, objects: list[model.Layer2CatalogObject]) -> bytes: hdul = create_fits_hdul(objects) with io.BytesIO() as f: diff --git a/app/domain/responders/interface.py b/app/domain/responders/interface.py index 7bebc46b..8da74aa1 100644 --- a/app/domain/responders/interface.py +++ b/app/domain/responders/interface.py @@ -10,5 +10,5 @@ class ObjectResponder(ABC): """ @abstractmethod - def build_response(self, objects: list[model.Layer2Object]) -> Any: + def build_response(self, objects: list[model.Layer2CatalogObject]) -> Any: pass diff --git a/app/domain/responders/json_responder.py b/app/domain/responders/json_responder.py index 0aeeb26c..033e969a 100644 --- a/app/domain/responders/json_responder.py +++ b/app/domain/responders/json_responder.py @@ -5,7 +5,7 @@ from app.presentation import dataapi -def objects_to_response(objects: list[model.Layer2Object]) -> list[dataapi.PGCObject]: +def objects_to_response(objects: list[model.Layer2CatalogObject]) -> list[dataapi.PGCObject]: response_objects = [] for obj in objects: catalog_data = {o.catalog().value: o.layer2_data() for o in obj.data} @@ -15,5 +15,5 @@ def objects_to_response(objects: list[model.Layer2Object]) -> list[dataapi.PGCOb class JSONResponder(interface.ObjectResponder): - def build_response(self, objects: list[model.Layer2Object]) -> Any: + def build_response(self, objects: list[model.Layer2CatalogObject]) -> Any: return objects_to_response(objects) diff --git a/app/domain/responders/structured_responder.py b/app/domain/responders/structured_responder.py index 9fd9ed23..4f879213 100644 --- a/app/domain/responders/structured_responder.py +++ b/app/domain/responders/structured_responder.py @@ -88,7 +88,7 @@ def _equatorial_to_galactic( return lon, lat, e_lon, e_lat - def build_response(self, objects: list[layer2.Layer2Object]) -> Any: + def build_response(self, objects: list[layer2.Layer2CatalogObject]) -> Any: catalog_schema = DATA_SCHEMA pgc_objects = [] diff --git a/tests/integration/layer2_import_test.py b/tests/integration/layer2_import_test.py index 8943d3cc..5ae44ffe 100644 --- a/tests/integration/layer2_import_test.py +++ b/tests/integration/layer2_import_test.py @@ -58,7 +58,7 @@ def test_import_two_catalogs(self): 10, 0, ) - expected = model.Layer2Object( + expected = model.Layer2CatalogObject( 1234, [model.ICRSCatalogObject(ra=12, e_ra=0.2, dec=13, e_dec=0.2), model.DesignationCatalogObject("test1")] ) diff --git a/tests/integration/layer2_repository_test.py b/tests/integration/layer2_repository_test.py index 62f2e2d4..59ec3e82 100644 --- a/tests/integration/layer2_repository_test.py +++ b/tests/integration/layer2_repository_test.py @@ -21,7 +21,7 @@ def setUpClass(cls) -> None: def tearDown(self): self.pg_storage.clear() - def _save_layer2_data(self, objects: list[model.Layer2Object]) -> None: + def _save_layer2_data(self, objects: list[model.Layer2CatalogObject]) -> None: by_table: dict[str, list[tuple[int, model.CatalogObject]]] = {} for obj in objects: for catalog_obj in obj.data: @@ -43,9 +43,9 @@ def _get_table(self, table_name: str) -> int: return table_resp.table_id def test_one_object(self): - objects: list[model.Layer2Object] = [ - model.Layer2Object(1, [model.DesignationCatalogObject(design="test")]), - model.Layer2Object(2, [model.DesignationCatalogObject(design="test2")]), + objects: list[model.Layer2CatalogObject] = [ + model.Layer2CatalogObject(1, [model.DesignationCatalogObject(design="test")]), + model.Layer2CatalogObject(2, [model.DesignationCatalogObject(design="test2")]), ] self.common_repo.register_pgcs([1, 2]) @@ -58,14 +58,14 @@ def test_one_object(self): 10, 0, ) - expected = [model.Layer2Object(1, [model.DesignationCatalogObject(design="test")])] + expected = [model.Layer2CatalogObject(1, [model.DesignationCatalogObject(design="test")])] self.assertEqual(actual, expected) def test_several_objects(self): - objects: list[model.Layer2Object] = [ - model.Layer2Object(1, [model.ICRSCatalogObject(ra=10, dec=10, e_ra=0.1, e_dec=0.1)]), - model.Layer2Object(2, [model.ICRSCatalogObject(ra=11, dec=11, e_ra=0.1, e_dec=0.1)]), + objects: list[model.Layer2CatalogObject] = [ + model.Layer2CatalogObject(1, [model.ICRSCatalogObject(ra=10, dec=10, e_ra=0.1, e_dec=0.1)]), + model.Layer2CatalogObject(2, [model.ICRSCatalogObject(ra=11, dec=11, e_ra=0.1, e_dec=0.1)]), ] self.common_repo.register_pgcs([1, 2]) @@ -79,16 +79,16 @@ def test_several_objects(self): 0, ) expected = [ - model.Layer2Object(1, [model.ICRSCatalogObject(ra=10, dec=10, e_ra=0.1, e_dec=0.1)]), - model.Layer2Object(2, [model.ICRSCatalogObject(ra=11, dec=11, e_ra=0.1, e_dec=0.1)]), + model.Layer2CatalogObject(1, [model.ICRSCatalogObject(ra=10, dec=10, e_ra=0.1, e_dec=0.1)]), + model.Layer2CatalogObject(2, [model.ICRSCatalogObject(ra=11, dec=11, e_ra=0.1, e_dec=0.1)]), ] self.assertEqual(actual, expected) def test_several_catalogs(self): objects = [ - model.Layer2Object(1, [model.ICRSCatalogObject(ra=10, dec=10, e_ra=0.1, e_dec=0.1)]), - model.Layer2Object( + model.Layer2CatalogObject(1, [model.ICRSCatalogObject(ra=10, dec=10, e_ra=0.1, e_dec=0.1)]), + model.Layer2CatalogObject( 2, [ model.ICRSCatalogObject(ra=11, dec=11, e_ra=0.1, e_dec=0.1), @@ -108,7 +108,7 @@ def test_several_catalogs(self): 0, ) expected = [ - model.Layer2Object( + model.Layer2CatalogObject( 2, [ model.ICRSCatalogObject(ra=11, dec=11, e_ra=0.1, e_dec=0.1), @@ -121,14 +121,14 @@ def test_several_catalogs(self): def test_several_filters(self): objects = [ - model.Layer2Object( + model.Layer2CatalogObject( 1, [ model.ICRSCatalogObject(ra=10, dec=10, e_ra=0.1, e_dec=0.1), model.DesignationCatalogObject(design="test"), ], ), - model.Layer2Object( + model.Layer2CatalogObject( 2, [ model.ICRSCatalogObject(ra=11, dec=11, e_ra=0.1, e_dec=0.1), @@ -158,7 +158,7 @@ def test_several_filters(self): ) expected = [ - model.Layer2Object( + model.Layer2CatalogObject( 2, [ model.ICRSCatalogObject(ra=11, dec=11, e_ra=0.1, e_dec=0.1), @@ -170,12 +170,12 @@ def test_several_filters(self): self.assertEqual(actual, expected) def test_pagination(self): - objects: list[model.Layer2Object] = [ - model.Layer2Object(1, [model.ICRSCatalogObject(ra=10, dec=10, e_ra=0.1, e_dec=0.1)]), - model.Layer2Object(2, [model.ICRSCatalogObject(ra=11, dec=11, e_ra=0.1, e_dec=0.1)]), - model.Layer2Object(3, [model.ICRSCatalogObject(ra=12, dec=12, e_ra=0.1, e_dec=0.1)]), - model.Layer2Object(4, [model.ICRSCatalogObject(ra=13, dec=13, e_ra=0.1, e_dec=0.1)]), - model.Layer2Object(5, [model.ICRSCatalogObject(ra=14, dec=14, e_ra=0.1, e_dec=0.1)]), + objects: list[model.Layer2CatalogObject] = [ + model.Layer2CatalogObject(1, [model.ICRSCatalogObject(ra=10, dec=10, e_ra=0.1, e_dec=0.1)]), + model.Layer2CatalogObject(2, [model.ICRSCatalogObject(ra=11, dec=11, e_ra=0.1, e_dec=0.1)]), + model.Layer2CatalogObject(3, [model.ICRSCatalogObject(ra=12, dec=12, e_ra=0.1, e_dec=0.1)]), + model.Layer2CatalogObject(4, [model.ICRSCatalogObject(ra=13, dec=13, e_ra=0.1, e_dec=0.1)]), + model.Layer2CatalogObject(5, [model.ICRSCatalogObject(ra=14, dec=14, e_ra=0.1, e_dec=0.1)]), ] self.common_repo.register_pgcs([1, 2, 3, 4, 5]) @@ -192,12 +192,12 @@ def test_pagination(self): self.assertEqual(len(actual), 2) def test_batch_query(self): - objects: list[model.Layer2Object] = [ - model.Layer2Object(1, [model.ICRSCatalogObject(ra=10, dec=10, e_ra=0.1, e_dec=0.1)]), - model.Layer2Object(2, [model.ICRSCatalogObject(ra=11, dec=11, e_ra=0.1, e_dec=0.1)]), - model.Layer2Object(3, [model.ICRSCatalogObject(ra=12, dec=12, e_ra=0.1, e_dec=0.1)]), - model.Layer2Object(4, [model.ICRSCatalogObject(ra=13, dec=13, e_ra=0.1, e_dec=0.1)]), - model.Layer2Object(5, [model.ICRSCatalogObject(ra=14, dec=14, e_ra=0.1, e_dec=0.1)]), + objects: list[model.Layer2CatalogObject] = [ + model.Layer2CatalogObject(1, [model.ICRSCatalogObject(ra=10, dec=10, e_ra=0.1, e_dec=0.1)]), + model.Layer2CatalogObject(2, [model.ICRSCatalogObject(ra=11, dec=11, e_ra=0.1, e_dec=0.1)]), + model.Layer2CatalogObject(3, [model.ICRSCatalogObject(ra=12, dec=12, e_ra=0.1, e_dec=0.1)]), + model.Layer2CatalogObject(4, [model.ICRSCatalogObject(ra=13, dec=13, e_ra=0.1, e_dec=0.1)]), + model.Layer2CatalogObject(5, [model.ICRSCatalogObject(ra=14, dec=14, e_ra=0.1, e_dec=0.1)]), ] self.common_repo.register_pgcs([1, 2, 3, 4, 5]) @@ -243,8 +243,8 @@ def test_get_orphaned_pgcs_returns_pgcs_without_layer1_data(self) -> None: self.common_repo.register_pgcs([1, 2]) self._save_layer2_data( [ - model.Layer2Object(1, [model.DesignationCatalogObject(design="a")]), - model.Layer2Object(2, [model.DesignationCatalogObject(design="b")]), + model.Layer2CatalogObject(1, [model.DesignationCatalogObject(design="a")]), + model.Layer2CatalogObject(2, [model.DesignationCatalogObject(design="b")]), ] ) @@ -259,7 +259,7 @@ def test_get_orphaned_pgcs_returns_empty_when_layer1_present(self) -> None: self.common_repo.register_pgcs([100]) self.layer0_repo.upsert_pgc({"r1": 100}) self.layer1_repo.save_structured_data("designation.data", ["design"], ["r1"], [["x"]]) - self._save_layer2_data([model.Layer2Object(100, [model.DesignationCatalogObject(design="x")])]) + self._save_layer2_data([model.Layer2CatalogObject(100, [model.DesignationCatalogObject(design="x")])]) orphaned = self.layer2_repo.get_orphaned_pgcs([model.RawCatalog.DESIGNATION]) @@ -273,8 +273,8 @@ def test_get_orphaned_pgcs_returns_only_pgcs_without_layer1_data(self) -> None: self.layer1_repo.save_structured_data("designation.data", ["design"], ["r1"], [["linked"]]) self._save_layer2_data( [ - model.Layer2Object(100, [model.DesignationCatalogObject(design="linked")]), - model.Layer2Object(200, [model.DesignationCatalogObject(design="orphan")]), + model.Layer2CatalogObject(100, [model.DesignationCatalogObject(design="linked")]), + model.Layer2CatalogObject(200, [model.DesignationCatalogObject(design="orphan")]), ] ) @@ -287,8 +287,8 @@ def test_remove_pgcs_removes_specified_pgcs(self) -> None: self.common_repo.register_pgcs([1, 2]) self._save_layer2_data( [ - model.Layer2Object(1, [model.DesignationCatalogObject(design="d1")]), - model.Layer2Object(2, [model.DesignationCatalogObject(design="d2")]), + model.Layer2CatalogObject(1, [model.DesignationCatalogObject(design="d1")]), + model.Layer2CatalogObject(2, [model.DesignationCatalogObject(design="d2")]), ] ) @@ -309,4 +309,4 @@ def test_remove_pgcs_removes_specified_pgcs(self) -> None: 10, 0, ) - self.assertEqual(actual, [model.Layer2Object(2, [model.DesignationCatalogObject(design="d2")])]) + self.assertEqual(actual, [model.Layer2CatalogObject(2, [model.DesignationCatalogObject(design="d2")])]) diff --git a/tests/unit/domain/fits_responder_test.py b/tests/unit/domain/fits_responder_test.py index cdc3c8b5..32ee1701 100644 --- a/tests/unit/domain/fits_responder_test.py +++ b/tests/unit/domain/fits_responder_test.py @@ -10,7 +10,7 @@ class ExtractObjectDataTest(unittest.TestCase): def setUp(self): self.objects = [ - model.Layer2Object( + model.Layer2CatalogObject( pgc=1234, data=[ model.DesignationCatalogObject(design="Galaxy1"), @@ -18,7 +18,7 @@ def setUp(self): model.RedshiftCatalogObject(cz=11.8, e_cz=0.2), ], ), - model.Layer2Object( + model.Layer2CatalogObject( pgc=5678, data=[ model.DesignationCatalogObject(design="Galaxy2"), @@ -68,7 +68,7 @@ def test_data_values(self): class CreateFitsHdulTest(unittest.TestCase): def setUp(self): self.objects = [ - model.Layer2Object( + model.Layer2CatalogObject( pgc=1234, data=[ model.DesignationCatalogObject(design="Galaxy1"), @@ -100,7 +100,7 @@ def test_table_columns(self): class FitsResponderTest(unittest.TestCase): def setUp(self): self.objects = [ - model.Layer2Object( + model.Layer2CatalogObject( pgc=1234, data=[ model.DesignationCatalogObject(design="Galaxy1"), From 282e24331294457566a1ed6fb8e4cd177fb67464 Mon Sep 17 00:00:00 2001 From: kraysent Date: Thu, 12 Mar 2026 12:58:13 +0000 Subject: [PATCH 4/9] add structured layer 2 objects and rename old methods --- app/data/model/layer2.py | 43 +++++++++++++++++++++ app/data/repositories/layer2/repository.py | 13 ++++--- app/domain/adminapi/crossmatch.py | 2 +- app/domain/dataapi/actions.py | 2 +- app/domain/dataapi/parameterized_query.py | 6 +-- tests/integration/layer2_import_test.py | 4 +- tests/integration/layer2_repository_test.py | 16 ++++---- 7 files changed, 66 insertions(+), 20 deletions(-) diff --git a/app/data/model/layer2.py b/app/data/model/layer2.py index 0e5b69ad..31ab0d55 100644 --- a/app/data/model/layer2.py +++ b/app/data/model/layer2.py @@ -14,3 +14,46 @@ def get[T](self, t: type[T]) -> T | None: return obj return None + + +@dataclass +class DesignationCatalog: + name: str + + +@dataclass +class ICRSCatalog: + ra: float + e_ra: float + dec: float + e_dec: float + + +@dataclass +class RedshiftCatalog: + cz: float + e_cz: float + + +@dataclass +class NatureCatalog: + type_name: str + + +@dataclass +class Catalogs: + """ + Dsscription of catalogs as they are stored on layer 2. To properly analyze them one probably needs + to read units from metadata of these tables. + """ + + designation: DesignationCatalog | None = None + icrs: ICRSCatalog | None = None + redshift: RedshiftCatalog | None = None + nature: NatureCatalog | None = None + + +@dataclass +class Layer2Object: + pgc: int + catalogs: Catalogs diff --git a/app/data/repositories/layer2/repository.py b/app/data/repositories/layer2/repository.py index 4b03ecf5..76ffbe76 100644 --- a/app/data/repositories/layer2/repository.py +++ b/app/data/repositories/layer2/repository.py @@ -7,6 +7,7 @@ from psycopg import rows from app.data import model +from app.data.model.layer2 import Layer2CatalogObject from app.data.repositories.layer2 import filters as repofilters from app.data.repositories.layer2 import params from app.lib import containers @@ -156,7 +157,7 @@ def _construct_batch_query( conditions=" OR ".join(condition_statements), ), params - def query_batch( + def query_catalogs_batch( self, catalogs: list[model.RawCatalog], search_types: Mapping[str, repofilters.Filter], @@ -220,13 +221,13 @@ def _group_by_pgc(self, objects: list[rows.DictRow]) -> list[model.Layer2Catalog return result - def query_pgc( + def query_catalogs_pgc( self, catalogs: list[model.RawCatalog], pgc_numbers: list[int], limit: int, offset: int = 0, - ): + ) -> list[Layer2CatalogObject]: if not catalogs: return [] @@ -275,7 +276,7 @@ def query_pgc( return self._group_by_pgc(objects) - def query( + def query_catalogs( self, catalogs: list[model.RawCatalog], filters: repofilters.Filter, @@ -283,7 +284,9 @@ def query( limit: int, offset: int, ) -> list[model.Layer2CatalogObject]: - res = self.query_batch(catalogs, {search_params.name(): filters}, {"obj": search_params}, limit, offset) + res = self.query_catalogs_batch( + catalogs, {search_params.name(): filters}, {"obj": search_params}, limit, offset + ) if "obj" not in res: return [] diff --git a/app/domain/adminapi/crossmatch.py b/app/domain/adminapi/crossmatch.py index 82afa79d..8d96e46b 100644 --- a/app/domain/adminapi/crossmatch.py +++ b/app/domain/adminapi/crossmatch.py @@ -210,7 +210,7 @@ def get_record_crossmatch(self, r: adminapi.GetRecordCrossmatchRequest) -> admin if len(candidate_pgcs) == 0: return response - layer2_objects = self.layer2_repo.query_pgc( + layer2_objects = self.layer2_repo.query_catalogs_pgc( catalogs=[model.RawCatalog.ICRS, model.RawCatalog.DESIGNATION, model.RawCatalog.REDSHIFT], pgc_numbers=list(candidate_pgcs), limit=len(candidate_pgcs), diff --git a/app/domain/dataapi/actions.py b/app/domain/dataapi/actions.py index 68f6bb34..f1b9e58e 100644 --- a/app/domain/dataapi/actions.py +++ b/app/domain/dataapi/actions.py @@ -28,7 +28,7 @@ def __init__( def query(self, query: dataapi.QueryRequest) -> dataapi.QueryResponse: filters, search_params = search_parsers.query_to_filters(query.q, search_parsers.DEFAULT_PARSERS) - objects = self.layer2_repo.query( + objects = self.layer2_repo.query_catalogs( ENABLED_CATALOGS, filters, search_params, diff --git a/app/domain/dataapi/parameterized_query.py b/app/domain/dataapi/parameterized_query.py index 8867bc0a..ab0ebb7e 100644 --- a/app/domain/dataapi/parameterized_query.py +++ b/app/domain/dataapi/parameterized_query.py @@ -40,7 +40,7 @@ def _build_filters_and_params( def query_fits(self, query: dataapi.FITSRequest) -> bytes: filters, search_params = self._build_filters_and_params(query) - objects = self.layer2_repo.query( + objects = self.layer2_repo.query_catalogs( self.enabled_catalogs, filters, search_params, @@ -55,7 +55,7 @@ def query_simple(self, query: dataapi.QuerySimpleRequest) -> dataapi.QuerySimple filters, search_params = self._build_filters_and_params(query) if not query.pgcs: - objects = self.layer2_repo.query( + objects = self.layer2_repo.query_catalogs( self.enabled_catalogs, filters, search_params, @@ -63,7 +63,7 @@ def query_simple(self, query: dataapi.QuerySimpleRequest) -> dataapi.QuerySimple query.page, ) else: - objects = self.layer2_repo.query_pgc( + objects = self.layer2_repo.query_catalogs_pgc( self.enabled_catalogs, query.pgcs, query.page_size, diff --git a/tests/integration/layer2_import_test.py b/tests/integration/layer2_import_test.py index 5ae44ffe..05ca20c3 100644 --- a/tests/integration/layer2_import_test.py +++ b/tests/integration/layer2_import_test.py @@ -51,7 +51,7 @@ def test_import_two_catalogs(self): self.task.run() - actual = self.layer2_repo.query( + actual = self.layer2_repo.query_catalogs( [model.RawCatalog.ICRS, model.RawCatalog.DESIGNATION], layer2.PGCOneOfFilter([1234]), layer2.CombinedSearchParams([]), @@ -82,7 +82,7 @@ def test_updated_objects(self): new_last_update_dt = self.layer2_repo.get_last_update_time(model.RawCatalog.DESIGNATION) self.assertGreater(new_last_update_dt, last_update_dt) - actual = self.layer2_repo.query( + actual = self.layer2_repo.query_catalogs( [model.RawCatalog.DESIGNATION], layer2.PGCOneOfFilter([1234]), layer2.CombinedSearchParams([]), diff --git a/tests/integration/layer2_repository_test.py b/tests/integration/layer2_repository_test.py index 59ec3e82..877d338a 100644 --- a/tests/integration/layer2_repository_test.py +++ b/tests/integration/layer2_repository_test.py @@ -51,7 +51,7 @@ def test_one_object(self): self.common_repo.register_pgcs([1, 2]) self._save_layer2_data(objects) - actual = self.layer2_repo.query( + actual = self.layer2_repo.query_catalogs( [model.RawCatalog.DESIGNATION], layer2.DesignationEqualsFilter("test"), layer2.CombinedSearchParams([]), @@ -71,7 +71,7 @@ def test_several_objects(self): self.common_repo.register_pgcs([1, 2]) self._save_layer2_data(objects) - actual = self.layer2_repo.query( + actual = self.layer2_repo.query_catalogs( [model.RawCatalog.ICRS], layer2.ICRSCoordinatesInRadiusFilter(10), layer2.ICRSSearchParams(12, 12), @@ -100,7 +100,7 @@ def test_several_catalogs(self): self.common_repo.register_pgcs([1, 2]) self._save_layer2_data(objects) - actual = self.layer2_repo.query( + actual = self.layer2_repo.query_catalogs( [model.RawCatalog.ICRS, model.RawCatalog.DESIGNATION], layer2.DesignationEqualsFilter("test2"), layer2.CombinedSearchParams([]), @@ -140,7 +140,7 @@ def test_several_filters(self): self.common_repo.register_pgcs([1, 2]) self._save_layer2_data(objects) - actual = self.layer2_repo.query( + actual = self.layer2_repo.query_catalogs( [model.RawCatalog.ICRS, model.RawCatalog.DESIGNATION], layer2.AndFilter( [ @@ -181,7 +181,7 @@ def test_pagination(self): self.common_repo.register_pgcs([1, 2, 3, 4, 5]) self._save_layer2_data(objects) - actual = self.layer2_repo.query( + actual = self.layer2_repo.query_catalogs( [model.RawCatalog.ICRS], layer2.ICRSCoordinatesInRadiusFilter(10), layer2.ICRSSearchParams(12, 12), @@ -203,7 +203,7 @@ def test_batch_query(self): self.common_repo.register_pgcs([1, 2, 3, 4, 5]) self._save_layer2_data(objects) - actual = self.layer2_repo.query_batch( + actual = self.layer2_repo.query_catalogs_batch( [model.RawCatalog.ICRS], {"icrs": layer2.ICRSCoordinatesInRadiusFilter(10)}, { @@ -294,7 +294,7 @@ def test_remove_pgcs_removes_specified_pgcs(self) -> None: self.layer2_repo.remove_pgcs([model.RawCatalog.DESIGNATION], [1]) - actual = self.layer2_repo.query( + actual = self.layer2_repo.query_catalogs( [model.RawCatalog.DESIGNATION], layer2.DesignationEqualsFilter("d1"), layer2.CombinedSearchParams([]), @@ -302,7 +302,7 @@ def test_remove_pgcs_removes_specified_pgcs(self) -> None: 0, ) self.assertEqual(actual, []) - actual = self.layer2_repo.query( + actual = self.layer2_repo.query_catalogs( [model.RawCatalog.DESIGNATION], layer2.DesignationEqualsFilter("d2"), layer2.CombinedSearchParams([]), From 6ae378cb9b479afee65a86f34df78addfde43384 Mon Sep 17 00:00:00 2001 From: kraysent Date: Thu, 12 Mar 2026 13:24:33 +0000 Subject: [PATCH 5/9] add alternative querying --- app/data/model/__init__.py | 3 +- app/data/repositories/layer2/repository.py | 123 +++++++++++++++++- app/domain/dataapi/actions.py | 2 +- app/domain/dataapi/parameterized_query.py | 4 +- app/domain/responders/fits_responder.py | 2 +- app/domain/responders/interface.py | 2 +- app/domain/responders/structured_responder.py | 2 +- tests/unit/domain/fits_responder_test.py | 2 +- 8 files changed, 130 insertions(+), 10 deletions(-) diff --git a/app/data/model/__init__.py b/app/data/model/__init__.py index bec97406..be6aba96 100644 --- a/app/data/model/__init__.py +++ b/app/data/model/__init__.py @@ -3,7 +3,7 @@ from app.data.model.helpers import get_catalog_object_type from app.data.model.icrs import ICRSCatalogObject from app.data.model.interface import CatalogObject, MeasuredValue, RawCatalog, get_object -from app.data.model.layer2 import Layer2CatalogObject +from app.data.model.layer2 import Layer2CatalogObject, Layer2Object from app.data.model.nature import NatureCatalogObject from app.data.model.records import ( CrossmatchRecordRow, @@ -41,6 +41,7 @@ "Record", "StructuredData", "Layer2CatalogObject", + "Layer2Object", "TableRecord", "RawCatalog", "CatalogObject", diff --git a/app/data/repositories/layer2/repository.py b/app/data/repositories/layer2/repository.py index 76ffbe76..98b4d409 100644 --- a/app/data/repositories/layer2/repository.py +++ b/app/data/repositories/layer2/repository.py @@ -7,10 +7,11 @@ from psycopg import rows from app.data import model -from app.data.model.layer2 import Layer2CatalogObject +from app.data.model import Layer2CatalogObject, Layer2Object +from app.data.model import layer2 as layer2_model from app.data.repositories.layer2 import filters as repofilters from app.data.repositories.layer2 import params -from app.lib import containers +from app.lib import concurrency, containers from app.lib.storage import postgres catalogs = [ @@ -221,6 +222,124 @@ def _group_by_pgc(self, objects: list[rows.DictRow]) -> list[model.Layer2Catalog return result + def _query_designations(self, pgcs: list[int]) -> dict[int, layer2_model.DesignationCatalog]: + if not pgcs: + return {} + rows = self._storage.query( + "SELECT pgc, design FROM layer2.designation WHERE pgc = ANY(%s) ORDER BY pgc", + params=[pgcs], + ) + return {int(row["pgc"]): layer2_model.DesignationCatalog(name=str(row["design"])) for row in rows} + + def _query_icrs(self, pgcs: list[int]) -> dict[int, layer2_model.ICRSCatalog]: + if not pgcs: + return {} + rows = self._storage.query( + "SELECT pgc, ra, e_ra, dec, e_dec FROM layer2.icrs WHERE pgc = ANY(%s) ORDER BY pgc", + params=[pgcs], + ) + result: dict[int, layer2_model.ICRSCatalog] = {} + for row in rows: + if all(row.get(k) is not None for k in ("ra", "e_ra", "dec", "e_dec")): + result[int(row["pgc"])] = layer2_model.ICRSCatalog( + ra=float(row["ra"]), + e_ra=float(row["e_ra"]), + dec=float(row["dec"]), + e_dec=float(row["e_dec"]), + ) + return result + + def _query_redshift(self, pgcs: list[int]) -> dict[int, layer2_model.RedshiftCatalog]: + if not pgcs: + return {} + rows = self._storage.query( + "SELECT pgc, cz, e_cz FROM layer2.cz WHERE pgc = ANY(%s) ORDER BY pgc", + params=[pgcs], + ) + return { + int(row["pgc"]): layer2_model.RedshiftCatalog(cz=float(row["cz"]), e_cz=float(row["e_cz"])) + for row in rows + if row.get("cz") is not None and row.get("e_cz") is not None + } + + def _query_nature(self, pgcs: list[int]) -> dict[int, layer2_model.NatureCatalog]: + if not pgcs: + return {} + rows = self._storage.query( + "SELECT pgc, type_name FROM layer2.nature WHERE pgc = ANY(%s) ORDER BY pgc", + params=[pgcs], + ) + return { + int(row["pgc"]): layer2_model.NatureCatalog(type_name=str(row["type_name"])) + for row in rows + if row.get("type_name") is not None + } + + def query_pgc( + self, + catalogs: list[model.RawCatalog], + pgc_numbers: list[int], + limit: int, + offset: int = 0, + ) -> list[Layer2Object]: + if not catalogs or not pgc_numbers: + return [] + + pgcs_page = sorted(pgc_numbers)[offset : offset + limit] + if not pgcs_page: + return [] + + errgr = concurrency.ErrorGroup() + designation_task: concurrency.TaskResult[dict[int, layer2_model.DesignationCatalog]] | None = None + icrs_task: concurrency.TaskResult[dict[int, layer2_model.ICRSCatalog]] | None = None + redshift_task: concurrency.TaskResult[dict[int, layer2_model.RedshiftCatalog]] | None = None + nature_task: concurrency.TaskResult[dict[int, layer2_model.NatureCatalog]] | None = None + + if model.RawCatalog.DESIGNATION in catalogs: + designation_task = errgr.run(self._query_designations, pgcs_page) + if model.RawCatalog.ICRS in catalogs: + icrs_task = errgr.run(self._query_icrs, pgcs_page) + if model.RawCatalog.REDSHIFT in catalogs: + redshift_task = errgr.run(self._query_redshift, pgcs_page) + if model.RawCatalog.NATURE in catalogs: + nature_task = errgr.run(self._query_nature, pgcs_page) + + errgr.wait() + + designation_map = designation_task.result() if designation_task is not None else {} + icrs_map = icrs_task.result() if icrs_task is not None else {} + redshift_map = redshift_task.result() if redshift_task is not None else {} + nature_map = nature_task.result() if nature_task is not None else {} + + return [ + self._layer2_object_from_maps(pgc, catalogs, designation_map, icrs_map, redshift_map, nature_map) + for pgc in pgcs_page + ] + + def _layer2_object_from_maps( + self, + pgc: int, + catalogs: list[model.RawCatalog], + designation_map: dict[int, layer2_model.DesignationCatalog], + icrs_map: dict[int, layer2_model.ICRSCatalog], + redshift_map: dict[int, layer2_model.RedshiftCatalog], + nature_map: dict[int, layer2_model.NatureCatalog], + ) -> Layer2Object: + designation = designation_map.get(pgc) if model.RawCatalog.DESIGNATION in catalogs else None + icrs = icrs_map.get(pgc) if model.RawCatalog.ICRS in catalogs else None + redshift = redshift_map.get(pgc) if model.RawCatalog.REDSHIFT in catalogs else None + nature = nature_map.get(pgc) if model.RawCatalog.NATURE in catalogs else None + + return Layer2Object( + pgc=pgc, + catalogs=layer2_model.Catalogs( + designation=designation, + icrs=icrs, + redshift=redshift, + nature=nature, + ), + ) + def query_catalogs_pgc( self, catalogs: list[model.RawCatalog], diff --git a/app/domain/dataapi/actions.py b/app/domain/dataapi/actions.py index f1b9e58e..c6250a46 100644 --- a/app/domain/dataapi/actions.py +++ b/app/domain/dataapi/actions.py @@ -36,7 +36,7 @@ def query(self, query: dataapi.QueryRequest) -> dataapi.QueryResponse: query.page, ) responder = responders.StructuredResponder(self.catalog_cfg) - pgc_objects = responder.build_response(objects).objects + pgc_objects = responder.build_response_from_catalog(objects).objects return dataapi.QueryResponse(objects=pgc_objects) def query_fits(self, query: dataapi.FITSRequest) -> bytes: diff --git a/app/domain/dataapi/parameterized_query.py b/app/domain/dataapi/parameterized_query.py index ab0ebb7e..f4e59239 100644 --- a/app/domain/dataapi/parameterized_query.py +++ b/app/domain/dataapi/parameterized_query.py @@ -49,7 +49,7 @@ def query_fits(self, query: dataapi.FITSRequest) -> bytes: ) responder = responders.FITSResponder() - return responder.build_response(objects) + return responder.build_response_from_catalog(objects) def query_simple(self, query: dataapi.QuerySimpleRequest) -> dataapi.QuerySimpleResponse: filters, search_params = self._build_filters_and_params(query) @@ -71,4 +71,4 @@ def query_simple(self, query: dataapi.QuerySimpleRequest) -> dataapi.QuerySimple ) responder = responders.StructuredResponder(self.catalog_config) - return responder.build_response(objects) + return responder.build_response_from_catalog(objects) diff --git a/app/domain/responders/fits_responder.py b/app/domain/responders/fits_responder.py index d12de1b1..abc3d015 100644 --- a/app/domain/responders/fits_responder.py +++ b/app/domain/responders/fits_responder.py @@ -87,7 +87,7 @@ def create_fits_hdul(objects: list[model.Layer2CatalogObject]) -> fits.HDUList: class FITSResponder(interface.ObjectResponder): - def build_response(self, objects: list[model.Layer2CatalogObject]) -> bytes: + def build_response_from_catalog(self, objects: list[model.Layer2CatalogObject]) -> bytes: hdul = create_fits_hdul(objects) with io.BytesIO() as f: diff --git a/app/domain/responders/interface.py b/app/domain/responders/interface.py index 8da74aa1..faf01356 100644 --- a/app/domain/responders/interface.py +++ b/app/domain/responders/interface.py @@ -10,5 +10,5 @@ class ObjectResponder(ABC): """ @abstractmethod - def build_response(self, objects: list[model.Layer2CatalogObject]) -> Any: + def build_response_from_catalog(self, objects: list[model.Layer2CatalogObject]) -> Any: pass diff --git a/app/domain/responders/structured_responder.py b/app/domain/responders/structured_responder.py index 4f879213..d3ee46bd 100644 --- a/app/domain/responders/structured_responder.py +++ b/app/domain/responders/structured_responder.py @@ -88,7 +88,7 @@ def _equatorial_to_galactic( return lon, lat, e_lon, e_lat - def build_response(self, objects: list[layer2.Layer2CatalogObject]) -> Any: + def build_response_from_catalog(self, objects: list[layer2.Layer2CatalogObject]) -> Any: catalog_schema = DATA_SCHEMA pgc_objects = [] diff --git a/tests/unit/domain/fits_responder_test.py b/tests/unit/domain/fits_responder_test.py index 32ee1701..8a09236a 100644 --- a/tests/unit/domain/fits_responder_test.py +++ b/tests/unit/domain/fits_responder_test.py @@ -111,7 +111,7 @@ def setUp(self): self.responder = fits_responder.FITSResponder() def test_build_response(self): - fits_data = self.responder.build_response(self.objects) + fits_data = self.responder.build_response_from_catalog(self.objects) self.assertIsInstance(fits_data, bytes) self.assertGreater(len(fits_data), 0) From f508fff31140283abf2eeeb1dd48fe90556ecf88 Mon Sep 17 00:00:00 2001 From: kraysent Date: Thu, 12 Mar 2026 13:37:23 +0000 Subject: [PATCH 6/9] add pgc responder method --- app/domain/dataapi/parameterized_query.py | 18 +++--- app/domain/responders/structured_responder.py | 63 +++++++++++++++++++ 2 files changed, 72 insertions(+), 9 deletions(-) diff --git a/app/domain/dataapi/parameterized_query.py b/app/domain/dataapi/parameterized_query.py index f4e59239..6b4fb222 100644 --- a/app/domain/dataapi/parameterized_query.py +++ b/app/domain/dataapi/parameterized_query.py @@ -54,6 +54,7 @@ def query_fits(self, query: dataapi.FITSRequest) -> bytes: def query_simple(self, query: dataapi.QuerySimpleRequest) -> dataapi.QuerySimpleResponse: filters, search_params = self._build_filters_and_params(query) + responder = responders.StructuredResponder(self.catalog_config) if not query.pgcs: objects = self.layer2_repo.query_catalogs( self.enabled_catalogs, @@ -62,13 +63,12 @@ def query_simple(self, query: dataapi.QuerySimpleRequest) -> dataapi.QuerySimple query.page_size, query.page, ) - else: - objects = self.layer2_repo.query_catalogs_pgc( - self.enabled_catalogs, - query.pgcs, - query.page_size, - query.page, - ) + return responder.build_response_from_catalog(objects) - responder = responders.StructuredResponder(self.catalog_config) - return responder.build_response_from_catalog(objects) + objects = self.layer2_repo.query_pgc( + self.enabled_catalogs, + query.pgcs, + query.page_size, + query.page, + ) + return responder.build_response(objects) diff --git a/app/domain/responders/structured_responder.py b/app/domain/responders/structured_responder.py index d3ee46bd..d0e2c905 100644 --- a/app/domain/responders/structured_responder.py +++ b/app/domain/responders/structured_responder.py @@ -149,3 +149,66 @@ def build_response_from_catalog(self, objects: list[layer2.Layer2CatalogObject]) pgc_objects.append(pgc_object) return dataapi.QuerySimpleResponse(objects=pgc_objects, schema=catalog_schema) + + def build_response(self, objects: list[layer2.Layer2Object]) -> Any: + catalog_schema = DATA_SCHEMA + pgc_objects = [] + + for obj in objects: + catalogs = dataapi.Catalogs() + + if obj.catalogs.designation is not None: + catalogs.designation = dataapi.Designation(name=obj.catalogs.designation.name) + + icrs = obj.catalogs.icrs + if icrs is not None: + ra, dec, e_ra, e_dec = self._equatorial(icrs.ra, icrs.dec, icrs.e_ra, icrs.e_dec) + lon, lat, e_lon, e_lat = self._equatorial_to_galactic(icrs.ra, icrs.dec, icrs.e_ra, icrs.e_dec) + + catalogs.coordinates = dataapi.Coordinates( + equatorial=dataapi.EquatorialCoordinates(ra=ra, dec=dec, e_ra=e_ra, e_dec=e_dec), + galactic=dataapi.GalacticCoordinates(lon=lon, lat=lat, e_lon=e_lon, e_lat=e_lat), + ) + + if obj.catalogs.redshift is not None: + redshift = obj.catalogs.redshift + catalogs.redshift = dataapi.Redshift( + z=self._heliocentric_to_redshift(redshift.cz), + e_z=self._heliocentric_to_redshift(redshift.e_cz), + ) + + if obj.catalogs.nature is not None: + catalogs.nature = dataapi.Nature(type_name=obj.catalogs.nature.type_name) + + if icrs is not None and obj.catalogs.redshift is not None: + redshift = obj.catalogs.redshift + catalogs.velocity = {} + + for key, apex in self.config.velocity.apexes.items(): + vel_wr_apex, vel_wr_apex_err = astronomy.velocity_wr_apex( + vel=redshift.cz * u.Unit("m/s"), + lon=lon * u.Unit("deg"), + lat=lat * u.Unit("deg"), + vel_apex=apex.vel.value * u.Unit("km/s"), + lon_apex=apex.lon.value * u.Unit("deg"), + lat_apex=apex.lat.value * u.Unit("deg"), + vel_err=redshift.e_cz * u.Unit("m/s"), + lon_err=e_lon * u.Unit("arcsec"), + lat_err=e_lat * u.Unit("arcsec"), + vel_apex_err=apex.vel.error * u.Unit("km/s"), + lon_apex_err=apex.lon.error * u.Unit("arcsec"), + lat_apex_err=apex.lat.error * u.Unit("arcsec"), + ) + + schema = VELOCITY_SCHEMA + + catalog_schema.units.velocity[key] = schema + catalogs.velocity[key] = dataapi.AbsoluteVelocity( + v=vel_wr_apex.to(u.Unit(schema.v)).value, + e_v=vel_wr_apex_err.to(u.Unit(schema.e_v)).value, + ) + + pgc_object = dataapi.PGCObject(pgc=obj.pgc, catalogs=catalogs) + pgc_objects.append(pgc_object) + + return dataapi.QuerySimpleResponse(objects=pgc_objects, schema=catalog_schema) From b0e10668e3a383e6187a4889506ce611015b7873 Mon Sep 17 00:00:00 2001 From: kraysent Date: Thu, 12 Mar 2026 13:39:48 +0000 Subject: [PATCH 7/9] increase connection pool size --- app/lib/storage/postgres/postgres_storage.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/lib/storage/postgres/postgres_storage.py b/app/lib/storage/postgres/postgres_storage.py index 75ee93b8..4058705c 100644 --- a/app/lib/storage/postgres/postgres_storage.py +++ b/app/lib/storage/postgres/postgres_storage.py @@ -68,8 +68,8 @@ def connect(self) -> None: self._logger.debug("connecting to Postgres", endpoint=self._config.endpoint, port=self._config.port) self._pool = ConnectionPool( self._config.get_dsn(), - min_size=2, - max_size=10, + min_size=10, + max_size=30, kwargs={"row_factory": rows.dict_row, "autocommit": True}, configure=self._configure_connection, ) From 5ecb48bf01aba5fc3b7f3a1edd5027eb180b613d Mon Sep 17 00:00:00 2001 From: kraysent Date: Thu, 12 Mar 2026 13:43:23 +0000 Subject: [PATCH 8/9] use pydantic to disallow other filters --- app/domain/dataapi/parameterized_query.py | 20 ++++++++++---------- app/presentation/dataapi/interface.py | 12 +++++++++++- 2 files changed, 21 insertions(+), 11 deletions(-) diff --git a/app/domain/dataapi/parameterized_query.py b/app/domain/dataapi/parameterized_query.py index 6b4fb222..98de16d7 100644 --- a/app/domain/dataapi/parameterized_query.py +++ b/app/domain/dataapi/parameterized_query.py @@ -52,23 +52,23 @@ def query_fits(self, query: dataapi.FITSRequest) -> bytes: return responder.build_response_from_catalog(objects) def query_simple(self, query: dataapi.QuerySimpleRequest) -> dataapi.QuerySimpleResponse: - filters, search_params = self._build_filters_and_params(query) - responder = responders.StructuredResponder(self.catalog_config) - if not query.pgcs: - objects = self.layer2_repo.query_catalogs( + if query.pgcs: + objects = self.layer2_repo.query_pgc( self.enabled_catalogs, - filters, - search_params, + query.pgcs, query.page_size, query.page, ) - return responder.build_response_from_catalog(objects) + return responder.build_response(objects) + + filters, search_params = self._build_filters_and_params(query) - objects = self.layer2_repo.query_pgc( + objects = self.layer2_repo.query_catalogs( self.enabled_catalogs, - query.pgcs, + filters, + search_params, query.page_size, query.page, ) - return responder.build_response(objects) + return responder.build_response_from_catalog(objects) diff --git a/app/presentation/dataapi/interface.py b/app/presentation/dataapi/interface.py index 222f9e7f..2940ded1 100644 --- a/app/presentation/dataapi/interface.py +++ b/app/presentation/dataapi/interface.py @@ -89,7 +89,7 @@ class Schema(pydantic.BaseModel): class QuerySimpleRequest(pydantic.BaseModel): pgcs: list[int] | None = pydantic.Field( default=None, - description="List of PGC numbers. If specified, all other filters will be ignored", + description="List of PGC numbers. If specified, no other filters are allowed", ) ra: float | None = pydantic.Field( default=None, @@ -124,6 +124,16 @@ class QuerySimpleRequest(pydantic.BaseModel): description="Page number", ) + @pydantic.model_validator(mode="after") + def _pgcs_exclusive_with_filters(self) -> "QuerySimpleRequest": + if self.pgcs: + filters = [self.ra, self.dec, self.radius, self.name, self.cz, self.cz_err_percent] + if any(f is not None for f in filters): + raise ValueError( + "When pgcs is specified, no other filters are allowed" + ) + return self + class QuerySimpleResponse(pydantic.BaseModel): objects: list[PGCObject] From ed8576d477abd0e9d6a428fc6dfd3c0c03ace695 Mon Sep 17 00:00:00 2001 From: kraysent Date: Thu, 12 Mar 2026 14:08:15 +0000 Subject: [PATCH 9/9] add additional catalogs query for pgc --- app/data/model/interface.py | 1 + app/data/model/layer2.py | 20 ++++++++ app/data/repositories/layer2/repository.py | 51 ++++++++++++++++++- app/domain/dataapi/parameterized_query.py | 10 +++- app/domain/responders/structured_responder.py | 14 +++++ app/presentation/dataapi/interface.py | 17 +++++-- .../V030__additional_designations.sql | 8 ++- 7 files changed, 114 insertions(+), 7 deletions(-) diff --git a/app/data/model/interface.py b/app/data/model/interface.py index f8df8c24..010a4222 100644 --- a/app/data/model/interface.py +++ b/app/data/model/interface.py @@ -32,6 +32,7 @@ class RawCatalog(enum.Enum): ICRS = "icrs" DESIGNATION = "designation" + ADDITIONAL_DESIGNATIONS = "additional_designations" REDSHIFT = "redshift" NATURE = "nature" diff --git a/app/data/model/layer2.py b/app/data/model/layer2.py index 31ab0d55..bcf4473f 100644 --- a/app/data/model/layer2.py +++ b/app/data/model/layer2.py @@ -40,6 +40,25 @@ class NatureCatalog: type_name: str +@dataclass +class Source: + bibcode: str + title: str + authors: list[str] + year: int + + +@dataclass +class AdditionalDesignation: + name: str + source: Source + + +@dataclass +class AdditionalDesignationsCatalog: + names: list[AdditionalDesignation] + + @dataclass class Catalogs: """ @@ -48,6 +67,7 @@ class Catalogs: """ designation: DesignationCatalog | None = None + additional_designations: AdditionalDesignationsCatalog | None = None icrs: ICRSCatalog | None = None redshift: RedshiftCatalog | None = None nature: NatureCatalog | None = None diff --git a/app/data/repositories/layer2/repository.py b/app/data/repositories/layer2/repository.py index 98b4d409..0e0b4030 100644 --- a/app/data/repositories/layer2/repository.py +++ b/app/data/repositories/layer2/repository.py @@ -275,6 +275,34 @@ def _query_nature(self, pgcs: list[int]) -> dict[int, layer2_model.NatureCatalog if row.get("type_name") is not None } + def _query_additional_designations(self, pgcs: list[int]) -> dict[int, layer2_model.AdditionalDesignationsCatalog]: + if not pgcs: + return {} + rows = self._storage.query( + "SELECT pgc, design, code, year, author, title FROM layer2.designations " + "WHERE pgc = ANY(%s) ORDER BY pgc, design", + params=[pgcs], + ) + result: dict[int, list[layer2_model.AdditionalDesignation]] = {} + for row in rows: + pgc = int(row["pgc"]) + author_val = row.get("author") + authors = ( + author_val if isinstance(author_val, list) else [str(author_val)] if author_val is not None else [] + ) + source = layer2_model.Source( + bibcode=str(row["code"]) if row.get("code") is not None else "", + title=str(row["title"]) if row.get("title") is not None else "", + authors=authors, + year=int(row["year"]) if row.get("year") is not None else 0, + ) + ad = layer2_model.AdditionalDesignation( + name=str(row["design"]) if row.get("design") is not None else "", + source=source, + ) + result.setdefault(pgc, []).append(ad) + return {pgc: layer2_model.AdditionalDesignationsCatalog(names=names) for pgc, names in result.items()} + def query_pgc( self, catalogs: list[model.RawCatalog], @@ -291,12 +319,17 @@ def query_pgc( errgr = concurrency.ErrorGroup() designation_task: concurrency.TaskResult[dict[int, layer2_model.DesignationCatalog]] | None = None + additional_designations_task: ( + concurrency.TaskResult[dict[int, layer2_model.AdditionalDesignationsCatalog]] | None + ) = None icrs_task: concurrency.TaskResult[dict[int, layer2_model.ICRSCatalog]] | None = None redshift_task: concurrency.TaskResult[dict[int, layer2_model.RedshiftCatalog]] | None = None nature_task: concurrency.TaskResult[dict[int, layer2_model.NatureCatalog]] | None = None if model.RawCatalog.DESIGNATION in catalogs: designation_task = errgr.run(self._query_designations, pgcs_page) + if model.RawCatalog.ADDITIONAL_DESIGNATIONS in catalogs: + additional_designations_task = errgr.run(self._query_additional_designations, pgcs_page) if model.RawCatalog.ICRS in catalogs: icrs_task = errgr.run(self._query_icrs, pgcs_page) if model.RawCatalog.REDSHIFT in catalogs: @@ -307,12 +340,23 @@ def query_pgc( errgr.wait() designation_map = designation_task.result() if designation_task is not None else {} + additional_designations_map = ( + additional_designations_task.result() if additional_designations_task is not None else {} + ) icrs_map = icrs_task.result() if icrs_task is not None else {} redshift_map = redshift_task.result() if redshift_task is not None else {} nature_map = nature_task.result() if nature_task is not None else {} return [ - self._layer2_object_from_maps(pgc, catalogs, designation_map, icrs_map, redshift_map, nature_map) + self._layer2_object_from_maps( + pgc, + catalogs, + designation_map, + additional_designations_map, + icrs_map, + redshift_map, + nature_map, + ) for pgc in pgcs_page ] @@ -321,11 +365,15 @@ def _layer2_object_from_maps( pgc: int, catalogs: list[model.RawCatalog], designation_map: dict[int, layer2_model.DesignationCatalog], + additional_designations_map: dict[int, layer2_model.AdditionalDesignationsCatalog], icrs_map: dict[int, layer2_model.ICRSCatalog], redshift_map: dict[int, layer2_model.RedshiftCatalog], nature_map: dict[int, layer2_model.NatureCatalog], ) -> Layer2Object: designation = designation_map.get(pgc) if model.RawCatalog.DESIGNATION in catalogs else None + additional_designations = ( + additional_designations_map.get(pgc) if model.RawCatalog.ADDITIONAL_DESIGNATIONS in catalogs else None + ) icrs = icrs_map.get(pgc) if model.RawCatalog.ICRS in catalogs else None redshift = redshift_map.get(pgc) if model.RawCatalog.REDSHIFT in catalogs else None nature = nature_map.get(pgc) if model.RawCatalog.NATURE in catalogs else None @@ -334,6 +382,7 @@ def _layer2_object_from_maps( pgc=pgc, catalogs=layer2_model.Catalogs( designation=designation, + additional_designations=additional_designations, icrs=icrs, redshift=redshift, nature=nature, diff --git a/app/domain/dataapi/parameterized_query.py b/app/domain/dataapi/parameterized_query.py index 98de16d7..08f87969 100644 --- a/app/domain/dataapi/parameterized_query.py +++ b/app/domain/dataapi/parameterized_query.py @@ -3,6 +3,14 @@ from app.domain import responders from app.presentation import dataapi +CATALOGS_FOR_PGC_QUERY = [ + model.RawCatalog.DESIGNATION, + model.RawCatalog.ADDITIONAL_DESIGNATIONS, + model.RawCatalog.ICRS, + model.RawCatalog.REDSHIFT, + model.RawCatalog.NATURE, +] + class ParameterizedQueryManager: def __init__( @@ -55,7 +63,7 @@ def query_simple(self, query: dataapi.QuerySimpleRequest) -> dataapi.QuerySimple responder = responders.StructuredResponder(self.catalog_config) if query.pgcs: objects = self.layer2_repo.query_pgc( - self.enabled_catalogs, + CATALOGS_FOR_PGC_QUERY, query.pgcs, query.page_size, query.page, diff --git a/app/domain/responders/structured_responder.py b/app/domain/responders/structured_responder.py index d0e2c905..1fb0c2a5 100644 --- a/app/domain/responders/structured_responder.py +++ b/app/domain/responders/structured_responder.py @@ -160,6 +160,20 @@ def build_response(self, objects: list[layer2.Layer2Object]) -> Any: if obj.catalogs.designation is not None: catalogs.designation = dataapi.Designation(name=obj.catalogs.designation.name) + if obj.catalogs.additional_designations is not None: + catalogs.additional_designations = [ + dataapi.AdditionalDesignation( + name=ad.name, + source=dataapi.Source( + bibcode=ad.source.bibcode, + title=ad.source.title, + authors=ad.source.authors, + year=ad.source.year, + ), + ) + for ad in obj.catalogs.additional_designations.names + ] + icrs = obj.catalogs.icrs if icrs is not None: ra, dec, e_ra, e_dec = self._equatorial(icrs.ra, icrs.dec, icrs.e_ra, icrs.e_dec) diff --git a/app/presentation/dataapi/interface.py b/app/presentation/dataapi/interface.py index 2940ded1..8bc09ed9 100644 --- a/app/presentation/dataapi/interface.py +++ b/app/presentation/dataapi/interface.py @@ -36,12 +36,25 @@ class Designation(pydantic.BaseModel): name: str +class Source(pydantic.BaseModel): + bibcode: str + title: str + authors: list[str] + year: int + + +class AdditionalDesignation(pydantic.BaseModel): + name: str + source: Source + + class Nature(pydantic.BaseModel): type_name: str class Catalogs(pydantic.BaseModel): designation: Designation | None = None + additional_designations: list[AdditionalDesignation] | None = None coordinates: Coordinates | None = None velocity: dict[str, AbsoluteVelocity] | None = None redshift: Redshift | None = None @@ -129,9 +142,7 @@ def _pgcs_exclusive_with_filters(self) -> "QuerySimpleRequest": if self.pgcs: filters = [self.ra, self.dec, self.radius, self.name, self.cz, self.cz_err_percent] if any(f is not None for f in filters): - raise ValueError( - "When pgcs is specified, no other filters are allowed" - ) + raise ValueError("When pgcs is specified, no other filters are allowed") return self diff --git a/postgres/migrations/V030__additional_designations.sql b/postgres/migrations/V030__additional_designations.sql index 6e6eae32..09ef79f2 100644 --- a/postgres/migrations/V030__additional_designations.sql +++ b/postgres/migrations/V030__additional_designations.sql @@ -8,11 +8,15 @@ SELECT , b.author , b.title FROM - designation.data AS d + designation.data AS d LEFT JOIN layer0.records AS r ON (d.record_id = r.id) LEFT JOIN layer0.tables AS t ON (r.table_id = t.id) LEFT JOIN common.bib AS b ON (t.bib = b.id) -WHERE r.pgc IS NOT NULL; +WHERE r.pgc IS NOT NULL + AND NOT EXISTS ( + SELECT 1 FROM layer2.designation AS ld + WHERE ld.pgc = r.pgc AND ld.design = d.design + ); CREATE INDEX IF NOT EXISTS layer0_records_id_pgc_not_null ON layer0.records (id)