From 856f13a42a9f07fab873bea9b03d1828129a870c Mon Sep 17 00:00:00 2001 From: kraysent Date: Mon, 20 Apr 2026 20:54:51 +0100 Subject: [PATCH 1/2] add crossmatch status results to get /table --- app/data/model/__init__.py | 2 + app/data/model/table.py | 5 +++ app/data/repositories/layer0/records.py | 21 ++++++++++ app/data/repositories/layer0/repository.py | 3 ++ app/domain/adminapi/table_upload.py | 26 +++++++++--- app/presentation/adminapi/interface.py | 15 ++++++- tests/regression/upload_simple_table.py | 49 +++++++++++++++++++++- 7 files changed, 112 insertions(+), 9 deletions(-) diff --git a/app/data/model/__init__.py b/app/data/model/__init__.py index 08750101..9f971d19 100644 --- a/app/data/model/__init__.py +++ b/app/data/model/__init__.py @@ -29,6 +29,7 @@ Layer0RawData, Layer0TableListItem, Layer0TableMeta, + TableCrossmatchSummary, TableRecord, TableStatistics, ) @@ -41,6 +42,7 @@ "Layer0TableListItem", "ColumnDescription", "TableStatistics", + "TableCrossmatchSummary", "get_object", "DesignationRecord", "ICRSRecord", diff --git a/app/data/model/table.py b/app/data/model/table.py index d4436d00..1e17073f 100644 --- a/app/data/model/table.py +++ b/app/data/model/table.py @@ -65,3 +65,8 @@ class TableStatistics: last_modified_dt: datetime.datetime total_rows: int total_original_rows: int + + +@dataclass +class TableCrossmatchSummary: + counts: dict[str, int] diff --git a/app/data/repositories/layer0/records.py b/app/data/repositories/layer0/records.py index 5e37fd44..50b9b934 100644 --- a/app/data/repositories/layer0/records.py +++ b/app/data/repositories/layer0/records.py @@ -174,6 +174,27 @@ def get_table_statistics(self, table_name: str) -> model.TableStatistics: total_original_rows, ) + def get_table_crossmatch_summary(self, table_name: str) -> model.TableCrossmatchSummary: + table_id_row = self._storage.query_one(template.FETCH_RAWDATA_REGISTRY, params=[table_name]) + table_id = table_id_row["id"] + + rows = self._storage.query( + """ + SELECT CASE + WHEN c.record_id IS NULL THEN 'unprocessed' + ELSE c.triage_status + END AS triage, + COUNT(1) AS cnt + FROM layer0.records AS o + LEFT JOIN layer0.crossmatch AS c ON c.record_id = o.id + WHERE o.table_id = %s + GROUP BY triage + """, + params=[table_id], + ) + + return model.TableCrossmatchSummary(counts={row["triage"]: row["cnt"] for row in rows}) + def set_crossmatch_results(self, rows: list[tuple[str, enums.RecordTriageStatus, list[int]]]) -> None: if not rows: return diff --git a/app/data/repositories/layer0/repository.py b/app/data/repositories/layer0/repository.py index 600fd700..adc1e027 100644 --- a/app/data/repositories/layer0/repository.py +++ b/app/data/repositories/layer0/repository.py @@ -104,6 +104,9 @@ def register_records(self, table_name: str, record_ids: list[str]) -> None: def get_table_statistics(self, table_name: str) -> model.TableStatistics: return self.records_repo.get_table_statistics(table_name) + def get_table_crossmatch_summary(self, table_name: str) -> model.TableCrossmatchSummary: + return self.records_repo.get_table_crossmatch_summary(table_name) + def get_processed_records( self, limit: int, diff --git a/app/domain/adminapi/table_upload.py b/app/domain/adminapi/table_upload.py index 48c9397d..0ea98268 100644 --- a/app/domain/adminapi/table_upload.py +++ b/app/domain/adminapi/table_upload.py @@ -196,12 +196,25 @@ def get_table(self, r: adminapi.GetTableRequest) -> adminapi.GetTableResponse: raise RuntimeError(f"Table {r.table_name} has no ID") table_stats = self.layer0_repo.get_table_statistics(r.table_name) + crossmatch_summary = self.layer0_repo.get_table_crossmatch_summary(r.table_name) rows_num = table_stats.total_original_rows metadata = {"datatype": meta.datatype, "modification_dt": meta.modification_dt} - - statistics = None - if table_stats.statuses: - statistics = table_stats.statuses + pending_count = crossmatch_summary.counts.get(adminapi.CrossmatchTriageStatus.PENDING.value, 0) + resolved_count = crossmatch_summary.counts.get(adminapi.CrossmatchTriageStatus.RESOLVED.value, 0) + unprocessed_count = crossmatch_summary.counts.get(adminapi.CrossmatchTriageStatus.UNPROCESSED.value, 0) + + if resolved_count == 0 and pending_count == 0: + crossmatch_result = adminapi.TableCrossmatchResultStatus.NOT_STARTED + elif pending_count > 0 or unprocessed_count > 0: + crossmatch_result = adminapi.TableCrossmatchResultStatus.IN_PROGRESS + else: + crossmatch_result = adminapi.TableCrossmatchResultStatus.DONE + + crossmatch_statuses = { + adminapi.CrossmatchTriageStatus.UNPROCESSED: unprocessed_count, + adminapi.CrossmatchTriageStatus.PENDING: pending_count, + adminapi.CrossmatchTriageStatus.RESOLVED: resolved_count, + } return adminapi.GetTableResponse( id=meta.table_id, @@ -210,7 +223,10 @@ def get_table(self, r: adminapi.GetTableRequest) -> adminapi.GetTableResponse: rows_num=rows_num, meta=metadata, bibliography=_bibliography_to_presentation(bibliography), - statistics=statistics, + crossmatch=adminapi.TableCrossmatchResults( + result=crossmatch_result, + statuses=crossmatch_statuses, + ), ) def get_records(self, r: adminapi.GetRecordsRequest) -> adminapi.GetRecordsResponse: diff --git a/app/presentation/adminapi/interface.py b/app/presentation/adminapi/interface.py index ff874cf2..a7833e0d 100644 --- a/app/presentation/adminapi/interface.py +++ b/app/presentation/adminapi/interface.py @@ -7,7 +7,7 @@ from astropy import units as u from app.lib.storage import enums, mapping -from app.presentation.adminapi.records import GetRecordsRequest, GetRecordsResponse +from app.presentation.adminapi.records import CrossmatchTriageStatus, GetRecordsRequest, GetRecordsResponse DatatypeEnum = enum.StrEnum( "DatatypeEnum", @@ -71,6 +71,17 @@ class GetTableListResponse(pydantic.BaseModel): tables: list[TableListItem] +class TableCrossmatchResultStatus(enum.Enum): + DONE = "done" + IN_PROGRESS = "in_progress" + NOT_STARTED = "not_started" + + +class TableCrossmatchResults(pydantic.BaseModel): + result: TableCrossmatchResultStatus + statuses: dict[CrossmatchTriageStatus, int] + + class GetTableResponse(pydantic.BaseModel): id: int description: str @@ -78,7 +89,7 @@ class GetTableResponse(pydantic.BaseModel): rows_num: int meta: dict[str, Any] bibliography: Bibliography - statistics: dict[enums.RecordCrossmatchStatus, int] | None = None + crossmatch: TableCrossmatchResults class CreateTableRequest(pydantic.BaseModel): diff --git a/tests/regression/upload_simple_table.py b/tests/regression/upload_simple_table.py index 489e21c5..fe94db6d 100644 --- a/tests/regression/upload_simple_table.py +++ b/tests/regression/upload_simple_table.py @@ -363,7 +363,14 @@ def check_table_list(session: requests.Session, table_name: str): @lib.test_logging_decorator -def check_get_table(session: requests.Session, table_name: str, expected_columns: int, expected_rows: int): +def check_get_table( + session: requests.Session, + table_name: str, + expected_columns: int, + expected_rows: int, + expected_result: adminapi.TableCrossmatchResultStatus, + expected_statuses: dict[adminapi.CrossmatchTriageStatus, int], +): request_data = adminapi.GetTableRequest(table_name=table_name) response = session.get("/v1/table", params=request_data.model_dump(mode="json")) response.raise_for_status() @@ -375,6 +382,9 @@ def check_get_table(session: requests.Session, table_name: str, expected_columns assert table_info["rows_num"] == expected_rows assert "bibliography" in table_info assert "meta" in table_info + assert "crossmatch" in table_info + assert table_info["crossmatch"]["result"] == expected_result.value + assert table_info["crossmatch"]["statuses"] == {status.value: count for status, count in expected_statuses.items()} @lib.test_logging_decorator @@ -497,7 +507,18 @@ def run(): ) check_table_list(adminapi_session, table_name) - check_get_table(adminapi_session, table_name, expected_columns=6, expected_rows=OBJECTS_NUM) + check_get_table( + adminapi_session, + table_name, + expected_columns=6, + expected_rows=OBJECTS_NUM, + expected_result=adminapi.TableCrossmatchResultStatus.NOT_STARTED, + expected_statuses={ + adminapi.CrossmatchTriageStatus.UNPROCESSED: OBJECTS_NUM, + adminapi.CrossmatchTriageStatus.PENDING: 0, + adminapi.CrossmatchTriageStatus.RESOLVED: 0, + }, + ) records = get_records(adminapi_session, table_name, OBJECTS_NUM * 2) upload_structured_data(adminapi_session, records) @@ -509,6 +530,18 @@ def run(): record_ids, triage_statuses=[enums.RecordTriageStatus.RESOLVED] * len(record_ids), ) + check_get_table( + adminapi_session, + table_name, + expected_columns=6, + expected_rows=OBJECTS_NUM, + expected_result=adminapi.TableCrossmatchResultStatus.DONE, + expected_statuses={ + adminapi.CrossmatchTriageStatus.UNPROCESSED: 0, + adminapi.CrossmatchTriageStatus.PENDING: 0, + adminapi.CrossmatchTriageStatus.RESOLVED: OBJECTS_NUM, + }, + ) submit_crossmatch(table_name) layer2_import() @@ -543,6 +576,18 @@ def run(): triage_statuses=[enums.RecordTriageStatus.PENDING] * n_pending + [enums.RecordTriageStatus.RESOLVED] * n_resolved, ) + check_get_table( + adminapi_session, + table_name_2, + expected_columns=6, + expected_rows=TABLE2_OBJECTS_NUM, + expected_result=adminapi.TableCrossmatchResultStatus.IN_PROGRESS, + expected_statuses={ + adminapi.CrossmatchTriageStatus.UNPROCESSED: 0, + adminapi.CrossmatchTriageStatus.PENDING: n_pending, + adminapi.CrossmatchTriageStatus.RESOLVED: n_resolved, + }, + ) check_triage_via_records(adminapi_session, table_name_2, expected_pending=n_pending, expected_resolved=n_resolved) From c610f20c4c2d843376ce0c835ea4e3d7b9199701 Mon Sep 17 00:00:00 2001 From: kraysent Date: Mon, 20 Apr 2026 21:07:23 +0100 Subject: [PATCH 2/2] fix query --- app/data/repositories/layer0/records.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/data/repositories/layer0/records.py b/app/data/repositories/layer0/records.py index 50b9b934..80fb0ea7 100644 --- a/app/data/repositories/layer0/records.py +++ b/app/data/repositories/layer0/records.py @@ -182,7 +182,7 @@ def get_table_crossmatch_summary(self, table_name: str) -> model.TableCrossmatch """ SELECT CASE WHEN c.record_id IS NULL THEN 'unprocessed' - ELSE c.triage_status + ELSE c.triage_status::text END AS triage, COUNT(1) AS cnt FROM layer0.records AS o