From 628f5bab014307a22802c53b706e8780ad0c6348 Mon Sep 17 00:00:00 2001 From: David Nguyen Date: Wed, 17 Dec 2025 22:21:29 +0000 Subject: [PATCH 1/6] add coadd self-referential relationship --- mapcat/database/atomic_coadd.py | 20 +++++++++++++++++++- mapcat/database/links.py | 24 ++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/mapcat/database/atomic_coadd.py b/mapcat/database/atomic_coadd.py index 636baaa..de14324 100644 --- a/mapcat/database/atomic_coadd.py +++ b/mapcat/database/atomic_coadd.py @@ -9,7 +9,7 @@ if TYPE_CHECKING: from .atomic_map import AtomicMapTable -from .links import AtomicMapToCoaddTable +from .links import AtomicMapToCoaddTable, CoaddMapToCoaddTable class AtomicMapCoaddTable(SQLModel, table=True): @@ -30,3 +30,21 @@ class AtomicMapCoaddTable(SQLModel, table=True): back_populates="coadds", link_model=AtomicMapToCoaddTable, ) + + child_coadds: list["AtomicMapCoaddTable"] = Relationship( + back_populates="parent_coadds", + link_model=CoaddMapToCoaddTable, + sa_relationship_kwargs={ + "primaryjoin": "AtomicMapCoaddTable.coadd_id == CoaddMapToCoaddTable.parent_coadd_id", + "secondaryjoin": "AtomicMapCoaddTable.coadd_id == CoaddMapToCoaddTable.child_coadd_id", + }, + ) + + parent_coadds: list["AtomicMapCoaddTable"] = Relationship( + back_populates="child_coadds", + link_model=CoaddMapToCoaddTable, + sa_relationship_kwargs={ + "primaryjoin": "AtomicMapCoaddTable.coadd_id == CoaddMapToCoaddTable.child_coadd_id", + "secondaryjoin": "AtomicMapCoaddTable.coadd_id == CoaddMapToCoaddTable.parent_coadd_id", + }, + ) diff --git a/mapcat/database/links.py b/mapcat/database/links.py index c506f63..1db33b2 100644 --- a/mapcat/database/links.py +++ b/mapcat/database/links.py @@ -73,3 +73,27 @@ class AtomicMapToCoaddTable(SQLModel, table=True): index=True, ondelete="CASCADE", ) + + +class CoaddMapToCoaddTable(SQLModel, table=True): + """ + Link table for many-to-many relationship between coadd maps and other coadds. + """ + + __tablename__ = "link_coadd_map_to_coadd" + + parent_coadd_id: int = Field( + foreign_key="atomic_map_coadds.coadd_id", + primary_key=True, + nullable=False, + index=True, + ondelete="CASCADE", + ) + + child_coadd_id: int = Field( + foreign_key="atomic_map_coadds.coadd_id", + primary_key=True, + nullable=False, + index=True, + ondelete="CASCADE", + ) From f1e237f230be3a43c038ab4566419a7e726797a6 Mon Sep 17 00:00:00 2001 From: David Nguyen Date: Wed, 17 Dec 2025 22:30:28 +0000 Subject: [PATCH 2/6] rename and add coadd fields --- mapcat/database/atomic_coadd.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/mapcat/database/atomic_coadd.py b/mapcat/database/atomic_coadd.py index de14324..f52e659 100644 --- a/mapcat/database/atomic_coadd.py +++ b/mapcat/database/atomic_coadd.py @@ -18,13 +18,15 @@ class AtomicMapCoaddTable(SQLModel, table=True): coadd_id: int = Field(primary_key=True) coadd_name: str = Field() - coadd_path: str = Field() + prefix_path: str = Field() platform: str = Field() interval: str = Field() start_time: float = Field() - end_time: float = Field() - frequency: str = Field() + stop_time: float = Field() + freq_channel: str = Field() + geom_file_path: str = Field() + split_label: str = Field() atomic_maps: list["AtomicMapTable"] = Relationship( back_populates="coadds", From 9574c045c9b2d9c2fbc96ef85b6190b221f0b8e2 Mon Sep 17 00:00:00 2001 From: David Nguyen Date: Thu, 18 Dec 2025 21:29:52 +0000 Subject: [PATCH 3/6] create alembic revision --- ...670a1fdbe_update_atomic_map_coadd_table.py | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 mapcat/alembic/versions/fd6670a1fdbe_update_atomic_map_coadd_table.py diff --git a/mapcat/alembic/versions/fd6670a1fdbe_update_atomic_map_coadd_table.py b/mapcat/alembic/versions/fd6670a1fdbe_update_atomic_map_coadd_table.py new file mode 100644 index 0000000..e16103c --- /dev/null +++ b/mapcat/alembic/versions/fd6670a1fdbe_update_atomic_map_coadd_table.py @@ -0,0 +1,56 @@ +"""update atomic map coadd table + +Revision ID: fd6670a1fdbe +Revises: 1195d17201ba +Create Date: 2025-12-18 19:50:23.313924 + +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = 'fd6670a1fdbe' +down_revision: Union[str, None] = '1195d17201ba' +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "link_coadd_map_to_coadd", + sa.Column( + "parent_coadd_id", + sa.Integer(), + sa.ForeignKey("atomic_map_coadds.coadd_id"), + nullable=False, + ), + sa.Column( + "child_coadd_id", + sa.Integer(), + sa.ForeignKey("atomic_map_coadds.coadd_id"), + nullable=False, + ), + ) + + op.add_column('atomic_map_coadds', sa.Column('stop_time', sa.Float(), nullable=False)) + op.add_column('atomic_map_coadds', sa.Column('prefix_path', sa.String(), nullable=False)) + op.add_column('atomic_map_coadds', sa.Column('freq_channel', sa.String(), nullable=False)) + op.add_column('atomic_map_coadds', sa.Column('geom_file_path', sa.String(), nullable=False)) + op.add_column('atomic_map_coadds', sa.Column('split_label', sa.String(), nullable=False)) + op.drop_column('atomic_map_coadds', 'end_time') + op.drop_column('atomic_map_coadds', 'coadd_path') + op.drop_column('atomic_map_coadds', 'frequency') + + +def downgrade() -> None: + op.add_column('atomic_map_coadds', sa.Column('frequency', sa.String(), nullable=False)) + op.add_column('atomic_map_coadds', sa.Column('coadd_path', sa.String(), nullable=False)) + op.add_column('atomic_map_coadds', sa.Column('end_time', sa.Float(), nullable=False)) + op.drop_column('atomic_map_coadds', 'split_label') + op.drop_column('atomic_map_coadds', 'geom_file_path') + op.drop_column('atomic_map_coadds', 'freq_channel') + op.drop_column('atomic_map_coadds', 'prefix_path') + op.drop_column('atomic_map_coadds', 'stop_time') + op.drop_table('link_coadd_map_to_coadd') From 74f0af0d6c6c9016937f4580e53e2223fc4e3bce Mon Sep 17 00:00:00 2001 From: David Nguyen Date: Thu, 18 Dec 2025 23:19:58 +0000 Subject: [PATCH 4/6] add coadd tests --- tests/test_mapcat.py | 259 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 259 insertions(+) diff --git a/tests/test_mapcat.py b/tests/test_mapcat.py index 35811e5..65fd6d3 100644 --- a/tests/test_mapcat.py +++ b/tests/test_mapcat.py @@ -13,6 +13,8 @@ SkyCoverageTable, TimeDomainProcessingTable, TODDepthOneTable, + AtomicMapTable, + AtomicMapCoaddTable, ) @@ -332,3 +334,260 @@ def test_add_remove_child_tables(database_sessionmaker): x = session.get(SkyCoverageTable, sky_id) if x is None: raise ValueError("Not found") + + +def test_create_atomic_map_coadd(database_sessionmaker): + # Create a daily (child) coadd + with database_sessionmaker() as session: + data = AtomicMapCoaddTable( + coadd_name="myDailyCoadd", + prefix_path="/PATH/TO/DAILY/COADD", + platform="satp3", + interval="daily", + start_time=1755604800.0, + stop_time=1755691200.0, + freq_channel="f090", + geom_file_path="/PATH/TO/GEOM/FILE", + split_label="full", + ) + + session.add(data) + session.commit() + session.refresh(data) + + daily_coadd_id = data.coadd_id + + # Get daily coadd back + with database_sessionmaker() as session: + cmap = session.get(AtomicMapCoaddTable, daily_coadd_id) + + assert cmap.coadd_id == daily_coadd_id + assert cmap.coadd_name == "myDailyCoadd" + assert cmap.prefix_path == "/PATH/TO/DAILY/COADD" + assert cmap.platform == "satp3" + assert cmap.interval == "daily" + assert cmap.start_time == 1755604800.0 + assert cmap.stop_time == 1755691200.0 + assert cmap.freq_channel == "f090" + assert cmap.geom_file_path == "/PATH/TO/GEOM/FILE" + assert cmap.split_label == "full" + + # Make child atomic table + with database_sessionmaker() as session: + atomic_map = AtomicMapTable( + obs_id="obs_1755643930_satp3_1111111", + telescope="satp3", + freq_channel="f090", + wafer="ws0", + ctime=1755643932, + split_label="full", + map_path=None, + ivar_path=None, + valid=True, + split_detail="", + prefix_path="PATH/TO/ATOMIC/MAP", + elevation=59.9997, + azimuth=290.9026, + pwv=None, + dpwv=None, + total_weight_qu=660616773042064.2, + mean_weight_qu=4206115923.380497, + median_weight_qu=3255646486.1875, + leakage_avg=None, + noise_avg=None, + ampl_2f_avg=None, + gain_avg=None, + tau_avg=None, + f_hwp=None, + roll_angle=0.0117, + scan_speed=None, + scan_acc=None, + sun_distance=69.36016359440542, + ambient_temperature=None, + uv=None, + ra_center=None, + dec_center=None, + number_dets=568, + moon_distance=None, + wind_speed=None, + wind_direction=None, + rqu_avg=None, + coadds=[cmap], + ) + + session.add(atomic_map) + session.commit() + + atomic_map_id = atomic_map.atomic_map_id + + # Get atomic table back + with database_sessionmaker() as session: + atomic = session.get(AtomicMapTable, atomic_map_id) + atomic_coadds = atomic.coadds + + assert atomic.atomic_map_id == atomic_map_id + assert atomic.obs_id == "obs_1755643930_satp3_1111111" + assert atomic.telescope == "satp3" + assert atomic.freq_channel == "f090" + assert atomic.wafer == "ws0" + assert atomic.ctime == 1755643932 + assert atomic.split_label == "full" + assert atomic.map_path == None + assert atomic.ivar_path == None + assert atomic.valid == True + assert atomic.split_detail == "" + assert atomic.prefix_path == "PATH/TO/ATOMIC/MAP" + assert atomic.elevation == 59.9997 + assert atomic.azimuth == 290.9026 + assert atomic.pwv == None + assert atomic.dpwv == None + assert atomic.total_weight_qu == 660616773042064.2 + assert atomic.mean_weight_qu == 4206115923.380497 + assert atomic.median_weight_qu == 3255646486.1875 + assert atomic.leakage_avg == None + assert atomic.noise_avg == None + assert atomic.ampl_2f_avg == None + assert atomic.gain_avg == None + assert atomic.tau_avg == None + assert atomic.f_hwp == None + assert atomic.roll_angle == 0.0117 + assert atomic.scan_speed == None + assert atomic.scan_acc == None + assert atomic.sun_distance == 69.36016359440542 + assert atomic.ambient_temperature == None + assert atomic.uv == None + assert atomic.ra_center == None + assert atomic.dec_center == None + assert atomic.number_dets == 568 + assert atomic.moon_distance == None + assert atomic.wind_speed == None + assert atomic.wind_direction == None + assert atomic.rqu_avg == None + + assert atomic_coadds[0].coadd_id == daily_coadd_id + + # Create a weekly (parent) coadd + with database_sessionmaker() as session: + data = AtomicMapCoaddTable( + coadd_name="myWeeklyCoadd", + prefix_path="/PATH/TO/WEEKLY/COADD", + platform="satp3", + interval="weekly", + start_time=1755432000.0, + stop_time=1756036800.0, + freq_channel="f090", + geom_file_path="/PATH/TO/GEOM/FILE", + split_label="full", + child_coadds=[cmap] + ) + + session.add(data) + session.commit() + session.refresh(data) + + weekly_coadd_id = data.coadd_id + + # Get weekly coadd map back + with database_sessionmaker() as session: + cmap2 = session.get(AtomicMapCoaddTable, weekly_coadd_id) + weekly_child_coadds = cmap2.child_coadds + + assert cmap2.coadd_id == weekly_coadd_id + assert weekly_child_coadds[0].coadd_id == daily_coadd_id + + # Check daily coadd has weekly coadd as parent + with database_sessionmaker() as session: + cmap = session.get(AtomicMapCoaddTable, daily_coadd_id) + daily_parent_coadds = cmap.parent_coadds + + assert daily_parent_coadds[0].coadd_id == weekly_coadd_id + + # Check bad map ID raises ValueError + with pytest.raises(ValueError): + with database_sessionmaker() as session: + result = session.get(AtomicMapCoaddTable, 999999) + if result is None: + raise ValueError("Map ID does not exist") + + +def test_add_remove_atomic_map_coadd(database_sessionmaker): + # Create daily and weekly coadds + with database_sessionmaker() as session: + daily = AtomicMapCoaddTable( + coadd_name="myDailyCoadd", + prefix_path="/PATH/TO/DAILY/COADD", + platform="satp3", + interval="daily", + start_time=1755604800.0, + stop_time=1755691200.0, + freq_channel="f090", + geom_file_path="/PATH/TO/GEOM/FILE", + split_label="full", + ) + + weekly = AtomicMapCoaddTable( + coadd_name="myWeeklyCoadd", + prefix_path="/PATH/TO/WEEKLY/COADD", + platform="satp3", + interval="weekly", + start_time=1755432000.0, + stop_time=1756036800.0, + freq_channel="f090", + geom_file_path="/PATH/TO/GEOM/FILE", + split_label="full", + child_coadds=[daily] + ) + + session.add_all( + [ + daily, + weekly, + ] + ) + session.commit() + + daily_coadd_id = daily.coadd_id + weekly_coadd_id = weekly.coadd_id + + # Remove weekly coadd and check daily coadd parents + with database_sessionmaker() as session: + x = session.get(AtomicMapCoaddTable, weekly_coadd_id) + session.delete(x) + session.commit() + + with pytest.raises(ValueError): + with database_sessionmaker() as session: + daily = session.get(AtomicMapCoaddTable, daily_coadd_id) + if len(daily.parent_coadds) == 0: + raise ValueError("Daily map has no parent coadds") + + # Check in reverse + with database_sessionmaker() as session: + weekly2 = AtomicMapCoaddTable( + coadd_name="myWeeklyCoadd2", + prefix_path="/PATH/TO/WEEKLY/COADD", + platform="satp3", + interval="weekly", + start_time=1755432000.0, + stop_time=1756036800.0, + freq_channel="f090", + geom_file_path="/PATH/TO/GEOM/FILE", + split_label="full", + child_coadds=[daily] + ) + + session.add(weekly2) + session.commit() + + weekly2_coadd_id = weekly2.coadd_id + + with database_sessionmaker() as session: + x = session.get(AtomicMapCoaddTable, daily_coadd_id) + session.delete(x) + session.commit() + + with pytest.raises(ValueError): + with database_sessionmaker() as session: + weekly2 = session.get(AtomicMapCoaddTable, weekly2_coadd_id) + if len(weekly2.child_coadds) == 0: + raise ValueError("Weekly map has no child coadds") \ No newline at end of file From b7f5a8bcf958712f0cf1634a2afebd4e19c8ce98 Mon Sep 17 00:00:00 2001 From: David Nguyen Date: Mon, 22 Dec 2025 18:40:31 +0000 Subject: [PATCH 5/6] fix asserts --- tests/test_mapcat.py | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/tests/test_mapcat.py b/tests/test_mapcat.py index 65fd6d3..06e4007 100644 --- a/tests/test_mapcat.py +++ b/tests/test_mapcat.py @@ -432,37 +432,37 @@ def test_create_atomic_map_coadd(database_sessionmaker): assert atomic.wafer == "ws0" assert atomic.ctime == 1755643932 assert atomic.split_label == "full" - assert atomic.map_path == None - assert atomic.ivar_path == None - assert atomic.valid == True + assert atomic.map_path is None + assert atomic.ivar_path is None + assert atomic.valid assert atomic.split_detail == "" assert atomic.prefix_path == "PATH/TO/ATOMIC/MAP" assert atomic.elevation == 59.9997 assert atomic.azimuth == 290.9026 - assert atomic.pwv == None - assert atomic.dpwv == None + assert atomic.pwv is None + assert atomic.dpwv is None assert atomic.total_weight_qu == 660616773042064.2 assert atomic.mean_weight_qu == 4206115923.380497 assert atomic.median_weight_qu == 3255646486.1875 - assert atomic.leakage_avg == None - assert atomic.noise_avg == None - assert atomic.ampl_2f_avg == None - assert atomic.gain_avg == None - assert atomic.tau_avg == None - assert atomic.f_hwp == None + assert atomic.leakage_avg is None + assert atomic.noise_avg is None + assert atomic.ampl_2f_avg is None + assert atomic.gain_avg is None + assert atomic.tau_avg is None + assert atomic.f_hwp is None assert atomic.roll_angle == 0.0117 - assert atomic.scan_speed == None - assert atomic.scan_acc == None + assert atomic.scan_speed is None + assert atomic.scan_acc is None assert atomic.sun_distance == 69.36016359440542 - assert atomic.ambient_temperature == None - assert atomic.uv == None - assert atomic.ra_center == None - assert atomic.dec_center == None + assert atomic.ambient_temperature is None + assert atomic.uv is None + assert atomic.ra_center is None + assert atomic.dec_center is None assert atomic.number_dets == 568 - assert atomic.moon_distance == None - assert atomic.wind_speed == None - assert atomic.wind_direction == None - assert atomic.rqu_avg == None + assert atomic.moon_distance is None + assert atomic.wind_speed is None + assert atomic.wind_direction is None + assert atomic.rqu_avg is None assert atomic_coadds[0].coadd_id == daily_coadd_id From 407bfe0eaa7e4b04c02f4ba93dcba9a6b2ac860e Mon Sep 17 00:00:00 2001 From: David Nguyen Date: Mon, 22 Dec 2025 18:43:28 +0000 Subject: [PATCH 6/6] fix import --- tests/test_mapcat.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_mapcat.py b/tests/test_mapcat.py index 06e4007..ba2c330 100644 --- a/tests/test_mapcat.py +++ b/tests/test_mapcat.py @@ -7,14 +7,14 @@ from sqlalchemy.orm import sessionmaker from mapcat.database import ( + AtomicMapCoaddTable, + AtomicMapTable, DepthOneMapTable, PipelineInformationTable, PointingResidualTable, SkyCoverageTable, TimeDomainProcessingTable, TODDepthOneTable, - AtomicMapTable, - AtomicMapCoaddTable, )