From c9d37f35b8df2bf6bce9ad79b4447d632e658475 Mon Sep 17 00:00:00 2001 From: Erik Jensen Date: Thu, 30 Jan 2025 14:23:40 +0100 Subject: [PATCH 1/2] Add ability to delete reference stars --- flows/__init__.py | 2 +- flows/catalogs.py | 16 ++++++++++++++++ flows/fileio.py | 7 +++++++ run_catalogs.py | 14 ++++++++++++-- 4 files changed, 36 insertions(+), 3 deletions(-) diff --git a/flows/__init__.py b/flows/__init__.py index 079901c..3a47db2 100644 --- a/flows/__init__.py +++ b/flows/__init__.py @@ -2,7 +2,7 @@ FLOWS pipeline package """ from .photometry import timed_photometry as photometry -from .catalogs import download_catalog +from .catalogs import download_catalog, delete_catalog from .visibility import visibility from .version import get_version from .load_image import load_image diff --git a/flows/catalogs.py b/flows/catalogs.py index 22bc318..623f7f3 100644 --- a/flows/catalogs.py +++ b/flows/catalogs.py @@ -24,6 +24,7 @@ from astroquery.simbad import Simbad from bottleneck import anynan from tendrils.utils import load_config, query_ztf_id +from tendrils import api from .aadc_db import AADC_DB @@ -567,3 +568,18 @@ def download_catalog(target=None, radius=24 * u.arcmin, radius_ztf=3 * u.arcsec, except: # noqa: E722, pragma: no cover db.conn.rollback() raise + + +def delete_catalog(target): + cat = api.get_catalog(target) + ids = list(cat['references']['starid']) + + id_list = ",".join([str(id) for id in ids]) + + print(id_list) + + with AADC_DB() as db: + if target is not None and isinstance(target, (int, float)): + db.cursor.execute("DELETE FROM flows.refcat2 WHERE starid IN (%s);" % id_list) + db.cursor.execute("UPDATE flows.targets SET catalog_downloaded=FALSE,ztf_id=NULL WHERE targetid=%s;" % int(target)) + db.conn.commit() diff --git a/flows/fileio.py b/flows/fileio.py index c1ea16e..212e6ca 100644 --- a/flows/fileio.py +++ b/flows/fileio.py @@ -192,6 +192,13 @@ def get_filter(self): def load_references(self, catalog: Optional[Dict] = None) -> refclean.References: use_filter = self.get_filter() references = api.get_catalog(self.target.name)['references'] if catalog is None else catalog['references'] + + filter = (references["J_mag"] > 10) & (references["H_mag"] > 10) & (references["g_mag"] is not None) \ + & (references["H_mag"] is not None) & (references["gaia_variability"] != 1) \ + & (references["g_mag"] - references["r_mag"] < 10) & (references["starid"] != 107441912858406185) \ + & (references["starid"] != 107441912838383932) + + references = references[filter] references.sort(use_filter) # Check that there actually are reference stars in that filter: if allnan(references[use_filter]): diff --git a/run_catalogs.py b/run_catalogs.py index 5fcc46c..cbeb0ec 100644 --- a/run_catalogs.py +++ b/run_catalogs.py @@ -4,7 +4,7 @@ import argparse import logging from tendrils import api -from flows import download_catalog +from flows import download_catalog, delete_catalog def parse(): @@ -16,7 +16,8 @@ def parse(): parser.add_argument('-q', '--quiet', help='Only report warnings and errors.', action='store_true') parser.add_argument('-c', '--commit', help='Commit downloaded catalog(s) to flows database.', action='store_true') parser.add_argument('-p', '--print', help='Print single catalog which already exists in flows database.', action='store_true') - parser.add_argument('-t', '--target', type=str, help='Optionally specify just one target for downloading/committing/printing.', nargs='?', default=None) + parser.add_argument('-t', '--target', type=str, help='Optionally specify just one target for downloading/committing/printing/deleting.', nargs='?', default=None) + parser.add_argument('-D', '--delete', help='Delete single catalog which already exists in flows database.', action='store_true') return parser.parse_args() def set_logging_level(args): @@ -42,6 +43,15 @@ def main(): logger.addHandler(console) logger.setLevel(logging_level) + if args.delete: + if args.target is not None: + logger.info("Deleting catalog for target=%s...", args.target) + delete_catalog(int(args.target)) + logger.info("Done deleting catalog for target=%s...", args.target) + else: + logger.warning("Need to specify target to delete") + exit(0) + targets = api.get_catalog_missing() if args.target is not None: if args.print is False: From 17f6e51a4fc34dacb8a5c6c6c35adb46c54149a6 Mon Sep 17 00:00:00 2001 From: Erik Jensen Date: Thu, 30 Jan 2025 14:28:30 +0100 Subject: [PATCH 2/2] Add refcat2 limiting magnitude as parameter; bump VERSION --- VERSION | 2 +- flows/catalogs.py | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/VERSION b/VERSION index d327411..9a437dc 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -master-v1.2.1 +master-v1.2.2 diff --git a/flows/catalogs.py b/flows/catalogs.py index 623f7f3..c26d357 100644 --- a/flows/catalogs.py +++ b/flows/catalogs.py @@ -258,7 +258,7 @@ def query_skymapper(coo_centre, radius=24 * u.arcmin): # -------------------------------------------------------------------------------------------------- -def query_local_refcat2(coo_centre, radius=24 * u.arcmin): +def query_local_refcat2(coo_centre, radius=24 * u.arcmin, limiting_magnitude=19): """ Uses refcat.c from https://archive.stsci.edu/hlsp/atlas-refcat2 to do a cone-search around the position. NOTE: In going from mast casjobs to local refcat.c, we have lost the unique object identifier, 'objid' @@ -290,6 +290,7 @@ def query_local_refcat2(coo_centre, radius=24 * u.arcmin): refcat_output = subprocess.run(["refcat", str(coo_centre.ra.deg), str(coo_centre.dec.deg), "-rad", str(Angle(radius).deg), + "-mlim", str(limiting_magnitude), "-var", "ra,dec,pmra,pmdec,gaia,bp,rp,dupvar,g,r,i,z,j,h,k", "-nohdr"], encoding="utf-8", capture_output=True, check=True) @@ -332,7 +333,7 @@ def query_local_refcat2(coo_centre, radius=24 * u.arcmin): logger.debug("Found %d results", len(results)) return results -def query_all(coo_centre, radius=24 * u.arcmin, dist_cutoff=2 * u.arcsec): +def query_all(coo_centre, radius=24 * u.arcmin, dist_cutoff=2 * u.arcsec, limiting_magnitude=19): """ Query all catalogs (REFCAT2, APASS, SDSS and SkyMapper) and return merged catalog. @@ -355,7 +356,7 @@ def query_all(coo_centre, radius=24 * u.arcmin, dist_cutoff=2 * u.arcsec): """ # Query the REFCAT2 catalog using CasJobs around the target position: - results = query_local_refcat2(coo_centre, radius=radius) + results = query_local_refcat2(coo_centre, radius=radius, limiting_magnitude=limiting_magnitude) AT_results = Table(results) refcat = SkyCoord(ra=AT_results['ra'], dec=AT_results['decl'], unit=u.deg, frame='icrs')