diff --git a/.gitignore b/.gitignore index 19d4a93..078abcf 100644 --- a/.gitignore +++ b/.gitignore @@ -8,4 +8,6 @@ *.xdv *.dvi *.pdf +*.tdo authors.tex +RTN-114.synctex.gz diff --git a/RTN-114.tex b/RTN-114.tex index 5abc14a..79047d0 100644 --- a/RTN-114.tex +++ b/RTN-114.tex @@ -3,13 +3,53 @@ % Author guide: % https://journals.aas.org/aastexguide/ -\documentclass[modern]{aastex7} +% dvipsnames passed to xcolor +\documentclass[twocolumn,twocolappendix, linenumbers, twocolappendix,trackchanges, dvipsnames]{aastex7} % Package imports go here. + +\usepackage{amsmath} +\usepackage{textcomp} +\usepackage{gensymb} +\usepackage{graphicx} +\usepackage{mathtools, nccmath} +\usepackage{hyperref} +\usepackage[htt]{hyphenat} % Hyphenate texttt blocks +\usepackage{xspace} % grids of plots +\usepackage{subcaption} +\captionsetup{compatibility=false} +%\usepackage[dvipsnames]{xcolor} % already loaded in by aaxtex7, creates option class otherwise + +% TODO notes +\usepackage[colorinlistoftodos]{todonotes} +\let\Oldtodo\todo +\renewcommand{\todo}[1]{\Oldtodo[inline]{#1}} +\newcommand{\eric}[2][]{\Oldtodo[inline,color=cyan, #1]{Eric: #2}} +\newcommand{\ian}[2][]{\Oldtodo[inline,color=red, #1]{Ian: #2}} +\newcommand{\bruno}[2][]{\Oldtodo[inline,color=green, #1]{Bruno: #2}} +\newcommand{\bri}[2][]{\Oldtodo[inline,color=purple, #1]{Bri: #2}} +\newcommand{\jake}[2][]{\Oldtodo[inline,color=yellow, #1]{Jake: #2}} +\newcommand{\gregory}[2][]{\Oldtodo[inline,color=lime, #1]{Gregory: #2}} +\newcommand{\melissa}[2][]{\Oldtodo[inline,color=Orchid, #1]{Melissa: #2}} +\newcommand{\TODO}[1]{\textcolor{red}{\textbf{TODO:} #1}} + % Local commands go here. \newcommand{\docRef}{RTN-114} \newcommand{\docUpstreamLocation}{\url{https://github.com/lsst/rtn-114}} +\newcommand{\degsq}{$\rm deg^{2}$} + +%\providecommand{\secref}[1]{\hyperref[#1]{\autoref{#1}}} +\providecommand{\secref}[1]{\hyperref[#1]{\S\ref{#1}}} +\providecommand{\appref}[1]{\hyperref[#1]{Appendix \ref{#1}}} +\providecommand{\tabref}[1]{\hyperref[#1]{\autoref{#1}}} +\providecommand{\figref}[1]{\hyperref[#1]{\autoref{#1}}} +\providecommand{\eqnref}[1]{\hyperref[#1]{Eq.~\ref{#1}}} +\providecommand{\recref}[1]{\hyperref[#1]{REC-\ref{#1}}} + + +% All graphics paths here +\graphicspath{./} %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %% @@ -44,19 +84,22 @@ %%\graphicspath{{./}{figures/}} + +% Include the parameters file with the data +\input{parameters} + + \begin{document} \doi{10.71929/rubin/3019817} \input{authors} +\correspondingauthor{Eric C. Bellm, Ian Sullivan} + \date{\today} \title{Alert Production with the Vera C. Rubin Observatory} % This can write metadata into the PDF. % Update keywords and author information as necessary. -\hypersetup{ - pdftitle={Alert Production with the Vera C. Rubin Observatory}, - pdfauthor={RubinBuilderPaper}, - pdfkeywords={} -} +\hypersetup{pdftitle={\@title}, pdfauthor={\@author}, pdfkeywords={\@keywords}} \input{abstract} @@ -66,8 +109,24 @@ %% You can use the \uat command to link your UAT concepts back its source. %% \keywords{\uat{Galaxies}{573} --- \uat{Cosmology}{343}} -%% Main content as individual inputs. -\input{body} +\listoftodos + +\input{intro} +\input{observations} + +\input{data_products} +\input{pipeline} + +\input{processing} + +\input{performance} + +\input{data_services} + +\input{support} + +\input{conclusion} + %% Modify acknowledgments as needed. @@ -87,13 +146,18 @@ %% parentheses, after the keyword, but they are not verified. \vspace{5mm} -\facility{Rubin} +\facility{Rubin:Simonyi (LSSTCam), Rubin:USDAC} %% Similar to \facility{}, there is the optional \software command to allow %% authors a place to specify which programs were used during the creation of %% the manuscript. Authors should list each code and include either a %% citation or url to the code inside ()s when available. -\software{LSST Science Pipelines} +\eric{review and update citations to use current DOIs} +\software{Rubin Data Butler \citep{2022SPIE12189E..11J}, + LSST Science Pipelines \citep{10.71929/rubin/2570545}, + LSST Feature Based Scheduler v3.0 \citep{peter_yoachim_2024_13985198, Naghib_2019} + Astropy \citep{2013A&A...558A..33A, 2018AJ....156..123A, 2022ApJ...935..167A} + } %% Appendix material should be preceded with a single \appendix command. %% There should be a \section command for each appendix. Mark appendix @@ -106,6 +170,7 @@ %% \appendix +%\printglossaries % Include all the relevant bib files so that references can be found from % lsst-texmf. diff --git a/abstract.tex b/abstract.tex index a7d9165..4a1a20b 100644 --- a/abstract.tex +++ b/abstract.tex @@ -1,5 +1,7 @@ \begin{abstract} -Describes the LSST Alert Production pipelines, data products, execution environment, and early performance. + +Describes the LSST Alert Production pipelines, data products, execution environment, and early performance. + \end{abstract} diff --git a/conclusion.tex b/conclusion.tex new file mode 100644 index 0000000..d2d9e46 --- /dev/null +++ b/conclusion.tex @@ -0,0 +1,4 @@ +%\section{Discussion} +\section{Conclusion} \label{sec:conclusion} + +\eric{summarize} diff --git a/data_products.tex b/data_products.tex new file mode 100644 index 0000000..283efc6 --- /dev/null +++ b/data_products.tex @@ -0,0 +1,50 @@ +\section{Prompt Data Products} \label{sec:data-products} + + +\eric{ +Summary of relevant aspects of the DPDD \citep{LSE-163}, including latency considerations and user access. +Describe key requirements that drive technical decisions. +} + + + + +\subsection{Alerts} \label{sec:alerts} + +Alert packets are the primary data product for time-critical LSST science. +Each alert packet is triggered by a single positive or negative detection on a difference image. +This measurement is packaged with a variety of contextual information in order to enable downstream users to quickly identify events of interest without requiring followup queries to Rubin databases. + +Each alert packet contains + +\begin{itemize} + \item The triggering DIASource record + \item The corresponding DIAObject or SSObject record + \item Any DIASource and DIAForcedSource records that exist, and difference image noise estimates where they do not, taken from the previous 12 months. + \item Cutout images of the science, template, and difference images. +\end{itemize} +\eric{confirm contents} + +The cutout images are packaged in the FITS format using the astropy \texttt{CCDData} class and include mask and variance planes, an estimate of the PSF, a World Coordinate System, and other metadata. +Most cutouts are 30$\times$30 pixels but are expanded to a larger size for extended sources. +The maximimum cutout size of 102$\times$102 pixels allows cutouts to contain trailed Near Earth Asteroids traveling slower than ten degrees per day while avoiding the performance impacts that would result from larger cutouts. + +Alerts are transmitted and stored in Apache Avro\footnote{\url{https://avro.apache.org/}} format, a strongly-schemaed compact binary serialization. +However, the Avro standard includes its schemas as uncompressed ASCII. +To conserve bandwidth, we therefore send alerts without including the schemas required to deserialize them. +Instead, the schemas are stored in a separate schema registry and are referenced by a unique identifier in the alert packet. +\citep{DMTN-093} describes the alert packaging and schema versioning used in the alerts. +Avro alert schemas are programatically generated from the database schema definitions stored in the \texttt{lsst.sdm.schemas} package and can be found in the \texttt{lsst.alert.packet} repository\footnote{\url{https://github.com/lsst/alert_packet/}}. + +\subsection{Images} \label{sec:images} + +\subsection{Prompt Catalogs} \label{sec:catalogs} + +\eric{point to schema browser. forward reference the PPDB.} + +\subsection{Data Rights} \label{sec:data-rights} + +\citet{RDO-013} describes the Rubin Data Rights policies. +In brief, alert packets are world-public and can be freely shared with anyone. +Similarly, the contents of the PPDB are freely sharable, although direct access to the PPDB is restricted to Data Rights holders. +All other data products are proprietary. \ No newline at end of file diff --git a/data_services.tex b/data_services.tex new file mode 100644 index 0000000..ecfef5e --- /dev/null +++ b/data_services.tex @@ -0,0 +1,46 @@ +\section{Data Access Services} \label{sec:services} + +\subsection{Community Alert Brokers} \label{sec:brokers} + +Near real-time access to alert packets is available through services provided by community alert brokers. +Due to limited bandwidth from the USDF, seven brokers were selected through a proposal process \citep{LDM-612} to receive the alert stream directly from the alert distribution system at the USDF (\S \ref{sec:alertdist}): + +\begin{itemize} + \item ALeRCE: Automatic Learning for the Rapid Classification of Events \citep{Forster:21:ALeRCE} + \item AMPEL: Alert Management, Photometry, and Evaluation of Light Curves \citep{Nordin:19:AMPEL} + \item ANTARES: Arizona-NOIRLab Temporal Analysis and Response to Events System \citep{Matheson:21:ANTARES} + \item Babamul \citep{2025arXiv251100164J} + \item Fink \citep{Moller:21:FINK} + \item Lasair \citep{Smith:19:Lasair} + \item Pitt-Google +\end{itemize} + +As alert packets are world-public and freely sharable (\S \ref{sec:data-rights}), additional brokers and other services may operate ``downstream'' by connecting to full or filtered streams provided by the upstream brokers. +As of this writing, downstream brokers include + +\begin{itemize} + \item Point of Interest Broker + \item SNAPS: Solar System Notification Alert Processing System +\end{itemize} + +\eric{check broker sites if there are newer citations} + +\subsection{Alert Filtering Service} + +\melissa{ANTARES system, community alert filters} + +\subsection{Rubin Science Platform} + +\gregory{todo} + +\subsubsection{Alert Archive} + +\subsubsection{Image Services} + +\subsubsection{Butler Data Products} + +\subsubsection{Prompt Products Database} \label{sec:ppdb} + +\subsection{IDACs} + +\todo{confirm IDAC section} \ No newline at end of file diff --git a/intro.tex b/intro.tex new file mode 100644 index 0000000..43deeb1 --- /dev/null +++ b/intro.tex @@ -0,0 +1,38 @@ +\section{Introduction} + +Time-domain measurements are among of the foundational techniques of astronomical science: +repeated observations of the night sky enable discovery of transient, variable, and moving objects. +The cadence of the observations, the intrinsic luminosities and timescales of the phenomena, and the spatial volume probed by the observations set the rate of discovery \citep{Bellm:16:Cadences}. +In recent years, scientists have conducted dedicated optical time-domain surveys with large CCD mosaic cameras (e.g., PanSTARRS, DECam, ZTF, HSC, Gaia, Kepler, TESS) as well as distributed telescope networks (e.g., ASAS-SN, ATLAS, KMT-Net, \eric{more}). +These have yielded large new samples of supernovae, variable stars, active galactic nuclei, and solar system objects. +Additionally, they have uncovered the first exemplars of rare new classes of objects \eric{add some examples: FBOTS, ISOs, asteroids interior to venus, ...}. + +Frequently, time-critical followup observations are necessary to classify and characterize objects discovered in time-domain surveys. +Historically, human-composed circulars or telegrams were used to dissemenate discoveries to the wider community. +The increasing rate of transient candidates as well as the desire for rapid automated followup motivated machine-readable alternatives \eric{VOEvent, GCN, SciMMA/HopSkotch, TNS}. +These trends culminated in the public ``alert stream'' paradigm employed by ZTF \citep{Patterson:19:ZTFAlertDistribution}, in which hundreds of thousands of unfiltered difference image sources are shipped along with historical lightcurves and image cutouts to third-party alert brokers for classification, filtering, and followup. +This approach has enabled fully automated identification, reporting, and followup of supernova candidates \eric{cites}. + +The Legacy Survey of Space and Time (LSST) to be conducted by the Vera C.\ Rubin Observatory promises an order of magnitude increase in transient discovery. +Rubin's large collecting area, wide field of view, and fast readout and slew will deliver nearly a thousand \eric{confirm} exposures across a wide swath of the Southern Hemisphere sky to unprecedented depths. +This capability motivated the development of a rapid data processing pipeline to identify and publicize time-variable phenomena in LSST images: the Rubin Alert Production System (AP). +Along with the annual Data Release Processing (DRP), these productions make use of the Rubin Science Pipelines software as well as the larger systems and infrastructure of Rubin Data Management (DM). +\eric{cites throughout} + +In this paper, we describe the design, implementation, and initial performance of the Rubin Alert Production system. +In \S \ref{sec:observations} we detail the alert-producing observations taken to date. +\S \ref{sec:data-products} presents the science data products resulting from alert production and \S \ref{sec:pipeline} describes the data processing pipelines which produce them\footnote{ +Discovery of new solar system objects from daytime processing of the prompt data products is not described here but is presented in \todo{appropriate citation}.}. +We summarize the data processing environment and infrastructure in \S \ref{sec:data-processing}. +We characterize the early scientific and technical performance of alert production in \S \ref{sec:performance}. +In \S \ref{sec:services} and \S \ref{sec:support} we describe how data are served to scientists and the support channels available to users. +We conclude in \S \ref{sec:conclusion}. + + +%\note{See discussion on common material and timing considerations in \cite{PSTN-020}.} +%\note{This outline proceeds assuming this paper is being produced at the end of commissioning/early operations, and some form of routine alert production is happening in operations based on commissioning-derived or incremental templates. +%If only a small amount of LSSTCam commissioning data is available and is being released, only a minimal paper decribing the pipelines used to process that data would useful, and a more complete AP reference should come later.} +%\note{Information about the processing environment is presumed to be covered in \cite{PSTN-018}. +%} +%\note{SS Processing falls under Prompt Processing but detailed discussion is deferred to \cite{PSTN-025}.} + diff --git a/local.bib b/local.bib index e69de29..aa44719 100644 --- a/local.bib +++ b/local.bib @@ -0,0 +1,186 @@ + +@article{Duev:19:Braii, + author = {{Duev}, Dmitry A. and {Mahabal}, Ashish and {Masci}, Frank J. and {Graham}, Matthew J. and {Rusholme}, Ben and {Walters}, Richard and {Karmarkar}, Ishani and {Frederick}, Sara and {Kasliwal}, Mansi M. and {Rebbapragada}, Umaa and {Ward}, Charlotte}, + journal = {\mnras}, + month = nov, + number = {3}, + pages = {3582-3590}, + title = {{Real-bogus classification for the Zwicky Transient Facility using deep learning}}, + volume = {489}, + year = 2019} + +@article{Goldstein:15:DESRealBogus, + author = {{Goldstein}, D.~A. and {D'Andrea}, C.~B. and {Fischer}, J.~A. and {Foley}, R.~J. and {Gupta}, R.~R. and {Kessler}, R. and {Kim}, A.~G. and {Nichol}, R.~C. and {Nugent}, P.~E. and {Papadopoulos}, A. and {Sako}, M. and {Smith}, M. and {Sullivan}, M. and {Thomas}, R.~C. and {Wester}, W. and {Wolf}, R.~C. and {Abdalla}, F.~B. and {Banerji}, M. and {Benoit-L{\'e}vy}, A. and {Bertin}, E. and {Brooks}, D. and {Carnero Rosell}, A. and {Castander}, F.~J. and {da Costa}, L.~N. and {Covarrubias}, R. and {DePoy}, D.~L. and {Desai}, S. and {Diehl}, H.~T. and {Doel}, P. and {Eifler}, T.~F. and {Fausti Neto}, A. and {Finley}, D.~A. and {Flaugher}, B. and {Fosalba}, P. and {Frieman}, J. and {Gerdes}, D. and {Gruen}, D. and {Gruendl}, R.~A. and {James}, D. and {Kuehn}, K. and {Kuropatkin}, N. and {Lahav}, O. and {Li}, T.~S. and {Maia}, M.~A.~G. and {Makler}, M. and {March}, M. and {Marshall}, J.~L. and {Martini}, P. and {Merritt}, K.~W. and {Miquel}, R. and {Nord}, B. and {Ogando}, R. and {Plazas}, A.~A. and {Romer}, A.~K. and {Roodman}, A. and {Sanchez}, E. and {Scarpine}, V. and {Schubnell}, M. and {Sevilla-Noarbe}, I. and {Smith}, R.~C. and {Soares-Santos}, M. and {Sobreira}, F. and {Suchyta}, E. and {Swanson}, M.~E.~C. and {Tarle}, G. and {Thaler}, J. and {Walker}, A.~R.}, + journal = {\aj}, + month = sep, + pages = {82}, + title = {{Automated Transient Identification in the Dark Energy Survey}}, + volume = 150, + year = 2015} + +@article{Wright:15:PS1ML, + author = {{Wright}, D.~E. and {Smartt}, S.~J. and {Smith}, K.~W. and {Miller}, P. and {Kotak}, R. and {Rest}, A. and {Burgett}, W.~S. and {Chambers}, K.~C. and {Flewelling}, H. and {Hodapp}, K.~W. and {Huber}, M. and {Jedicke}, R. and {Kaiser}, N. and {Metcalfe}, N. and {Price}, P.~A. and {Tonry}, J.~L. and {Wainscoat}, R.~J. and {Waters}, C.}, + journal = {\mnras}, + month = may, + pages = {451-466}, + title = {{Machine learning for transient discovery in Pan-STARRS1 difference imaging}}, + volume = 449, + year = 2015} + +@article{Brink:13:RB2, + author = {{Brink}, H. and {Richards}, J.~W. and {Poznanski}, D. and {Bloom}, J.~S. and {Rice}, J. and {Negahban}, S. and {Wainwright}, M.}, + journal = {\mnras}, + month = oct, + pages = {1047-1060}, + title = {{Using machine learning for discovery in synoptic survey imaging data}}, + volume = 435, + year = 2013} + +@article{Bloom:12:RealBogus, + author = {{Bloom}, J.~S. and {Richards}, J.~W. and {Nugent}, P.~E. and {Quimby}, R.~M. and {Kasliwal}, M.~M. and {Starr}, D.~L. and {Poznanski}, D. and {Ofek}, E.~O. and {Cenko}, S.~B. and {Butler}, N.~R. and {Kulkarni}, S.~R. and {Gal-Yam}, A. and {Law}, N.}, + journal = {\pasp}, + month = nov, + pages = {1175}, + title = {{Automating Discovery and Classification of Transients and Variable Stars in the Synoptic Survey Era}}, + volume = 124, + year = 2012} + +@article{Bailey:07:NearbySNFML, + author = {{Bailey}, S. and {Aragon}, C. and {Romano}, R. and {Thomas}, R.~C. and {Weaver}, B.~A. and {Wong}, D.}, + journal = {\apj}, + month = aug, + pages = {1246-1253}, + title = {{How to Find More Supernovae with Less Work: Object Classification Techniques for Difference Imaging}}, + volume = 665, + year = 2007} + + +@article{Sedaghat:18:Transinet, + author = {{Sedaghat}, Nima and {Mahabal}, Ashish}, + journal = {\mnras}, + month = jun, + number = {4}, + pages = {5365-5376}, + title = {{Effective image differencing with convolutional neural networks for real-time transient hunting}}, + volume = {476}, + year = 2018} + + +i +@article{Acero-Cuellar:23:NoDifference, + author = {{Acero-Cuellar}, Tatiana and {Bianco}, Federica and {Dobler}, Gregory and {Sako}, Masao and {Qu}, Helen and {LSST Dark Energy Science Collaboration}}, + journal = {\aj}, + month = sep, + number = {3}, + pages = {115}, + title = {{What's the Difference? The Potential for Convolutional Neural Networks for Transient Detection without Template Subtraction}}, + volume = {166}, + year = 2023} + + +@article{Bellm:16:Cadences, + author = {{Bellm}, E.~C.}, + journal = {\pasp}, + month = aug, + number = 8, + pages = {084501}, + title = {{Volumetric Survey Speed: A Figure of Merit for Transient Surveys}}, + volume = 128, + year = 2016} + + +@article{Patterson:19:ZTFAlertDistribution, + author = {{Patterson}, M.~T. and {Bellm}, E.~C. and {Rusholme}, B. and {Masci}, F.~J. and {Juric}, M. and {Krughoff}, K.~S. and {Golkhou}, V.~Z. and {Graham}, M.~J. and {Kulkarni}, S.~R. and {Helou}, G. and {Zwicky Transient Facility Collaboration}}, + journal = {\pasp}, + month = jan, + number = 1, + pages = {018001}, + title = {{The Zwicky Transient Facility Alert Distribution System}}, + volume = 131, + year = 2019} + +@ARTICLE{2026arXiv260319541F, + author = {{Findeisen}, Krzysztof and {Lim}, Kian-Tat and {Speck}, Dan and {Chiang}, Hsin-Fang and {Howard}, Erin Leigh and {Sullivan}, Ian S. and {Bellm}, Eric C.}, + title = "{The Vera C. Rubin Observatory Prompt Processing System}", + journal = {arXiv e-prints}, + keywords = {Instrumentation and Methods for Astrophysics}, + year = 2026, + month = mar, + eid = {arXiv:2603.19541}, + pages = {arXiv:2603.19541}, + doi = {10.48550/arXiv.2603.19541}, +archivePrefix = {arXiv}, + eprint = {2603.19541}, + primaryClass = {astro-ph.IM}, + adsurl = {https://ui.adsabs.harvard.edu/abs/2026arXiv260319541F}, + adsnote = {Provided by the SAO/NASA Astrophysics Data System} +} + + +@article{Forster:21:ALeRCE, + author = {{F{\"o}rster}, F. and {Cabrera-Vives}, G. and {Castillo-Navarrete}, E. and {Est{\'e}vez}, P.~A. and {S{\'a}nchez-S{\'a}ez}, P. and {Arredondo}, J. and {Bauer}, F.~E. and {Carrasco-Davis}, R. and {Catelan}, M. and {Elorrieta}, F. and {Eyheramendy}, S. and {Huijse}, P. and {Pignata}, G. and {Reyes}, E. and {Reyes}, I. and {Rodr{\'\i}guez-Mancini}, D. and {Ruz-Mieres}, D. and {Valenzuela}, C. and {{\'A}lvarez-Maldonado}, I. and {Astorga}, N. and {Borissova}, J. and {Clocchiatti}, A. and {De Cicco}, D. and {Donoso-Oliva}, C. and {Hern{\'a}ndez-Garc{\'\i}a}, L. and {Graham}, M.~J. and {Jord{\'a}n}, A. and {Kurtev}, R. and {Mahabal}, A. and {Maureira}, J.~C. and {Mu{\~n}oz-Arancibia}, A. and {Molina-Ferreiro}, R. and {Moya}, A. and {Palma}, W. and {P{\'e}rez-Carrasco}, M. and {Protopapas}, P. and {Romero}, M. and {Sabatini-Gacitua}, L. and {S{\'a}nchez}, A. and {San Mart{\'\i}n}, J. and {Sep{\'u}lveda-Cobo}, C. and {Vera}, E. and {Vergara}, J.~R.}, + journal = {\aj}, + month = may, + number = {5}, + pages = {242}, + title = {{The Automatic Learning for the Rapid Classification of Events (ALeRCE) Alert Broker}}, + volume = {161}, + year = 2021} + + +@article{Nordin:19:AMPEL, + author = {{Nordin}, J. and {Brinnel}, V. and {van Santen}, J. and {Bulla}, M. and {Feindt}, U. and {Franckowiak}, A. and {Fremling}, C. and {Gal-Yam}, A. and {Giomi}, M. and {Kowalski}, M. and {Mahabal}, A. and {Miranda}, N. and {Rauch}, L. and {Rigault}, M. and {Schulze}, S. and {Sollerman}, J. and {Stein}, R. and {Yaron}, O. and {van Velzen}, S. and {Ward}, C.}, + journal = {arXiv e-prints}, + month = apr, + title = {{Transient processing and analysis using $\texttt{AMPEL}$: Alert Management, Photometry and Evaluation of Lightcurves}}, + year = 2019} + + +@article{Matheson:21:ANTARES, + author = {{Matheson}, Thomas and {Stubens}, Carl and {Wolf}, Nicholas and {Lee}, Chien-Hsiu and {Narayan}, Gautham and {Saha}, Abhijit and {Scott}, Adam and {Soraisam}, Monika and {Bolton}, Adam S. and {Hauger}, Benjamin and {Silva}, David R. and {Kececioglu}, John and {Scheidegger}, Carlos and {Snodgrass}, Richard and {Aleo}, Patrick D. and {Evans-Jacquez}, Eric and {Singh}, Navdeep and {Wang}, Zhe and {Yang}, Shuo and {Zhao}, Zhenge}, + journal = {\aj}, + month = mar, + number = {3}, + pages = {107}, + title = {{The ANTARES Astronomical Time-domain Event Broker}}, + volume = {161}, + year = 2021} + +@ARTICLE{2025arXiv251100164J, + author = {{Jegou du Laz}, Theophile and {Coughlin}, Michael W. and {Bachant}, Peter and {Simones}, Jacob E. and {Culino}, Thomas and {Le Calloch}, Antoine and {Sharma Chaudhary}, Sushant and {Hall}, Xander J. and {Barna}, Tyler and {Warshofsky}, Daniel and {Graham}, Matthew and {Kasliwal}, Mansi M. and {Mahabal}, Ashish and {Bloom}, Joshua S. and {Palmese}, Antonella and {Masci}, Frank J. and {Groom}, Steven L. and {Dekany}, Richard and {Riddle}, Reed L. and {Helou}, George}, + title = "{BOOM and Babamul: a real-time, multi-survey, optical alert broker system operating at scale}", + journal = {arXiv e-prints}, + keywords = {Instrumentation and Methods for Astrophysics}, + year = 2025, + month = oct, + eid = {arXiv:2511.00164}, + pages = {arXiv:2511.00164}, + doi = {10.48550/arXiv.2511.00164}, +archivePrefix = {arXiv}, + eprint = {2511.00164}, + primaryClass = {astro-ph.IM}, + adsurl = {https://ui.adsabs.harvard.edu/abs/2025arXiv251100164J}, + adsnote = {Provided by the SAO/NASA Astrophysics Data System} +} + + +@article{Moller:21:FINK, + author = {{M{\"o}ller}, Anais and {Peloton}, Julien and {Ishida}, Emille E.~O. and {Arnault}, Chris and {Bachelet}, Etienne and {Blaineau}, Tristan and {Boutigny}, Dominique and {Chauhan}, Abhishek and {Gangler}, Emmanuel and {Hernandez}, Fabio and {Hrivnac}, Julius and {Leoni}, Marco and {Leroy}, Nicolas and {Moniez}, Marc and {Pateyron}, Sacha and {Ramparison}, Adrien and {Turpin}, Damien and {Ansari}, R{\'e}za and {Allam}, Tarek, Jr. and {Bajat}, Armelle and {Biswas}, Biswajit and {Boucaud}, Alexandre and {Bregeon}, Johan and {Campagne}, Jean-Eric and {Cohen-Tanugi}, Johann and {Coleiro}, Alexis and {Dornic}, Damien and {Fouchez}, Dominique and {Godet}, Olivier and {Gris}, Philippe and {Karpov}, Sergey and {Nebot Gomez-Moran}, Ada and {Neveu}, J{\'e}r{\'e}my and {Plaszczynski}, Stephane and {Savchenko}, Volodymyr and {Webb}, Natalie}, + journal = {\mnras}, + month = mar, + number = {3}, + pages = {3272-3288}, + title = {{FINK, a new generation of broker for the LSST community}}, + volume = {501}, + year = 2021} + + +@article{Smith:19:Lasair, + author = {{Smith}, K.~W. and {Williams}, R.~D. and {Young}, D.~R. and {Ibsen}, A. and {Smartt}, S.~J. and {Lawrence}, A. and {Morris}, D. and {Voutsinas}, S. and {Nicholl}, M.}, + journal = {Research Notes of the American Astronomical Society}, + month = jan, + number = 1, + pages = {26}, + title = {{Lasair: The Transient Alert Broker for LSST:UK}}, + volume = 3, + year = 2019} diff --git a/lsst-texmf b/lsst-texmf index 9e89ee2..3615bf7 160000 --- a/lsst-texmf +++ b/lsst-texmf @@ -1 +1 @@ -Subproject commit 9e89ee2be24c94427b3e1a82890675b48cfd3111 +Subproject commit 3615bf77205b2832915cbf08a16fdcb1806c1abb diff --git a/manualParameters.csv b/manualParameters.csv new file mode 100644 index 0000000..2adbf96 --- /dev/null +++ b/manualParameters.csv @@ -0,0 +1,2 @@ +AlertIntegrationStartDate,2025-09-05, +PublicAlertsStartDate,2026-02-24, \ No newline at end of file diff --git a/observations.tex b/observations.tex new file mode 100644 index 0000000..b0b7fa6 --- /dev/null +++ b/observations.tex @@ -0,0 +1,23 @@ +\section{Observations} \label{sec:observations} + +\eric{brief summary of observations: commissioning, first alerts, ongoing data taking. +What we expect to be in the PPDB.} + +Unlike Rubin's Data Releases (e.g., DP1, DP2 \eric{cite}), which contain a fixed dataset acquired before the processing begins, Alert Production provides a continuous, ongoing release of data obtained by Rubin that is released within seconds to days of observation. +During commissioning, alerts were transmitted to community alert brokers for technical integration tests beginning on \AlertIntegrationStartDate. +Since data from this period are available in alert history and the Prompt Products Database (PPDB; \S \ref{sec:ppdb}) \eric{image release?}, we include them in this paper, although they were not available publicly at the time of processing. +Real-time public alert release began on \PublicAlertsStartDate. +In this paper we include data obtained until \eric{final date}. + +During the commissioning and early operations phases, scientific data taking was intermixed with ongoing technical commisisoning activities that did not produce data suitable for alert production. +Calibration, delivered image quality, and pipeline processing were under substantial development. +Additionally, template images were only available initially for a small subset of the sky area. +Accordingly, the quantity and quality of early alerts produced are not representive of steady-state performance, and not all data products are available. +We specify the range of data included in our performance analysis (\S \ref{sec:performance}). + + + + +\eric{consider a figure or histogram showing the alert rate over time, perhaps a sky plot of alert density?? or colored by earliest alert date?} + +\eric{scheduler discussion/citation? link to survey-strategy.lsst.io? mention status dashboards at prompt-products.lsst.io?} \ No newline at end of file diff --git a/parameters.tex b/parameters.tex new file mode 100644 index 0000000..a75b93a --- /dev/null +++ b/parameters.tex @@ -0,0 +1,9 @@ +% These parameters are automatically generated by scripts/ap_parameters.py. +% Do NOT manually edit this file. +% If you need to change/add a parameter, please edit the ap_parameters.py script and re-run it as a +% command-line executable. +% If you need to manually add a parameter because it can't be readily generated from data (such as a url) +% then please add it to the manualParameters.csv file. + +\newcommand{\AlertIntegrationStartDate}{2025-09-05\xspace} +\newcommand{\PublicAlertsStartDate}{2026-02-24\xspace} diff --git a/performance.tex b/performance.tex new file mode 100644 index 0000000..4ccd547 --- /dev/null +++ b/performance.tex @@ -0,0 +1,28 @@ +\section{Initial Performance} \label{sec:performance} + +\subsection{Template Properties} + +\subsection{Alert Rate} \label{sec:rates} + +\subsection{Alert Purity} \label{sec:purity} + +Raw and after ML scoring + + +\subsection{Alert Completeness} + +\subsection{Alert Latency} + +\subsection{Photometric Precision} + +\subsection{Astrometric Precision} + +\subsection{Association Accuracy} + +\subsection{Association of Known Solar System Objects} + +\jake{completeness, astrometric accuracy} + +\subsection{Direct Source Catalogs} + +\subsection{Crowded Field Performance} \ No newline at end of file diff --git a/pipeline.tex b/pipeline.tex new file mode 100644 index 0000000..74f8e6d --- /dev/null +++ b/pipeline.tex @@ -0,0 +1,103 @@ + +\section{Pipeline} \label{sec:pipeline} + +The Alert Production Pipelines are a specialized configuration of the LSST Science Pipelines. +\citet{10.71929/rubin/2570545} provides a general overview of their design and capabilities. +In this section we describe the detailed implementation used for Alert Production. + +%\note{following LDM-151} +\todo{consider a (simplified?) pipeline graph} + +\todo{emphasize not a static/permanent release; evolving over time} + +\subsection{Template Generation} + +%\note{Pre-DR1 this may require description independent of DRP.} +% +%High-level discussion of DCR correction could go here. + +\subsection{Preload} \label{sec:preload} + +\subsection{Single Frame Processing} + +ISR, PSF and Background fitting, photometric and astrometric calibration. Likely major commonalities with DRP; perhaps point to DP2 paper and describe any differences + +\subsection{Image Differencing} + +including background subtraction + +\subsection{Source Detection and Measurement} + +Including discussion of point source, dipole, and trail detection on difference images. + +%\note{Algorithmic basics could be discussed elsewhere} + +\subsection{Initial Filtering} \label{sec:filtering} + +\todo{glint detection} + +\todo{satellite filtering} + +\texttt{FilterDiaSourceCatalogTask} in \texttt{lsst.ap.association} + +\subsection{Reliability scoring} + +Image-differencing searches for transients typically contend with high rates of false positives. +Among the raw detections, artifacts may dominate real astrophysical sources by an order of magnitude. +These may be due to imperfectly-corrected instrument signatures, astrometric mismatches between the template and the science image, cosmic rays, or algorithmic failures in the differencing. +Modern surveys employ machine-learned classifiers to winnow these candidates \citep[e.g.,][]{Bailey:07:NearbySNFML,Bloom:12:RealBogus,Brink:13:RB2,Wright:15:PS1ML,Goldstein:15:DESRealBogus,Duev:19:Braii}. +These have steadily improved in performance and sophistication as the classifiers transitioned from Random Forest models trained on manually-constructed feature sets to deep neural networks trained directly on image pixels. +Some approaches have combined both detection and scoring without using the difference image \citep{Sedaghat:18:Transinet,Acero-Cuellar:23:NoDifference}. + +\citet{DMTN-337} describes the training and performance of the reliability classifier versions deployed in Alert Production. +Here we summarize the currently-deployed model \eric{version}. +Further improvements are planned as more data is taken. +\todo{overview of the algorithm, training, and performance of the ML spuriousness score; detailed discussion likely deferred to separate paper} + +\subsection{Catalog Transformation} + +\texttt{TransformDiaSourceCatalogTask} in \texttt{lsst.ap.association} + +\eric{schema tables?} + +\subsection{Source Association} + +Due to AP's real-time nature, it is not possible to perform \textit{post-facto} source association as in the annual data releases. +Instead, spatial association is performed on-the-fly by the \texttt{DiaPipelineTask} within the \texttt{lsst.ap.association} package. +A dedicated Alert Production Database (APDB; \S \ref{sec:apdb}) holds the current state of the system. + +During the preload step (\S \ref{sec:preload}), DIAObjects and SSObjects overlapping the expected field of view of the image are stored in the local prompt processing worker butler repository. +The new DIASources are first spatially associated with the existing DIAObjects by finding the closest match within a maximum distance of one arcsecond. +Pairs with the closest spatial separations are joined first to minimize misassociations. +When a match is found, the DIASource is added to the DIAObject's list of measurements. +Unassociated DIASources are then spatially associated with the predicted positions of the SSObjects at the time of the exposure. +For DIASources with no matching DIAObject or SSObject, a new DIAObject is created and the DIASource is added to it. + + + +%\note{does \cite{PSTN-045} describe the APDB/PPDB implementation?} + +\subsection{Alert Generation} + +Each DIASource not filtered for quality purposes (\S \ref{sec:filtering}) produces an alert (\S \ref{sec:alerts}). + +\eric{sizing estimates here or in initial performance section? if not, collapse into next section} + +\subsection{Alert Distribution} \label{sec:alertdist} + +We use the Kafka distributed streaming platform\footnote{\url{https://kafka.apache.org/}} to transmit alerts to brokers. +Kafka provides a fault-tolerant, low-overhead mechanism to transmit large volumes of data to multiple consumers and is widely used both in industry and in astronomy. +\eric{cite patterson, GCN, Hopskotch, other Rubin usage...} + +The Alert Distribution Kafka system is deployed at the USDF using Kubernetes. +\bri{details of the deployment} +\bri{schema management} + +Because of the large size of the alert packets and the short latency requirement, bandwidth out of the US Data Facility (USDF) at SLAC limits the number of direct recipients of the full alert stream. +Seven community alert brokers receive the full alert stream directly from the USDF (\S \ref{sec:brokers}). + +\subsection{Forced Photometry} + +\subsection{Source Injection} + +\subsection{Metrics} \ No newline at end of file diff --git a/processing.tex b/processing.tex new file mode 100644 index 0000000..ec73ca0 --- /dev/null +++ b/processing.tex @@ -0,0 +1,16 @@ +\section{Data Processing} \label{sec:data-processing} + + +\subsection{Prompt Processing Framework} \label{sec:pp} + +\citep{2026arXiv260319541F} + +\subsection{Alert Production Database} \label{sec:apdb} + +\subsection{Metrics and Monitoring} + +\subsection{Morning Batch Processing} + +\eric{deduplication} + +\ian{catchup, single frame detection, visit-level consolidation, metrics} \ No newline at end of file diff --git a/scripts/ap_parameters.py b/scripts/ap_parameters.py new file mode 100644 index 0000000..bc7f9d4 --- /dev/null +++ b/scripts/ap_parameters.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python +# +# This file is part of rtn-114 +# +# This product includes software developed by the LSST Project +# (https://www.lsst.org). +# See the COPYRIGHT file at the top-level directory of this distribution +# for details of code ownership. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# Script for generating the various latex parameters referred to in the AP +# paper. + +# Contact Authors: Eric Bellm, Ian Sullivan + +import csv +import numpy as np + +# Function to round to N significant figures +def round_sf(x, sig=3): + return np.round(x, sig - int(np.floor(np.log10(abs(x)))) - 1) + +def formatParameter(params, name): + ''' + Formats a named parameter ready for write-out to tex file. + ''' + + value = params[0][name] + if isinstance(value, float): + value = f'{value:.3f}'.rstrip('0').rstrip('.') + + if name in params[1]: + unit = params[1][name] + if unit[0:7] not in ['\\arcsec', '\\arcmin', '\\degree']: + unit = f'\\xspace {unit}' + else: + if '.' in value: + match unit[0:7]: + case '\\arcsec': + value = value.replace('.','\\farcs') + unit = unit.replace('\\arcsec', '') + case '\\arcmin': + value = value.replace('.','\\farcm') + unit = unit.replace('\\arcmin', '') + case '\\degree': + value = value.replace('.','\\fdg') + unit = unit.replace('\\degree', '') + else: + unit = '' + + return f'\\newcommand{{\\{name}}}{{{value}{unit}\\xspace}}\n' + +def addParameter(params, name, value, unit=None, **kwargs): + + ''' + Adds a named parameter to the params tuple of dictionaries. + + Ensures that the namings between the values and units + dictionaries are consistent. + ''' + if 'sig' in kwargs: + value = round_sf(value, **kwargs) + + params[0][name] = value + if unit: + params[1][name] = unit + + return params + + +#-------- Manually added parameters --------# +# A small number of parameters cannot be readily generated autimatically. +# These are kept in the manualParameters.csv file. +def manualParameters(params): + print('Adding manual parameters...') + with open('../manualParameters.csv', newline='') as f: + reader = csv.reader(f, delimiter=',') + for line in reader: + if line[2] == '': + line[2]=None + params = addParameter(params, line[0], line[1], unit=line[2]) + + return params + +if __name__ == "__main__": + + params = (dict(), dict()) + + # Using functions allows for easier debugging. + params = manualParameters(params) + + + #-------- Write the parameters to file --------# + boilerPlate = '''% These parameters are automatically generated by scripts/ap_parameters.py. +% Do NOT manually edit this file. +% If you need to change/add a parameter, please edit the ap_parameters.py script and re-run it as a +% command-line executable. +% If you need to manually add a parameter because it can't be readily generated from data (such as a url) +% then please add it to the manualParameters.csv file.\n +''' + with open("../parameters.tex", "w") as f: + f.write(boilerPlate) + for name in params[0]: + f.write(formatParameter(params, name)) + f.close() diff --git a/support.tex b/support.tex new file mode 100644 index 0000000..e292e7c --- /dev/null +++ b/support.tex @@ -0,0 +1,3 @@ +\section{Support for Community Science} \label{sec:support} + +\melissa{prompt-products.lsst.io, pipelines.lsst.io, community.lsst.io} \ No newline at end of file