diff --git a/.gitignore b/.gitignore index 7c345a38..4f028b2d 100755 --- a/.gitignore +++ b/.gitignore @@ -24,3 +24,6 @@ docs/sphinx/content/API /hypernets_processor/etc/working_directory.txt /hypernets_processor/etc/processor.config /data/calibration_files_ascii +.eggs/ +.vscode/ +__pycache__/ diff --git a/.readthedocs.yml b/.readthedocs.yml index d453251a..33d37de7 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -9,7 +9,7 @@ version: 2 build: os: ubuntu-22.04 tools: - python: "3.9" + python: "3.12" # Build documentation in the docs/ directory with Sphinx sphinx: diff --git a/data/postprocessing/irradiance_data/GHNA_clear_sky_aod0.0.nc b/data/postprocessing/irradiance_data/GHNA_clear_sky_aod0.0.nc new file mode 100644 index 00000000..12c07d60 Binary files /dev/null and b/data/postprocessing/irradiance_data/GHNA_clear_sky_aod0.0.nc differ diff --git a/data/postprocessing/irradiance_data/GHNA_clear_sky_aod0.0_irrwav.nc b/data/postprocessing/irradiance_data/GHNA_clear_sky_aod0.0_irrwav.nc new file mode 100644 index 00000000..da1d7ff6 Binary files /dev/null and b/data/postprocessing/irradiance_data/GHNA_clear_sky_aod0.0_irrwav.nc differ diff --git a/data/postprocessing/irradiance_data/GHNA_clear_sky_medianaod.nc b/data/postprocessing/irradiance_data/GHNA_clear_sky_medianaod.nc new file mode 100644 index 00000000..007692b9 Binary files /dev/null and b/data/postprocessing/irradiance_data/GHNA_clear_sky_medianaod.nc differ diff --git a/data/postprocessing/irradiance_data/GHNA_clear_sky_medianaod_irrwav.nc b/data/postprocessing/irradiance_data/GHNA_clear_sky_medianaod_irrwav.nc new file mode 100644 index 00000000..5be51ab7 Binary files /dev/null and b/data/postprocessing/irradiance_data/GHNA_clear_sky_medianaod_irrwav.nc differ diff --git a/data/postprocessing/irradiance_data/JAES_clear_sky_aod0.0.nc b/data/postprocessing/irradiance_data/JAES_clear_sky_aod0.0.nc new file mode 100644 index 00000000..c5a6d3e0 Binary files /dev/null and b/data/postprocessing/irradiance_data/JAES_clear_sky_aod0.0.nc differ diff --git a/data/postprocessing/irradiance_data/JAES_clear_sky_aod0.0_irrwav.nc b/data/postprocessing/irradiance_data/JAES_clear_sky_aod0.0_irrwav.nc new file mode 100644 index 00000000..78d76b7c Binary files /dev/null and b/data/postprocessing/irradiance_data/JAES_clear_sky_aod0.0_irrwav.nc differ diff --git a/data/postprocessing/irradiance_data/JAES_clear_sky_medianaod.nc b/data/postprocessing/irradiance_data/JAES_clear_sky_medianaod.nc new file mode 100644 index 00000000..c7bf2fcb Binary files /dev/null and b/data/postprocessing/irradiance_data/JAES_clear_sky_medianaod.nc differ diff --git a/data/postprocessing/irradiance_data/JAES_clear_sky_medianaod_irrwav.nc b/data/postprocessing/irradiance_data/JAES_clear_sky_medianaod_irrwav.nc new file mode 100644 index 00000000..8ef55897 Binary files /dev/null and b/data/postprocessing/irradiance_data/JAES_clear_sky_medianaod_irrwav.nc differ diff --git a/data/postprocessing/irradiance_data/JSIT_clear_sky_aod0.0.nc b/data/postprocessing/irradiance_data/JSIT_clear_sky_aod0.0.nc new file mode 100644 index 00000000..f71615f1 Binary files /dev/null and b/data/postprocessing/irradiance_data/JSIT_clear_sky_aod0.0.nc differ diff --git a/data/postprocessing/irradiance_data/JSIT_clear_sky_aod0.0_irrwav.nc b/data/postprocessing/irradiance_data/JSIT_clear_sky_aod0.0_irrwav.nc new file mode 100644 index 00000000..723ed741 Binary files /dev/null and b/data/postprocessing/irradiance_data/JSIT_clear_sky_aod0.0_irrwav.nc differ diff --git a/data/postprocessing/irradiance_data/JSIT_clear_sky_medianaod.nc b/data/postprocessing/irradiance_data/JSIT_clear_sky_medianaod.nc new file mode 100644 index 00000000..bf922f87 Binary files /dev/null and b/data/postprocessing/irradiance_data/JSIT_clear_sky_medianaod.nc differ diff --git a/data/postprocessing/irradiance_data/JSIT_clear_sky_medianaod_irrwav.nc b/data/postprocessing/irradiance_data/JSIT_clear_sky_medianaod_irrwav.nc new file mode 100644 index 00000000..ef4be36f Binary files /dev/null and b/data/postprocessing/irradiance_data/JSIT_clear_sky_medianaod_irrwav.nc differ diff --git a/data/postprocessing/irradiance_data/LOBE_clear_sky_aod0.0.nc b/data/postprocessing/irradiance_data/LOBE_clear_sky_aod0.0.nc new file mode 100644 index 00000000..bf906d29 Binary files /dev/null and b/data/postprocessing/irradiance_data/LOBE_clear_sky_aod0.0.nc differ diff --git a/data/postprocessing/irradiance_data/LOBE_clear_sky_aod0.0_irrwav.nc b/data/postprocessing/irradiance_data/LOBE_clear_sky_aod0.0_irrwav.nc new file mode 100644 index 00000000..cc0d0fb9 Binary files /dev/null and b/data/postprocessing/irradiance_data/LOBE_clear_sky_aod0.0_irrwav.nc differ diff --git a/data/postprocessing/irradiance_data/LOBE_clear_sky_medianaod.nc b/data/postprocessing/irradiance_data/LOBE_clear_sky_medianaod.nc new file mode 100644 index 00000000..abc9cd78 Binary files /dev/null and b/data/postprocessing/irradiance_data/LOBE_clear_sky_medianaod.nc differ diff --git a/data/postprocessing/irradiance_data/LOBE_clear_sky_medianaod_irrwav.nc b/data/postprocessing/irradiance_data/LOBE_clear_sky_medianaod_irrwav.nc new file mode 100644 index 00000000..1384b39b Binary files /dev/null and b/data/postprocessing/irradiance_data/LOBE_clear_sky_medianaod_irrwav.nc differ diff --git a/data/postprocessing/irradiance_data/PEAN_clear_sky_aod0.0.nc b/data/postprocessing/irradiance_data/PEAN_clear_sky_aod0.0.nc new file mode 100644 index 00000000..e92ff997 Binary files /dev/null and b/data/postprocessing/irradiance_data/PEAN_clear_sky_aod0.0.nc differ diff --git a/data/postprocessing/irradiance_data/PEAN_clear_sky_aod0.0_irrwav.nc b/data/postprocessing/irradiance_data/PEAN_clear_sky_aod0.0_irrwav.nc new file mode 100644 index 00000000..d9d6f0ff Binary files /dev/null and b/data/postprocessing/irradiance_data/PEAN_clear_sky_aod0.0_irrwav.nc differ diff --git a/data/postprocessing/irradiance_data/PEAN_clear_sky_medianaod.nc b/data/postprocessing/irradiance_data/PEAN_clear_sky_medianaod.nc new file mode 100644 index 00000000..5f28014e Binary files /dev/null and b/data/postprocessing/irradiance_data/PEAN_clear_sky_medianaod.nc differ diff --git a/data/postprocessing/irradiance_data/PEAN_clear_sky_medianaod_irrwav.nc b/data/postprocessing/irradiance_data/PEAN_clear_sky_medianaod_irrwav.nc new file mode 100644 index 00000000..50474f8a Binary files /dev/null and b/data/postprocessing/irradiance_data/PEAN_clear_sky_medianaod_irrwav.nc differ diff --git a/data/postprocessing/irradiance_data/WWUK_clear_sky_aod0.0.nc b/data/postprocessing/irradiance_data/WWUK_clear_sky_aod0.0.nc new file mode 100644 index 00000000..9c0a587a Binary files /dev/null and b/data/postprocessing/irradiance_data/WWUK_clear_sky_aod0.0.nc differ diff --git a/data/postprocessing/irradiance_data/WWUK_clear_sky_aod0.0_irrwav.nc b/data/postprocessing/irradiance_data/WWUK_clear_sky_aod0.0_irrwav.nc new file mode 100644 index 00000000..74d37724 Binary files /dev/null and b/data/postprocessing/irradiance_data/WWUK_clear_sky_aod0.0_irrwav.nc differ diff --git a/data/postprocessing/irradiance_data/WWUK_clear_sky_medianaod.nc b/data/postprocessing/irradiance_data/WWUK_clear_sky_medianaod.nc new file mode 100644 index 00000000..b4683ed9 Binary files /dev/null and b/data/postprocessing/irradiance_data/WWUK_clear_sky_medianaod.nc differ diff --git a/data/postprocessing/irradiance_data/WWUK_clear_sky_medianaod_irrwav.nc b/data/postprocessing/irradiance_data/WWUK_clear_sky_medianaod_irrwav.nc new file mode 100644 index 00000000..fa02b121 Binary files /dev/null and b/data/postprocessing/irradiance_data/WWUK_clear_sky_medianaod_irrwav.nc differ diff --git a/data/postprocessing/qc_reflectance_bounds/GHNA_2022May_2023Oct_bounds.nc b/data/postprocessing/qc_reflectance_bounds/GHNA_2022May_2023Oct_bounds.nc new file mode 100644 index 00000000..7cc17568 Binary files /dev/null and b/data/postprocessing/qc_reflectance_bounds/GHNA_2022May_2023Oct_bounds.nc differ diff --git a/data/postprocessing/qc_reflectance_bounds/GHNA_2023Oct_2024May_bounds.nc b/data/postprocessing/qc_reflectance_bounds/GHNA_2023Oct_2024May_bounds.nc new file mode 100644 index 00000000..e3674dc9 Binary files /dev/null and b/data/postprocessing/qc_reflectance_bounds/GHNA_2023Oct_2024May_bounds.nc differ diff --git a/data/postprocessing/qc_reflectance_bounds/GHNA_2024May_2025Aug_bounds.nc b/data/postprocessing/qc_reflectance_bounds/GHNA_2024May_2025Aug_bounds.nc new file mode 100644 index 00000000..43ebe829 Binary files /dev/null and b/data/postprocessing/qc_reflectance_bounds/GHNA_2024May_2025Aug_bounds.nc differ diff --git a/data/postprocessing/qc_reflectance_bounds/GHNA_2025Oct_2025Nov_bounds.nc b/data/postprocessing/qc_reflectance_bounds/GHNA_2025Oct_2025Nov_bounds.nc new file mode 100644 index 00000000..849101d0 Binary files /dev/null and b/data/postprocessing/qc_reflectance_bounds/GHNA_2025Oct_2025Nov_bounds.nc differ diff --git a/data/postprocessing/qc_reflectance_bounds/JAES_2023Apr_2023Sep_bounds.nc b/data/postprocessing/qc_reflectance_bounds/JAES_2023Apr_2023Sep_bounds.nc new file mode 100644 index 00000000..df29c2ec Binary files /dev/null and b/data/postprocessing/qc_reflectance_bounds/JAES_2023Apr_2023Sep_bounds.nc differ diff --git a/data/postprocessing/qc_reflectance_bounds/JAES_2023Apr_2024May_bounds.nc b/data/postprocessing/qc_reflectance_bounds/JAES_2023Apr_2024May_bounds.nc new file mode 100644 index 00000000..df29c2ec Binary files /dev/null and b/data/postprocessing/qc_reflectance_bounds/JAES_2023Apr_2024May_bounds.nc differ diff --git a/data/postprocessing/qc_reflectance_bounds/JAES_2024Jun_2024Nov_bounds.nc b/data/postprocessing/qc_reflectance_bounds/JAES_2024Jun_2024Nov_bounds.nc new file mode 100644 index 00000000..6e92944a Binary files /dev/null and b/data/postprocessing/qc_reflectance_bounds/JAES_2024Jun_2024Nov_bounds.nc differ diff --git a/data/postprocessing/qc_reflectance_bounds/JAES_2024Jun_2024Sep_bounds.nc b/data/postprocessing/qc_reflectance_bounds/JAES_2024Jun_2024Sep_bounds.nc new file mode 100644 index 00000000..6e92944a Binary files /dev/null and b/data/postprocessing/qc_reflectance_bounds/JAES_2024Jun_2024Sep_bounds.nc differ diff --git a/data/postprocessing/qc_reflectance_bounds/JAES_2024Nov_2025Jun_bounds.nc b/data/postprocessing/qc_reflectance_bounds/JAES_2024Nov_2025Jun_bounds.nc new file mode 100644 index 00000000..e38f9120 Binary files /dev/null and b/data/postprocessing/qc_reflectance_bounds/JAES_2024Nov_2025Jun_bounds.nc differ diff --git a/data/postprocessing/qc_reflectance_bounds/JAES_2025Jun_2025Aug_bounds.nc b/data/postprocessing/qc_reflectance_bounds/JAES_2025Jun_2025Aug_bounds.nc new file mode 100644 index 00000000..e86ac83c Binary files /dev/null and b/data/postprocessing/qc_reflectance_bounds/JAES_2025Jun_2025Aug_bounds.nc differ diff --git a/data/postprocessing/qc_reflectance_bounds/JSIT_2024Nov_2025Aug_bounds.nc b/data/postprocessing/qc_reflectance_bounds/JSIT_2024Nov_2025Aug_bounds.nc new file mode 100644 index 00000000..9f40c08e Binary files /dev/null and b/data/postprocessing/qc_reflectance_bounds/JSIT_2024Nov_2025Aug_bounds.nc differ diff --git a/data/postprocessing/qc_reflectance_bounds/JSIT_2025Jul_2025Aug_bounds.nc b/data/postprocessing/qc_reflectance_bounds/JSIT_2025Jul_2025Aug_bounds.nc new file mode 100644 index 00000000..5f4ad32f Binary files /dev/null and b/data/postprocessing/qc_reflectance_bounds/JSIT_2025Jul_2025Aug_bounds.nc differ diff --git a/data/postprocessing/qc_reflectance_bounds/JSIT_2025Mar_2025Jul_bounds.nc b/data/postprocessing/qc_reflectance_bounds/JSIT_2025Mar_2025Jul_bounds.nc new file mode 100644 index 00000000..c4e52464 Binary files /dev/null and b/data/postprocessing/qc_reflectance_bounds/JSIT_2025Mar_2025Jul_bounds.nc differ diff --git a/data/postprocessing/qc_reflectance_bounds/LOBE_2024May_2024Jul_bounds.nc b/data/postprocessing/qc_reflectance_bounds/LOBE_2024May_2024Jul_bounds.nc new file mode 100644 index 00000000..65f8f2c5 Binary files /dev/null and b/data/postprocessing/qc_reflectance_bounds/LOBE_2024May_2024Jul_bounds.nc differ diff --git a/data/postprocessing/qc_reflectance_bounds/LOBE_2024Oct_2024Nov_bounds.nc b/data/postprocessing/qc_reflectance_bounds/LOBE_2024Oct_2024Nov_bounds.nc new file mode 100644 index 00000000..3498bcb3 Binary files /dev/null and b/data/postprocessing/qc_reflectance_bounds/LOBE_2024Oct_2024Nov_bounds.nc differ diff --git a/data/postprocessing/qc_reflectance_bounds/LOBE_2025Apr_2025May_bounds.nc b/data/postprocessing/qc_reflectance_bounds/LOBE_2025Apr_2025May_bounds.nc new file mode 100644 index 00000000..26dfa155 Binary files /dev/null and b/data/postprocessing/qc_reflectance_bounds/LOBE_2025Apr_2025May_bounds.nc differ diff --git a/data/postprocessing/qc_reflectance_bounds/LOBE_2025Jun_2025Aug_bounds.nc b/data/postprocessing/qc_reflectance_bounds/LOBE_2025Jun_2025Aug_bounds.nc new file mode 100644 index 00000000..4ea764f0 Binary files /dev/null and b/data/postprocessing/qc_reflectance_bounds/LOBE_2025Jun_2025Aug_bounds.nc differ diff --git a/data/postprocessing/qc_reflectance_bounds/WWUK_2022May_2022Oct_bounds.nc b/data/postprocessing/qc_reflectance_bounds/WWUK_2022May_2022Oct_bounds.nc new file mode 100644 index 00000000..0a068075 Binary files /dev/null and b/data/postprocessing/qc_reflectance_bounds/WWUK_2022May_2022Oct_bounds.nc differ diff --git a/data/postprocessing/qc_reflectance_bounds/WWUK_2023Apr_2023Nov_bounds.nc b/data/postprocessing/qc_reflectance_bounds/WWUK_2023Apr_2023Nov_bounds.nc new file mode 100644 index 00000000..dee0fcbf Binary files /dev/null and b/data/postprocessing/qc_reflectance_bounds/WWUK_2023Apr_2023Nov_bounds.nc differ diff --git a/docs/atbd/HYPROCESSOR_ATBD.aux b/docs/atbd/HYPROCESSOR_ATBD.aux deleted file mode 100755 index bde77999..00000000 --- a/docs/atbd/HYPROCESSOR_ATBD.aux +++ /dev/null @@ -1,45 +0,0 @@ -\relax -\providecommand\hyper@newdestlabel[2]{} -\@nameuse{bbl@beforestart} -\providecommand\HyperFirstAtBeginDocument{\AtBeginDocument} -\HyperFirstAtBeginDocument{\ifx\hyper@anchor\@undefined -\global\let\oldcontentsline\contentsline -\gdef\contentsline#1#2#3#4{\oldcontentsline{#1}{#2}{#3}} -\global\let\oldnewlabel\newlabel -\gdef\newlabel#1#2{\newlabelxx{#1}#2} -\gdef\newlabelxx#1#2#3#4#5#6{\oldnewlabel{#1}{{#2}{#3}}} -\AtEndDocument{\ifx\hyper@anchor\@undefined -\let\contentsline\oldcontentsline -\let\newlabel\oldnewlabel -\fi} -\fi} -\global\let\hyper@last\relax -\gdef\HyperFirstAtBeginDocument#1{#1} -\providecommand\HyField@AuxAddToFields[1]{} -\providecommand\HyField@AuxAddToCoFields[2]{} -\babel@aux{english}{} -\gdef \LT@i {\LT@entry - {1}{55.82893pt}\LT@entry - {1}{57.9557pt}\LT@entry - {1}{154.66377pt}\LT@entry - {1}{154.66377pt}} -\@writefile{toc}{\contentsline {section}{Executive summary}{3}{section*.2}\protected@file@percent } -\@writefile{toc}{\contentsline {section}{\numberline {1}Introduction}{5}{section.1}\protected@file@percent } -\newlabel{sec-intro}{{1}{5}{Introduction}{section.1}{}} -\@writefile{toc}{\contentsline {subsection}{\numberline {1.1}Scope and objectives}{5}{subsection.1.1}\protected@file@percent } -\@writefile{toc}{\contentsline {subsection}{\numberline {1.2}Algorithm justification}{5}{subsection.1.2}\protected@file@percent } -\@writefile{toc}{\contentsline {subsubsection}{\numberline {1.2.1}Quantities of interest}{5}{subsubsection.1.2.1}\protected@file@percent } -\newlabel{HCRF_def}{{1}{6}{Hemispherical-conical reflectance factor (HCRF)}{equation.1.1}{}} -\newlabel{BRDF_def}{{2}{6}{Bidirectional reflectance distribution function}{equation.1.2}{}} -\@writefile{toc}{\contentsline {subsubsection}{\numberline {1.2.2}Uncertainty}{6}{subsubsection.1.2.2}\protected@file@percent } -\@writefile{toc}{\contentsline {section}{\numberline {2}Instrument and Network Design}{7}{section.2}\protected@file@percent } -\newlabel{sec-instr}{{2}{7}{Instrument and Network Design}{section.2}{}} -\@writefile{toc}{\contentsline {section}{\numberline {3}Processor Algorithm Description}{8}{section.3}\protected@file@percent } -\newlabel{sec-algo}{{3}{8}{Processor Algorithm Description}{section.3}{}} -\@writefile{toc}{\contentsline {section}{\numberline {4}Processor Implementation}{9}{section.4}\protected@file@percent } -\newlabel{sec-imple}{{4}{9}{Processor Implementation}{section.4}{}} -\@writefile{toc}{\contentsline {section}{\numberline {5}Products}{10}{section.5}\protected@file@percent } -\newlabel{sec-prod}{{5}{10}{Products}{section.5}{}} -\@writefile{toc}{\contentsline {section}{Appendix \numberline {A}...}{11}{appendix.1.A}\protected@file@percent } -\bibdata{HYPROCESSOR_ATBD} -\bibstyle{plainnat} diff --git a/docs/atbd/HYPROCESSOR_ATBD.hst b/docs/atbd/HYPROCESSOR_ATBD.hst deleted file mode 100755 index 17520328..00000000 --- a/docs/atbd/HYPROCESSOR_ATBD.hst +++ /dev/null @@ -1,3 +0,0 @@ -\def \vhCurrentVersion {0.0} -\def \vhCurrentDate {22.08.2019} -\newsetsimple \vhAllAuthorsSet {Samuel Hunt, Agnieszka Bia\l {}ek, Niall Origo} diff --git a/docs/atbd/HYPROCESSOR_ATBD.log b/docs/atbd/HYPROCESSOR_ATBD.log deleted file mode 100755 index b596e678..00000000 --- a/docs/atbd/HYPROCESSOR_ATBD.log +++ /dev/null @@ -1,762 +0,0 @@ -This is pdfTeX, Version 3.14159265-2.6-1.40.21 (MiKTeX 2.9.7300 64-bit) (preloaded format=pdflatex 2020.3.5) 30 SEP 2021 15:10 -entering extended mode -**./HYPROCESSOR_ATBD.tex -(HYPROCESSOR_ATBD.tex -LaTeX2e <2020-02-02> patch level 2 -L3 programming layer <2020-02-14> -("C:\Program Files\MiKTeX 2.9\tex/latex/base\article.cls" -Document Class: article 2019/12/20 v1.4l Standard LaTeX document class -("C:\Program Files\MiKTeX 2.9\tex/latex/base\size10.clo" -File: size10.clo 2019/12/20 v1.4l Standard LaTeX file (size option) -) -\c@part=\count167 -\c@section=\count168 -\c@subsection=\count169 -\c@subsubsection=\count170 -\c@paragraph=\count171 -\c@subparagraph=\count172 -\c@figure=\count173 -\c@table=\count174 -\abovecaptionskip=\skip47 -\belowcaptionskip=\skip48 -\bibindent=\dimen134 -) -("C:\Program Files\MiKTeX 2.9\tex/generic/babel\babel.sty" -Package: babel 2020/02/14 3.40 The Babel package - -("C:\Program Files\MiKTeX 2.9\tex/generic/babel\switch.def" -File: switch.def 2020/02/14 3.40 Babel switching mechanism -) -************************************* -* Local config file bblopts.cfg used -* -("C:\Program Files\MiKTeX 2.9\tex/latex/arabi\bblopts.cfg" -File: bblopts.cfg 2005/09/08 v0.1 add Arabic and Farsi to "declared" options of - babel -) -("C:\Program Files\MiKTeX 2.9\tex/latex/babel-english\english.ldf" -Language: english 2017/06/06 v3.3r English support from the babel system - -("C:\Program Files\MiKTeX 2.9\tex/generic/babel\babel.def" -File: babel.def 2020/02/14 3.40 Babel common definitions -\babel@savecnt=\count175 -\U@D=\dimen135 - -("C:\Program Files\MiKTeX 2.9\tex/generic/babel\txtbabel.def") -\bbl@readstream=\read2 -\bbl@dirlevel=\count176 -) -Package babel Info: \l@canadian = using hyphenrules for english -(babel) (\language0) on input line 102. -Package babel Info: \l@australian = using hyphenrules for ukenglish -(babel) (\language72) on input line 105. -Package babel Info: \l@newzealand = using hyphenrules for ukenglish -(babel) (\language72) on input line 108. -)) -("C:\Program Files\MiKTeX 2.9\tex/latex/base\inputenc.sty" -Package: inputenc 2018/08/11 v1.3c Input encoding file -\inpenc@prehook=\toks14 -\inpenc@posthook=\toks15 - -(C:\Users\pdv\AppData\Roaming\MiKTeX\2.9\tex/latex/ucs\utf8x.def -File: utf8x.def 2004/10/17 UCS: Input encoding UTF-8 -)) -(C:\Users\pdv\AppData\Roaming\MiKTeX\2.9\tex/latex/ucs\ucs.sty -Package: ucs 2013/05/11 v2.2 UCS: Unicode input support - -(C:\Users\pdv\AppData\Roaming\MiKTeX\2.9\tex/latex/ucs\uni-global.def -File: uni-global.def 2013/05/13 UCS: Unicode global data -) -\uc@secondtry=\count177 -\uc@combtoks=\toks16 -\uc@combtoksb=\toks17 -\uc@temptokena=\toks18 -) -(C:\Users\pdv\AppData\Roaming\MiKTeX\2.9\tex/latex/pdflscape\pdflscape.sty -Package: pdflscape 2019/12/05 v0.12 Display of landscape pages in PDF (HO) - -("C:\Program Files\MiKTeX 2.9\tex/generic/iftex\iftex.sty" -Package: iftex 2019/11/07 v1.0c TeX engine tests -) -("C:\Program Files\MiKTeX 2.9\tex/latex/graphics\lscape.sty" -Package: lscape 2000/10/22 v3.01 Landscape Pages (DPC) - -("C:\Program Files\MiKTeX 2.9\tex/latex/graphics\graphics.sty" -Package: graphics 2019/11/30 v1.4a Standard LaTeX Graphics (DPC,SPQR) - -("C:\Program Files\MiKTeX 2.9\tex/latex/graphics\trig.sty" -Package: trig 2016/01/03 v1.10 sin cos tan (DPC) -) -("C:\Program Files\MiKTeX 2.9\tex/latex/graphics-cfg\graphics.cfg" -File: graphics.cfg 2016/06/04 v1.11 sample graphics configuration -) -Package graphics Info: Driver file: pdftex.def on input line 105. - -("C:\Program Files\MiKTeX 2.9\tex/latex/graphics-def\pdftex.def" -File: pdftex.def 2018/01/08 v1.0l Graphics/color driver for pdftex -))) -Package pdflscape Info: Auto-detected driver: pdftex on input line 81. -) -(C:\Users\pdv\AppData\Roaming\MiKTeX\2.9\tex/latex/pdfpages\pdfpages.sty -Package: pdfpages 2020/01/28 v0.5q Insert pages of external PDF documents (AM) - -("C:\Program Files\MiKTeX 2.9\tex/latex/base\ifthen.sty" -Package: ifthen 2014/09/29 v1.1c Standard LaTeX ifthen package (DPC) -) -("C:\Program Files\MiKTeX 2.9\tex/latex/tools\calc.sty" -Package: calc 2017/05/25 v4.3 Infix arithmetic (KKT,FJ) -\calc@Acount=\count178 -\calc@Bcount=\count179 -\calc@Adimen=\dimen136 -\calc@Bdimen=\dimen137 -\calc@Askip=\skip49 -\calc@Bskip=\skip50 -LaTeX Info: Redefining \setlength on input line 80. -LaTeX Info: Redefining \addtolength on input line 81. -\calc@Ccount=\count180 -\calc@Cskip=\skip51 -) -(C:\Users\pdv\AppData\Roaming\MiKTeX\2.9\tex/latex/eso-pic\eso-pic.sty -Package: eso-pic 2020/09/27 v2.0i eso-pic (RN) - -(C:\Users\pdv\AppData\Roaming\MiKTeX\2.9\tex/generic/atbegshi\atbegshi.sty -Package: atbegshi 2019/12/05 v1.19 At begin shipout hook (HO) - -(C:\Users\pdv\AppData\Roaming\MiKTeX\2.9\tex/generic/infwarerr\infwarerr.sty -Package: infwarerr 2019/12/03 v1.5 Providing info/warning/error messages (HO) -) -(C:\Users\pdv\AppData\Roaming\MiKTeX\2.9\tex/generic/ltxcmds\ltxcmds.sty -Package: ltxcmds 2019/12/15 v1.24 LaTeX kernel commands for general use (HO) -)) -\ESO@tempdima=\dimen138 -\ESO@tempdimb=\dimen139 - -("C:\Program Files\MiKTeX 2.9\tex/latex/graphics\keyval.sty" -Package: keyval 2014/10/28 v1.15 key=value parser (DPC) -\KV@toks@=\toks19 -) -("C:\Program Files\MiKTeX 2.9\tex/latex/xcolor\xcolor.sty" -Package: xcolor 2016/05/11 v2.12 LaTeX color extensions (UK) - -("C:\Program Files\MiKTeX 2.9\tex/latex/graphics-cfg\color.cfg" -File: color.cfg 2016/01/02 v1.6 sample color configuration -) -Package xcolor Info: Driver file: pdftex.def on input line 225. -Package xcolor Info: Model `cmy' substituted by `cmy0' on input line 1348. -Package xcolor Info: Model `hsb' substituted by `rgb' on input line 1352. -Package xcolor Info: Model `RGB' extended on input line 1364. -Package xcolor Info: Model `HTML' substituted by `rgb' on input line 1366. -Package xcolor Info: Model `Hsb' substituted by `hsb' on input line 1367. -Package xcolor Info: Model `tHsb' substituted by `hsb' on input line 1368. -Package xcolor Info: Model `HSB' substituted by `hsb' on input line 1369. -Package xcolor Info: Model `Gray' substituted by `gray' on input line 1370. -Package xcolor Info: Model `wave' substituted by `hsb' on input line 1371. -)) -("C:\Program Files\MiKTeX 2.9\tex/latex/graphics\graphicx.sty" -Package: graphicx 2019/11/30 v1.2a Enhanced LaTeX Graphics (DPC,SPQR) -\Gin@req@height=\dimen140 -\Gin@req@width=\dimen141 -) -\AM@pagewidth=\dimen142 -\AM@pageheight=\dimen143 - -(C:\Users\pdv\AppData\Roaming\MiKTeX\2.9\tex/latex/pdfpages\pppdftex.def -File: pppdftex.def 2020/01/28 v0.5q Pdfpages driver for pdfTeX (AM) -) -\AM@pagebox=\box45 -\AM@global@opts=\toks20 -\AM@pagecnt=\count181 -\AM@toc@title=\toks21 -\c@AM@survey=\count182 -\AM@templatesizebox=\box46 -) -(C:\Users\pdv\AppData\Roaming\MiKTeX\2.9\tex/latex/multirow\multirow.sty -Package: multirow 2019/05/31 v2.5 Span multiple rows of a table -\multirow@colwidth=\skip52 -\multirow@cntb=\count183 -\multirow@dima=\skip53 -\bigstrutjot=\dimen144 -) -("C:\Program Files\MiKTeX 2.9\tex/latex/tools\array.sty" -Package: array 2019/08/31 v2.4l Tabular extension package (FMi) -\col@sep=\dimen145 -\ar@mcellbox=\box47 -\extrarowheight=\dimen146 -\NC@list=\toks22 -\extratabsurround=\skip54 -\backup@length=\skip55 -\ar@cellbox=\box48 -) -(C:\Users\pdv\AppData\Roaming\MiKTeX\2.9\tex/latex/placeins\placeins.sty -Package: placeins 2005/04/18 v 2.2 -) -(C:\Users\pdv\AppData\Roaming\MiKTeX\2.9\tex/latex/threeparttable\threeparttabl -e.sty -Package: threeparttable 2003/06/13 v 3.0 -\@tempboxb=\box49 -) (C:\Users\pdv\AppData\Roaming\MiKTeX\2.9\tex/latex/natbib\natbib.sty -Package: natbib 2010/09/13 8.31b (PWD, AO) -\bibhang=\skip56 -\bibsep=\skip57 -LaTeX Info: Redefining \cite on input line 694. -\c@NAT@ctr=\count184 -) -(C:\Users\pdv\AppData\Roaming\MiKTeX\2.9\tex/latex/vhistory\vhistory.sty -Package: vhistory - -(C:\Users\pdv\AppData\Roaming\MiKTeX\2.9\tex/latex/carlisle\ltxtable.sty -Package: ltxtable 1995/12/11 v0.2 longtable/tabularx merge (DPC) - -("C:\Program Files\MiKTeX 2.9\tex/latex/tools\tabularx.sty" -Package: tabularx 2020/01/15 v2.11c `tabularx' package (DPC) -\TX@col@width=\dimen147 -\TX@old@table=\dimen148 -\TX@old@col=\dimen149 -\TX@target=\dimen150 -\TX@delta=\dimen151 -\TX@cols=\count185 -\TX@ftn=\toks23 -) -("C:\Program Files\MiKTeX 2.9\tex/latex/tools\longtable.sty" -Package: longtable 2020/01/07 v4.13 Multi-page Table package (DPC) -\LTleft=\skip58 -\LTright=\skip59 -\LTpre=\skip60 -\LTpost=\skip61 -\LTchunksize=\count186 -\LTcapwidth=\dimen152 -\LT@head=\box50 -\LT@firsthead=\box51 -\LT@foot=\box52 -\LT@lastfoot=\box53 -\LT@cols=\count187 -\LT@rows=\count188 -\c@LT@tables=\count189 -\c@LT@chunks=\count190 -\LT@p@ftn=\toks24 -)) -(C:\Users\pdv\AppData\Roaming\MiKTeX\2.9\tex/latex/vhistory\sets.sty -Package: sets -\c@s@tlength=\count191 -) -File HYPROCESSOR_ATBD.hst not found. -\vh@verfile=\write3 -) (C:\Users\pdv\AppData\Roaming\MiKTeX\2.9\tex/latex/eurosym\eurosym.sty -Package: eurosym 1998/08/06 v1.1 European currency symbol ``Euro'' -\@eurobox=\box54 -) -(C:\Users\pdv\AppData\Roaming\MiKTeX\2.9\tex/latex/enumitem\enumitem.sty -Package: enumitem 2019/06/20 v3.9 Customized lists -\labelindent=\skip62 -\enit@outerparindent=\dimen153 -\enit@toks=\toks25 -\enit@inbox=\box55 -\enit@count@id=\count192 -\enitdp@description=\count193 -) -(C:\Users\pdv\AppData\Roaming\MiKTeX\2.9\tex/latex/geometry\geometry.sty -Package: geometry 2020/01/02 v5.9 Page Geometry - -("C:\Program Files\MiKTeX 2.9\tex/generic/iftex\ifvtex.sty" -Package: ifvtex 2019/10/25 v1.7 ifvtex legacy package. Use iftex instead. -) -\Gm@cnth=\count194 -\Gm@cntv=\count195 -\c@Gm@tempcnt=\count196 -\Gm@bindingoffset=\dimen154 -\Gm@wd@mp=\dimen155 -\Gm@odd@mp=\dimen156 -\Gm@even@mp=\dimen157 -\Gm@layoutwidth=\dimen158 -\Gm@layoutheight=\dimen159 -\Gm@layouthoffset=\dimen160 -\Gm@layoutvoffset=\dimen161 -\Gm@dimlist=\toks26 - -(C:\Users\pdv\AppData\Roaming\MiKTeX\2.9\tex/latex/geometry\geometry.cfg)) -("C:\Program Files\MiKTeX 2.9\tex/latex/amsmath\amsmath.sty" -Package: amsmath 2020/01/20 v2.17e AMS math features -\@mathmargin=\skip63 - -For additional information on amsmath, use the `?' option. -("C:\Program Files\MiKTeX 2.9\tex/latex/amsmath\amstext.sty" -Package: amstext 2000/06/29 v2.01 AMS text - -("C:\Program Files\MiKTeX 2.9\tex/latex/amsmath\amsgen.sty" -File: amsgen.sty 1999/11/30 v2.0 generic functions -\@emptytoks=\toks27 -\ex@=\dimen162 -)) -("C:\Program Files\MiKTeX 2.9\tex/latex/amsmath\amsbsy.sty" -Package: amsbsy 1999/11/29 v1.2d Bold Symbols -\pmbraise@=\dimen163 -) -("C:\Program Files\MiKTeX 2.9\tex/latex/amsmath\amsopn.sty" -Package: amsopn 2016/03/08 v2.02 operator names -) -\inf@bad=\count197 -LaTeX Info: Redefining \frac on input line 227. -\uproot@=\count198 -\leftroot@=\count199 -LaTeX Info: Redefining \overline on input line 389. -\classnum@=\count266 -\DOTSCASE@=\count267 -LaTeX Info: Redefining \ldots on input line 486. -LaTeX Info: Redefining \dots on input line 489. -LaTeX Info: Redefining \cdots on input line 610. -\Mathstrutbox@=\box56 -\strutbox@=\box57 -\big@size=\dimen164 -LaTeX Font Info: Redeclaring font encoding OML on input line 733. -LaTeX Font Info: Redeclaring font encoding OMS on input line 734. -\macc@depth=\count268 -\c@MaxMatrixCols=\count269 -\dotsspace@=\muskip16 -\c@parentequation=\count270 -\dspbrk@lvl=\count271 -\tag@help=\toks28 -\row@=\count272 -\column@=\count273 -\maxfields@=\count274 -\andhelp@=\toks29 -\eqnshift@=\dimen165 -\alignsep@=\dimen166 -\tagshift@=\dimen167 -\tagwidth@=\dimen168 -\totwidth@=\dimen169 -\lineht@=\dimen170 -\@envbody=\toks30 -\multlinegap=\skip64 -\multlinetaggap=\skip65 -\mathdisplay@stack=\toks31 -LaTeX Info: Redefining \[ on input line 2859. -LaTeX Info: Redefining \] on input line 2860. -) -("C:\Program Files\MiKTeX 2.9\tex/latex/amsfonts\amssymb.sty" -Package: amssymb 2013/01/14 v3.01 AMS font symbols - -("C:\Program Files\MiKTeX 2.9\tex/latex/amsfonts\amsfonts.sty" -Package: amsfonts 2013/01/14 v3.01 Basic AMSFonts support -\symAMSa=\mathgroup4 -\symAMSb=\mathgroup5 -LaTeX Font Info: Redeclaring math symbol \hbar on input line 98. -LaTeX Font Info: Overwriting math alphabet `\mathfrak' in version `bold' -(Font) U/euf/m/n --> U/euf/b/n on input line 106. -)) -(C:\Users\pdv\AppData\Roaming\MiKTeX\2.9\tex/latex/caption\caption.sty -Package: caption 2020/10/26 v3.5g Customizing captions (AR) - -(C:\Users\pdv\AppData\Roaming\MiKTeX\2.9\tex/latex/caption\caption3.sty -Package: caption3 2020/10/21 v2.2e caption3 kernel (AR) -\captionmargin=\dimen171 -\captionmargin@=\dimen172 -\captionwidth=\dimen173 -\caption@tempdima=\dimen174 -\caption@indent=\dimen175 -\caption@parindent=\dimen176 -\caption@hangindent=\dimen177 -Package caption Info: Standard document class detected. -) -\c@caption@flags=\count275 -\c@continuedfloat=\count276 -Package caption Info: longtable package is loaded. - -(C:\Users\pdv\AppData\Roaming\MiKTeX\2.9\tex/latex/caption\ltcaption.sty -Package: ltcaption 2020/05/30 v1.4b longtable captions (AR) -) -Package caption Info: threeparttable package is loaded. -) -(C:\Users\pdv\AppData\Roaming\MiKTeX\2.9\tex/latex/fancyhdr\fancyhdr.sty -Package: fancyhdr 2019/01/31 v3.10 Extensive control of page headers and footer -s -\f@nch@headwidth=\skip66 -\f@nch@O@elh=\skip67 -\f@nch@O@erh=\skip68 -\f@nch@O@olh=\skip69 -\f@nch@O@orh=\skip70 -\f@nch@O@elf=\skip71 -\f@nch@O@erf=\skip72 -\f@nch@O@olf=\skip73 -\f@nch@O@orf=\skip74 -) -("C:\Program Files\MiKTeX 2.9\tex/latex/hyperref\hyperref.sty" -Package: hyperref 2020/01/14 v7.00d Hypertext links for LaTeX - -("C:\Program Files\MiKTeX 2.9\tex/latex/pdftexcmds\pdftexcmds.sty" -Package: pdftexcmds 2019/11/24 v0.31 Utility functions of pdfTeX for LuaTeX (HO -) -Package pdftexcmds Info: \pdf@primitive is available. -Package pdftexcmds Info: \pdf@ifprimitive is available. -Package pdftexcmds Info: \pdfdraftmode found. -) -(C:\Users\pdv\AppData\Roaming\MiKTeX\2.9\tex/generic/kvsetkeys\kvsetkeys.sty -Package: kvsetkeys 2019/12/15 v1.18 Key value parser (HO) -) -(C:\Users\pdv\AppData\Roaming\MiKTeX\2.9\tex/generic/kvdefinekeys\kvdefinekeys. -sty -Package: kvdefinekeys 2019-12-19 v1.6 Define keys (HO) -) -(C:\Users\pdv\AppData\Roaming\MiKTeX\2.9\tex/generic/pdfescape\pdfescape.sty -Package: pdfescape 2019/12/09 v1.15 Implements pdfTeX's escape features (HO) -) -(C:\Users\pdv\AppData\Roaming\MiKTeX\2.9\tex/latex/hycolor\hycolor.sty -Package: hycolor 2020-01-27 v1.10 Color options for hyperref/bookmark (HO) -) -(C:\Users\pdv\AppData\Roaming\MiKTeX\2.9\tex/latex/letltxmacro\letltxmacro.sty -Package: letltxmacro 2019/12/03 v1.6 Let assignment for LaTeX macros (HO) -) (C:\Users\pdv\AppData\Roaming\MiKTeX\2.9\tex/latex/auxhook\auxhook.sty -Package: auxhook 2019-12-17 v1.6 Hooks for auxiliary files (HO) -) -(C:\Users\pdv\AppData\Roaming\MiKTeX\2.9\tex/latex/kvoptions\kvoptions.sty -Package: kvoptions 2019/11/29 v3.13 Key value format for package options (HO) -) -\@linkdim=\dimen178 -\Hy@linkcounter=\count277 -\Hy@pagecounter=\count278 - -("C:\Program Files\MiKTeX 2.9\tex/latex/hyperref\pd1enc.def" -File: pd1enc.def 2020/01/14 v7.00d Hyperref: PDFDocEncoding definition (HO) -) -(C:\Users\pdv\AppData\Roaming\MiKTeX\2.9\tex/generic/intcalc\intcalc.sty -Package: intcalc 2019/12/15 v1.3 Expandable calculations with integers (HO) -) -(C:\Users\pdv\AppData\Roaming\MiKTeX\2.9\tex/generic/etexcmds\etexcmds.sty -Package: etexcmds 2019/12/15 v1.7 Avoid name clashes with e-TeX commands (HO) -) -\Hy@SavedSpaceFactor=\count279 -Package hyperref Info: Hyper figures OFF on input line 4547. -Package hyperref Info: Link nesting OFF on input line 4552. -Package hyperref Info: Hyper index ON on input line 4555. -Package hyperref Info: Plain pages OFF on input line 4562. -Package hyperref Info: Backreferencing OFF on input line 4567. -Package hyperref Info: Implicit mode ON; LaTeX internals redefined. -Package hyperref Info: Bookmarks ON on input line 4800. -\c@Hy@tempcnt=\count280 - -("C:\Program Files\MiKTeX 2.9\tex/latex/url\url.sty" -\Urlmuskip=\muskip17 -Package: url 2013/09/16 ver 3.4 Verb mode for urls, etc. -) -LaTeX Info: Redefining \url on input line 5159. -\XeTeXLinkMargin=\dimen179 - -(C:\Users\pdv\AppData\Roaming\MiKTeX\2.9\tex/generic/bitset\bitset.sty -Package: bitset 2019/12/09 v1.3 Handle bit-vector datatype (HO) - -(C:\Users\pdv\AppData\Roaming\MiKTeX\2.9\tex/generic/bigintcalc\bigintcalc.sty -Package: bigintcalc 2019/12/15 v1.5 Expandable calculations on big integers (HO -) -)) -\Fld@menulength=\count281 -\Field@Width=\dimen180 -\Fld@charsize=\dimen181 -Package hyperref Info: Hyper figures OFF on input line 6430. -Package hyperref Info: Link nesting OFF on input line 6435. -Package hyperref Info: Hyper index ON on input line 6438. -Package hyperref Info: backreferencing OFF on input line 6445. -Package hyperref Info: Link coloring OFF on input line 6450. -Package hyperref Info: Link coloring with OCG OFF on input line 6455. -Package hyperref Info: PDF/A mode OFF on input line 6460. -LaTeX Info: Redefining \ref on input line 6500. -LaTeX Info: Redefining \pageref on input line 6504. -\Hy@abspage=\count282 -\c@Item=\count283 -\c@Hfootnote=\count284 -) -Package hyperref Info: Driver (autodetected): hpdftex. - ("C:\Program Files\MiKTeX 2.9\tex/latex/hyperref\hpdftex.def" -File: hpdftex.def 2020/01/14 v7.00d Hyperref driver for pdfTeX - -(C:\Users\pdv\AppData\Roaming\MiKTeX\2.9\tex/latex/atveryend\atveryend.sty -Package: atveryend 2019-12-11 v1.11 Hooks at the very end of document (HO) -Package atveryend Info: \enddocument detected (standard20110627). -) -\Fld@listcount=\count285 -\c@bookmark@seq@number=\count286 - -(C:\Users\pdv\AppData\Roaming\MiKTeX\2.9\tex/latex/rerunfilecheck\rerunfilechec -k.sty -Package: rerunfilecheck 2019/12/05 v1.9 Rerun checks for auxiliary files (HO) - -(C:\Users\pdv\AppData\Roaming\MiKTeX\2.9\tex/generic/uniquecounter\uniquecounte -r.sty -Package: uniquecounter 2019/12/15 v1.4 Provide unlimited unique counter (HO) -) -Package uniquecounter Info: New unique counter `rerunfilecheck' on input line 2 -86. -) -\Hy@SectionHShift=\skip75 -) -(C:\Users\pdv\AppData\Roaming\MiKTeX\2.9\tex/latex/setspace\setspace.sty -Package: setspace 2011/12/19 v6.7a set line spacing -) -(C:\Users\pdv\AppData\Roaming\MiKTeX\2.9\tex/latex/nomencl\nomencl.sty -Package: nomencl 2020/12/29 v5.5 Nomenclature package - -("C:\Program Files\MiKTeX 2.9\tex/latex/xkeyval\xkeyval.sty" -Package: xkeyval 2014/12/03 v2.7a package option processing (HA) - -("C:\Program Files\MiKTeX 2.9\tex/generic/xkeyval\xkeyval.tex" -("C:\Program Files\MiKTeX 2.9\tex/generic/xkeyval\xkvutils.tex" -\XKV@toks=\toks32 -\XKV@tempa@toks=\toks33 -) -\XKV@depth=\count287 -File: xkeyval.tex 2014/12/03 v2.7a key=value parser (HA) -)) -(C:\Users\pdv\AppData\Roaming\MiKTeX\2.9\tex/latex/koma-script\tocbasic.sty -Package: tocbasic 2021/06/25 v3.34 KOMA-Script package (handling toc-files) - -(C:\Users\pdv\AppData\Roaming\MiKTeX\2.9\tex/latex/koma-script\scrbase.sty -Package: scrbase 2021/06/25 v3.34 KOMA-Script package (KOMA-Script-independent -basics and keyval usage) - -(C:\Users\pdv\AppData\Roaming\MiKTeX\2.9\tex/latex/koma-script\scrlfile.sty -Package: scrlfile 2021/06/25 v3.34 KOMA-Script package (file load hooks) - -(C:\Users\pdv\AppData\Roaming\MiKTeX\2.9\tex/latex/koma-script\scrlfile-patchol -dlatex.sty -Package: scrlfile-patcholdlatex 2021/06/25 v3.34 KOMA-Script package (patching -old LaTeX kernels) - -(C:\Users\pdv\AppData\Roaming\MiKTeX\2.9\tex/latex/koma-script\scrlogo.sty -Package: scrlogo 2021/06/25 v3.34 KOMA-Script package (logo) -)) -("C:\Program Files\MiKTeX 2.9\tex/latex/l3packages/xparse\xparse.sty" -("C:\Program Files\MiKTeX 2.9\tex/latex/l3kernel\expl3.sty" -Package: expl3 2020-02-14 L3 programming layer (loader) - -("C:\Program Files\MiKTeX 2.9\tex/latex/l3backend\l3backend-pdfmode.def" -File: l3backend-pdfmode.def 2020-02-03 L3 backend support: PDF mode -\l__kernel_color_stack_int=\count288 -\l__pdf_internal_box=\box58 -)) -Package: xparse 2020-02-14 L3 Experimental document command parser -\l__xparse_current_arg_int=\count289 -\g__xparse_grabber_int=\count290 -\l__xparse_m_args_int=\count291 -\l__xparse_v_nesting_int=\count292 -)) -Skipping: [2021/05/01] Usage of raw or classic option list on input line 252. -Applying: [0000/00/00] Usage of raw or classic option list on input line 368. -) -\scr@dte@tocline@numberwidth=\skip76 -\scr@dte@tocline@numbox=\box59 -) -Package tocbasic Info: setting babel extension for `nlo' on input line 187. -Package tocbasic Info: setting babel extension for `nls' on input line 188. -\nomlabelwidth=\dimen182 -\nom@tempdim=\dimen183 -\nomitemsep=\skip77 -) -\@nomenclaturefile=\write4 -\openout4 = `HYPROCESSOR_ATBD.nlo'. - -Package nomencl Info: Writing nomenclature file HYPROCESSOR_ATBD.nlo on input l -ine 32. -("C:\Program Files\MiKTeX 2.9\tex/latex/etoolbox\etoolbox.sty" -Package: etoolbox 2019/09/21 v2.5h e-TeX tools for LaTeX (JAW) -\etb@tempcnta=\count293 -) -\myTitleBlock=\box60 -LaTeX Font Info: Trying to load font information for U+msa on input line 60. - - -("C:\Program Files\MiKTeX 2.9\tex/latex/amsfonts\umsa.fd" -File: umsa.fd 2013/01/14 v3.01 AMS symbols A -) -LaTeX Font Info: Trying to load font information for U+msb on input line 60. - - -("C:\Program Files\MiKTeX 2.9\tex/latex/amsfonts\umsb.fd" -File: umsb.fd 2013/01/14 v3.01 AMS symbols B -) - -File: LOGOHYPERNETS_BGNRV3.png Graphic file (type png) - -Package pdftex.def Info: LOGOHYPERNETS_BGNRV3.png used on input line 62. -(pdftex.def) Requested size: 170.71652pt x 37.81267pt. - -(C:\Users\pdv\AppData\Roaming\MiKTeX\2.9\tex/latex/appendix\appendix.sty -Package: appendix 2020/02/08 v1.2c extra appendix facilities -\c@@pps=\count294 -\c@@ppsavesec=\count295 -\c@@ppsaveapp=\count296 -) -No file HYPROCESSOR_ATBD.aux. -\openout1 = `HYPROCESSOR_ATBD.aux'. - -LaTeX Font Info: Checking defaults for OML/cmm/m/it on input line 78. -LaTeX Font Info: ... okay on input line 78. -LaTeX Font Info: Checking defaults for OMS/cmsy/m/n on input line 78. -LaTeX Font Info: ... okay on input line 78. -LaTeX Font Info: Checking defaults for OT1/cmr/m/n on input line 78. -LaTeX Font Info: ... okay on input line 78. -LaTeX Font Info: Checking defaults for T1/cmr/m/n on input line 78. -LaTeX Font Info: ... okay on input line 78. -LaTeX Font Info: Checking defaults for TS1/cmr/m/n on input line 78. -LaTeX Font Info: ... okay on input line 78. -LaTeX Font Info: Checking defaults for OMX/cmex/m/n on input line 78. -LaTeX Font Info: ... okay on input line 78. -LaTeX Font Info: Checking defaults for U/cmr/m/n on input line 78. -LaTeX Font Info: ... okay on input line 78. -LaTeX Font Info: Checking defaults for PD1/pdf/m/n on input line 78. -LaTeX Font Info: ... okay on input line 78. -(C:\Users\pdv\AppData\Roaming\MiKTeX\2.9\tex/latex/ucs\ucsencs.def -File: ucsencs.def 2011/01/21 Fixes to fontencodings LGR, T3 -) -("C:\Program Files\MiKTeX 2.9\tex/context/base/mkii\supp-pdf.mkii" -[Loading MPS to PDF converter (version 2006.09.02).] -\scratchcounter=\count297 -\scratchdimen=\dimen184 -\scratchbox=\box61 -\nofMPsegments=\count298 -\nofMParguments=\count299 -\everyMPshowfont=\toks34 -\MPscratchCnt=\count300 -\MPscratchDim=\dimen185 -\MPnumerator=\count301 -\makeMPintoPDFobject=\count302 -\everyMPtoPDFconversion=\toks35 -) ("C:\Program Files\MiKTeX 2.9\tex/latex/epstopdf-pkg\epstopdf-base.sty" -Package: epstopdf-base 2020-01-24 v2.11 Base part for package epstopdf - -(C:\Users\pdv\AppData\Roaming\MiKTeX\2.9\tex/latex/grfext\grfext.sty -Package: grfext 2019/12/03 v1.3 Manage graphics extensions (HO) -) -Package epstopdf-base Info: Redefining graphics rule for `.eps' on input line 4 -85. -Package grfext Info: Graphics extension search list: -(grfext) [.pdf,.png,.jpg,.mps,.jpeg,.jbig2,.jb2,.PDF,.PNG,.JPG,.JPE -G,.JBIG2,.JB2,.eps] -(grfext) \AppendGraphicsExtensions on input line 504. -) -\AtBeginShipoutBox=\box62 - -*geometry* driver: auto-detecting -*geometry* detected driver: pdftex -*geometry* verbose mode - [ preamble ] result: -* driver: pdftex -* paper: -* layout: -* layoutoffset:(h,v)=(0.0pt,0.0pt) -* modes: -* h-part:(L,W,R)=(92.14519pt, 430.00462pt, 92.14519pt) -* v-part:(T,H,B)=(51.21504pt, 648.35757pt, 95.39737pt) -* \paperwidth=614.295pt -* \paperheight=794.96999pt -* \textwidth=430.00462pt -* \textheight=648.35757pt -* \oddsidemargin=19.8752pt -* \evensidemargin=19.8752pt -* \topmargin=-58.05495pt -* \headheight=55.0pt -* \headsep=25.0pt -* \topskip=10.0pt -* \footskip=30.0pt -* \marginparwidth=65.0pt -* \marginparsep=11.0pt -* \columnsep=10.0pt -* \skip\footins=9.0pt plus 4.0pt minus 2.0pt -* \hoffset=0.0pt -* \voffset=0.0pt -* \mag=1000 -* \@twocolumnfalse -* \@twosidefalse -* \@mparswitchfalse -* \@reversemarginfalse -* (1in=72.27pt=25.4mm, 1cm=28.453pt) - -Package caption Info: Begin \AtBeginDocument code. -Package caption Info: hyperref package is loaded. -Package caption Info: End \AtBeginDocument code. -Package hyperref Info: Link coloring OFF on input line 78. -("C:\Program Files\MiKTeX 2.9\tex/latex/hyperref\nameref.sty" -Package: nameref 2019/09/16 v2.46 Cross-referencing by name of section - -(C:\Users\pdv\AppData\Roaming\MiKTeX\2.9\tex/latex/refcount\refcount.sty -Package: refcount 2019/12/15 v3.6 Data extraction from label references (HO) -) -(C:\Users\pdv\AppData\Roaming\MiKTeX\2.9\tex/generic/gettitlestring\gettitlestr -ing.sty -Package: gettitlestring 2019/12/15 v1.6 Cleanup title references (HO) -) -\c@section@level=\count303 -) -LaTeX Info: Redefining \ref on input line 78. -LaTeX Info: Redefining \pageref on input line 78. -LaTeX Info: Redefining \nameref on input line 78. -\@outlinefile=\write5 -\openout5 = `HYPROCESSOR_ATBD.out'. - -File: LOGOHYPERNETS_BGNRV3.png Graphic file (type png) - -Package pdftex.def Info: LOGOHYPERNETS_BGNRV3.png used on input line 85. -(pdftex.def) Requested size: 341.43306pt x 75.62534pt. - - -LaTeX Warning: No \author given. - -(C:\Users\pdv\AppData\Roaming\MiKTeX\2.9\tex/latex/ucs\uni-0.def -File: uni-0.def 2013/05/13 UCS: Unicode data U+0000..U+00FF -) [1{C:/Users/pdv/AppData/Local/MiKTeX/2.9/pdftex/config/pdftex.map} - - <./LOGOHYPERNETS_BGNRV3.png>] -\openout3 = `HYPROCESSOR_ATBD.ver'. - -\vh@hstfile=\write6 -\openout6 = `HYPROCESSOR_ATBD.hst'. - - (HYPROCESSOR_ATBD.ver) (HYPROCESSOR_ATBD.ver -Underfull \vbox (badness 10000) detected at line 10 - [] - -) [2] [3] -\tf@toc=\write7 -\openout7 = `HYPROCESSOR_ATBD.toc'. - - [4] (HYPROCESSOR_ATBD_INTRO.tex [5]) [6] (HYPROCESSOR_ATBD_INSTR.tex) [7] (HYP -ROCESSOR_ATBD_ALGO.tex) [8] (HYPROCESSOR_ATBD_IMPLE.tex) [9] -(HYPROCESSOR_ATBD_PROD.tex) [10] [11] -No file HYPROCESSOR_ATBD.bbl. -Package atveryend Info: Empty hook `BeforeClearDocument' on input line 143. -Package atveryend Info: Empty hook `AfterLastShipout' on input line 143. -(HYPROCESSOR_ATBD.aux) -Package atveryend Info: Executing hook `AtVeryEndDocument' on input line 143. -Package atveryend Info: Executing hook `AtEndAfterFileList' on input line 143. - - -Package rerunfilecheck Warning: File `HYPROCESSOR_ATBD.out' has changed. -(rerunfilecheck) Rerun to get outlines right -(rerunfilecheck) or use package `bookmark'. - -Package rerunfilecheck Info: Checksums for `HYPROCESSOR_ATBD.out': -(rerunfilecheck) Before: -(rerunfilecheck) After: A48F371C96199EDD81B50335DB6EBCBE;676. - -LaTeX Warning: Label(s) may have changed. Rerun to get cross-references right. - -Package atveryend Info: Empty hook `AtVeryVeryEnd' on input line 143. - ) -Here is how much of TeX's memory you used: - 13727 strings out of 481556 - 215498 string characters out of 2923641 - 554306 words of memory out of 3000000 - 28634 multiletter control sequences out of 15000+200000 - 541312 words of font info for 59 fonts, out of 3000000 for 9000 - 1141 hyphenation exceptions out of 8191 - 69i,15n,59p,926b,732s stack positions out of 5000i,500n,10000p,200000b,50000s - -Output written on HYPROCESSOR_ATBD.pdf (11 pages, 209666 bytes). -PDF statistics: - 112 PDF objects out of 1000 (max. 8388607) - 30 named destinations out of 1000 (max. 500000) - 6 words of extra memory for PDF output out of 10000 (max. 10000000) - diff --git a/docs/atbd/HYPROCESSOR_ATBD.out b/docs/atbd/HYPROCESSOR_ATBD.out deleted file mode 100755 index b871acfa..00000000 --- a/docs/atbd/HYPROCESSOR_ATBD.out +++ /dev/null @@ -1,11 +0,0 @@ -\BOOKMARK [1][-]{section*.2}{Executive summary}{}% 1 -\BOOKMARK [1][-]{section.1}{Introduction}{}% 2 -\BOOKMARK [2][-]{subsection.1.1}{Scope and objectives}{section.1}% 3 -\BOOKMARK [2][-]{subsection.1.2}{Algorithm justification}{section.1}% 4 -\BOOKMARK [3][-]{subsubsection.1.2.1}{Quantities of interest}{subsection.1.2}% 5 -\BOOKMARK [3][-]{subsubsection.1.2.2}{Uncertainty}{subsection.1.2}% 6 -\BOOKMARK [1][-]{section.2}{Instrument and Network Design}{}% 7 -\BOOKMARK [1][-]{section.3}{Processor Algorithm Description}{}% 8 -\BOOKMARK [1][-]{section.4}{Processor Implementation}{}% 9 -\BOOKMARK [1][-]{section.5}{Products}{}% 10 -\BOOKMARK [1][-]{appendix.1.A}{Appendix ...}{}% 11 diff --git a/docs/atbd/HYPROCESSOR_ATBD.pdf b/docs/atbd/HYPROCESSOR_ATBD.pdf deleted file mode 100755 index 14e1964d..00000000 Binary files a/docs/atbd/HYPROCESSOR_ATBD.pdf and /dev/null differ diff --git a/docs/atbd/HYPROCESSOR_ATBD.synctex.gz b/docs/atbd/HYPROCESSOR_ATBD.synctex.gz deleted file mode 100755 index 3a691f4d..00000000 Binary files a/docs/atbd/HYPROCESSOR_ATBD.synctex.gz and /dev/null differ diff --git a/docs/atbd/HYPROCESSOR_ATBD.toc b/docs/atbd/HYPROCESSOR_ATBD.toc deleted file mode 100755 index d56e56bc..00000000 --- a/docs/atbd/HYPROCESSOR_ATBD.toc +++ /dev/null @@ -1,12 +0,0 @@ -\babel@toc {english}{} -\contentsline {section}{Executive summary}{3}{section*.2}% -\contentsline {section}{\numberline {1}Introduction}{5}{section.1}% -\contentsline {subsection}{\numberline {1.1}Scope and objectives}{5}{subsection.1.1}% -\contentsline {subsection}{\numberline {1.2}Algorithm justification}{5}{subsection.1.2}% -\contentsline {subsubsection}{\numberline {1.2.1}Quantities of interest}{5}{subsubsection.1.2.1}% -\contentsline {subsubsection}{\numberline {1.2.2}Uncertainty}{6}{subsubsection.1.2.2}% -\contentsline {section}{\numberline {2}Instrument and Network Design}{7}{section.2}% -\contentsline {section}{\numberline {3}Processor Algorithm Description}{8}{section.3}% -\contentsline {section}{\numberline {4}Processor Implementation}{9}{section.4}% -\contentsline {section}{\numberline {5}Products}{10}{section.5}% -\contentsline {section}{Appendix \numberline {A}...}{11}{appendix.1.A}% diff --git a/docs/atbd/HYPROCESSOR_ATBD.ver b/docs/atbd/HYPROCESSOR_ATBD.ver deleted file mode 100755 index 3772cf0e..00000000 --- a/docs/atbd/HYPROCESSOR_ATBD.ver +++ /dev/null @@ -1,10 +0,0 @@ -\begin {longtable}{@{}|l|l|>{\hsize =\vhAuthorColWidth }X|>{\hsize =\vhChangeColWidth }X|@{}}\hline -\textbf {\vhversionname } - & \textbf {\vhdatename } - & \textbf {\vhauthorname } - & \textbf {\vhchangename }\\\hline -0.0 - & 22.08.2019 - & \listset {Samuel Hunt, Agnieszka Bia\l {}ek, Niall Origo} - & Initial draft\\\hline -\end {longtable} diff --git a/docs/atbd/~$pernets-NPL-Algorithm_Theoretical_Basis_Document-v0.0.docx b/docs/atbd/~$pernets-NPL-Algorithm_Theoretical_Basis_Document-v0.0.docx deleted file mode 100755 index 26e1d62b..00000000 Binary files a/docs/atbd/~$pernets-NPL-Algorithm_Theoretical_Basis_Document-v0.0.docx and /dev/null differ diff --git a/docs/file_formats/Fileformat_L1_SHunt.png b/docs/file_formats/Fileformat_L1_SHunt.png deleted file mode 100755 index f3fb528a..00000000 Binary files a/docs/file_formats/Fileformat_L1_SHunt.png and /dev/null differ diff --git a/docs/file_formats/Fileformat_L2_CGoyens.png b/docs/file_formats/Fileformat_L2_CGoyens.png deleted file mode 100755 index 51c44617..00000000 Binary files a/docs/file_formats/Fileformat_L2_CGoyens.png and /dev/null differ diff --git a/docs/file_formats/Hypernets-Product_Data_Format_Specification-v0.0.docx b/docs/file_formats/Hypernets-Product_Data_Format_Specification-v0.0.docx deleted file mode 100755 index 9b3122ec..00000000 Binary files a/docs/file_formats/Hypernets-Product_Data_Format_Specification-v0.0.docx and /dev/null differ diff --git a/docs/file_formats/Hypernets-Product_Data_Format_Specification-v0.5.docx b/docs/file_formats/Hypernets-Product_Data_Format_Specification-v0.5.docx deleted file mode 100755 index d63a0dd8..00000000 Binary files a/docs/file_formats/Hypernets-Product_Data_Format_Specification-v0.5.docx and /dev/null differ diff --git a/docs/file_formats/waterhypernet_datametadata.pdf b/docs/file_formats/waterhypernet_datametadata.pdf deleted file mode 100755 index cf6f6b27..00000000 Binary files a/docs/file_formats/waterhypernet_datametadata.pdf and /dev/null differ diff --git a/docs/file_formats/waterhypernet_datametadata.yml b/docs/file_formats/waterhypernet_datametadata.yml deleted file mode 100755 index b0dbce4e..00000000 --- a/docs/file_formats/waterhypernet_datametadata.yml +++ /dev/null @@ -1,106 +0,0 @@ -# data model segments - -- segment: &dmd Data metadata -- segment: &imd Instrument metadata -- segment: &smd System metadata -- segment: &simd Site metadata -- segment: &comd Components metadata -- segment: &camd Calibration metadata - -# Tables and columns - -- table: Global atrtributes - segment: *dmd - display: accent5 - columns: - system_id: {key: yes} - instrument_id: {key: yes} - site_id: {key: yes} - rad_vnir_id: {key: yes} - irrad_vnir_id: {key: yes} - latitude_average: - latitude_minimum: - latitude_maximum: - longitude_average: - longitude_minimum: - longitude_maximum: - -- table: sequence_cycle - segment: *dmd - display: accent5 - columns: - sequence_id: {key: yes} - subsequence_id: {key: yes} - measurement_configuration_file: - measurement_sequence_file: - relative_azimuth_angle: - timestamp: - solar_zenith_angle: - lu_datetime_average: - lu_datetime_start: - lu_datetime_finish: - ld_datetime_average: - ld_datetime_start: - ld_datetime_finish: - ed_datetime_average: - ed_datetime_start: - ed_datetime_finish: - lu_units: - ld_units: - ed_units: - lw_units: - rhow_units: - rhow_nosc_units: - -- table: Processing - segment: *dmd - display: accent5 - columns: - processor_name: - processor_version: - processor_configuration_file: - processor_atbd: - processor_input: - processor_output: - processor_data_level: - file_created: - inputfile: - fresnel_rhof: - fresnel_type: - fresnel_wind: - fresnel_sza: - fresnel_vza: - fresnel_raa: - rad_inclination: - irr_inclination: - lu_scans_total: - lu_scans_temporalvsincomplete: - lu_scans_good: - ld_scans_total: - ld_scans_temporalvsincomplete: - ld_scans_good: - ed_scans_total: - ed_scans_temporalvsincomplete: - ed_scans_good: - wavelen1: - wavelen2: - epsave: - epsmin: - epsmax: - epsstd: - -- table: Calibration - segment: *dmd - display: accent5 - columns: - rad_vnir_radcal_filename_1: - rad_vnir_radcal_filename_2: - rad_vnir_radcal_date_1: - rad_vnir_radcal_date_2: - rad_vnir_radcal_interp: - irr_vnir_radcal_filename_1: - irr_vnir_radcal_filename_2: - irr_vnir_radcal_date_1: - irr_vnir_radcal_date_2: - irr_vnir_radcal_interp: - diff --git a/docs/file_formats/waterhypernet_metadata.pdf b/docs/file_formats/waterhypernet_metadata.pdf deleted file mode 100755 index 33419137..00000000 Binary files a/docs/file_formats/waterhypernet_metadata.pdf and /dev/null differ diff --git a/docs/file_formats/waterhypernet_metadata_keysonly.pdf b/docs/file_formats/waterhypernet_metadata_keysonly.pdf deleted file mode 100755 index 0ac65d57..00000000 Binary files a/docs/file_formats/waterhypernet_metadata_keysonly.pdf and /dev/null differ diff --git a/docs/images/design-class_interations.png b/docs/images/design-class_interations.png deleted file mode 100755 index ead30376..00000000 Binary files a/docs/images/design-class_interations.png and /dev/null differ diff --git a/docs/images/design-flowchart.png b/docs/images/design-flowchart.png deleted file mode 100755 index eb5d7f31..00000000 Binary files a/docs/images/design-flowchart.png and /dev/null differ diff --git a/docs/sphinx/conf.py b/docs/sphinx/conf.py index 85807a88..cda4aa2a 100755 --- a/docs/sphinx/conf.py +++ b/docs/sphinx/conf.py @@ -18,7 +18,8 @@ import os import sys -sys.path.insert(0, os.path.abspath('sphinx')) + +sys.path.insert(0, os.path.abspath("sphinx")) def run_apidoc(_): @@ -37,15 +38,19 @@ def run_apidoc(_): try: # Sphinx 1.7+ from sphinx.ext import apidoc + apidoc.main(argv) except ImportError: # Sphinx 1.6 (and earlier) from sphinx import apidoc + argv.insert(0, apidoc.__file__) apidoc.main(argv) + def setup(app): - app.connect('builder-inited', run_apidoc) + app.connect("builder-inited", run_apidoc) + # -- General configuration ------------------------------------------------ @@ -57,41 +62,41 @@ def setup(app): # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ - 'sphinx.ext.autodoc', - 'sphinx.ext.mathjax', + "sphinx.ext.autodoc", + "sphinx.ext.mathjax", ] # Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] +templates_path = ["_templates"] # The suffix(es) of source filenames. # You can specify multiple suffix as a list of string: # # source_suffix = ['.rst', '.md'] -source_suffix = '.rst' +source_suffix = ".rst" # The encoding of source files. # # source_encoding = 'utf-8-sig' # The master toctree document. -master_doc = 'index' -atbd = 'atbd' -userguide = 'userguide' +master_doc = "index" +atbd = "atbd" +userguide = "userguide" # General information about the project. -project = u'hypernets_processor' -copyright = u'2020, Hypernets Project Team' -author = u'Hypernets Project Team' +project = "hypernets_processor" +copyright = "2020, Hypernets Project Team" +author = "Hypernets Project Team" # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. -version = u'2.0' +version = "2.0" # The full version, including alpha/beta/rc tags. -release = u'2.0' +release = "2.0" # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. @@ -134,7 +139,7 @@ def setup(app): # show_authors = False # The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' +pygments_style = "sphinx" # A list of ignored prefixes for module index sorting. # modindex_common_prefix = [] @@ -151,7 +156,7 @@ def setup(app): # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. # -html_theme = 'sphinx_rtd_theme' +html_theme = "sphinx_rtd_theme" # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the @@ -185,7 +190,7 @@ def setup(app): # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +html_static_path = ["_static"] # html_context = { # 'css_files': [ @@ -271,36 +276,43 @@ def setup(app): # html_search_scorer = 'scorer.js' # Output file base name for HTML help builder. -htmlhelp_basename = 'hypernets_processordocs' +htmlhelp_basename = "hypernets_processordocs" # -- Options for LaTeX output --------------------------------------------- latex_elements = { - # The paper size ('letterpaper' or 'a4paper'). - # - # 'papersize': 'letterpaper', - - # The font size ('10pt', '11pt' or '12pt'). - # - # 'pointsize': '10pt', - - # Additional stuff for the LaTeX preamble. - # - # 'preamble': '', - - # Latex figure (float) alignment - # - # 'figure_align': 'htbp', + # The paper size ('letterpaper' or 'a4paper'). + # + # 'papersize': 'letterpaper', + # The font size ('10pt', '11pt' or '12pt'). + # + # 'pointsize': '10pt', + # Additional stuff for the LaTeX preamble. + # + # 'preamble': '', + # Latex figure (float) alignment + # + # 'figure_align': 'htbp', } # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ - (atbd, 'hypernets_processor_atbd.tex', u'Hypernets Processor Algorithm Theoretical Basis Document', - u'Hypernets Project Team', 'manual'), - (userguide, 'hypernets_processor_userguide.tex', u'Hypernets Processor User Guide', - u'Hypernets Project Team', 'manual'), + ( + atbd, + "hypernets_processor_atbd.tex", + "Hypernets Processor Algorithm Theoretical Basis Document", + "Hypernets Project Team", + "manual", + ), + ( + userguide, + "hypernets_processor_userguide.tex", + "Hypernets Processor User Guide", + "Hypernets Project Team", + "manual", + ), ] # The name of an image file (relative to this directory) to place at the top of @@ -341,8 +353,13 @@ def setup(app): # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [ - (master_doc, 'hypernets_processor', u'hypernets_processor Documentation', - [author], 1) + ( + master_doc, + "hypernets_processor", + "hypernets_processor Documentation", + [author], + 1, + ) ] # If true, show URL addresses after external links. @@ -356,9 +373,15 @@ def setup(app): # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - (master_doc, 'hypernets_processor', u'Hypernets Processor User Guide', - author, 'hypernets_processor', 'One line description of project.', - 'Miscellaneous'), + ( + master_doc, + "hypernets_processor", + "Hypernets Processor User Guide", + author, + "hypernets_processor", + "One line description of project.", + "Miscellaneous", + ), ] # Documents to append as an appendix to all manuals. diff --git a/docs/sphinx/content/atbd/figures/plot_clearskycheck_irradiance_HYPERNETS_L_GHNA_L1B_IRR_20220804T1000_20231227T1443_v2.0.jpeg b/docs/sphinx/content/atbd/figures/plot_clearskycheck_irradiance_HYPERNETS_L_GHNA_L1B_IRR_20220804T1000_20231227T1443_v2.0.jpeg new file mode 100644 index 00000000..b4619707 Binary files /dev/null and b/docs/sphinx/content/atbd/figures/plot_clearskycheck_irradiance_HYPERNETS_L_GHNA_L1B_IRR_20220804T1000_20231227T1443_v2.0.jpeg differ diff --git a/docs/sphinx/content/atbd/processing/CLEAR_SKY_JSIT.png b/docs/sphinx/content/atbd/processing/CLEAR_SKY_JSIT.png new file mode 100644 index 00000000..14be843b Binary files /dev/null and b/docs/sphinx/content/atbd/processing/CLEAR_SKY_JSIT.png differ diff --git a/docs/sphinx/content/atbd/processing/CLOUDY_SCENE_LOBE.png b/docs/sphinx/content/atbd/processing/CLOUDY_SCENE_LOBE.png new file mode 100644 index 00000000..9f25eb65 Binary files /dev/null and b/docs/sphinx/content/atbd/processing/CLOUDY_SCENE_LOBE.png differ diff --git a/docs/sphinx/content/atbd/processing/MISALIGNMENT_MODEL.png b/docs/sphinx/content/atbd/processing/MISALIGNMENT_MODEL.png new file mode 100644 index 00000000..d22aca2d Binary files /dev/null and b/docs/sphinx/content/atbd/processing/MISALIGNMENT_MODEL.png differ diff --git a/docs/sphinx/content/atbd/processing/NDVI_JSIT.png b/docs/sphinx/content/atbd/processing/NDVI_JSIT.png new file mode 100644 index 00000000..761a9863 Binary files /dev/null and b/docs/sphinx/content/atbd/processing/NDVI_JSIT.png differ diff --git a/docs/sphinx/content/atbd/processing/NDVI_LOBE.png b/docs/sphinx/content/atbd/processing/NDVI_LOBE.png new file mode 100644 index 00000000..fb8cc643 Binary files /dev/null and b/docs/sphinx/content/atbd/processing/NDVI_LOBE.png differ diff --git a/docs/sphinx/content/atbd/processing/REFLECTANCE_BOUNDS_GHNA.png b/docs/sphinx/content/atbd/processing/REFLECTANCE_BOUNDS_GHNA.png new file mode 100644 index 00000000..5d71df87 Binary files /dev/null and b/docs/sphinx/content/atbd/processing/REFLECTANCE_BOUNDS_GHNA.png differ diff --git a/docs/sphinx/content/atbd/processing/post_processing.rst b/docs/sphinx/content/atbd/processing/post_processing.rst index 579d1225..fffec201 100644 --- a/docs/sphinx/content/atbd/processing/post_processing.rst +++ b/docs/sphinx/content/atbd/processing/post_processing.rst @@ -5,13 +5,136 @@ .. _post_processing: +Post Processing - Site-specific Quality Checks +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Post Processing - Site-specific Quality checks -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Once the L2A datasets have been produced, a final set of site-specific quality checks and masks are -applied. The site-specific quality checks and masks are determined after inspection of the first months of -data in consultation of the site owners. The resulting masks are applied to the L2A -dataset and stored as L2B. The same masks are also applied to the L1B datasets and stored as L1D. Only -the L1D and L2B data will be distributed to satellite validation users. +The LANDHYPERNET L2A data still contains erroneous sequences which are not +suitable for Cal/Val, and thus not suitable for public distribution. +The standard L2A processing as described in the previous sections applies general quality checks to the +data. However, no site-specific QC has been applied in the processing to L2A. -These post-processing quality checks are still under development and will be detailed here in the future. \ No newline at end of file +To ensure the highest quality of the L2A data over the Cal/Val sites, additional +quality checks are applied to the data. This is done in two stages: + +1. An offline analysis is performed for each deployment period at each site, to identify + periods with known issues (e.g. due to site heterogeneity, bad deployment conditions, + unsuitable viewing or solar angles, etc). Based on this offline analysis, a range of config + values are determined for each site and deployment period, which are then stored in the job.config file (see :ref:`config`). + +2. During processing to L2B, the site-specific quality checks are applied to the data, + using the config values determined in the offline analysis. Only data suitable for Cal/Val + are retained in the L2B data. + +Flags Set in the L2A Data +------------------------- + +Data flagged with a critical issue in L2A and L1B data are removed. Specifically, the following flags are removed (see also :ref:`using_hypernets`):: + + bad_flags=["pt_ref_invalid", "half_of_scans_masked", "not_enough_dark_scans", "not_enough_rad_scans", + "not_enough_irr_scans", "no_clear_sky_irradiance", "variable_irradiance", + "half_of_uncertainties_too_big", "discontinuity_VNIR_SWIR", "single_irradiance_used"] + +See :ref:`flags` for more information on these flags. + +Sequences Outside Any Valid Deployment Date Ranges +-------------------------------------------------- + +The datetime of the sequence being processed is simply checked against the valid deployment ranges +(see https://www.landhypernet.org.uk/site_descriptions for these date ranges for each site). + +Periods with Bad Deployment Conditions +-------------------------------------- + +Such date/time/datetime periods can be revealed as part of the offline analysis, or can be known by the site owner +due to expert knowledge about the site, knowledge about the deployment, etc. + +Periods with Site Heterogeneity +------------------------------- + +Homogeneous periods are defined using NDVI at vegetated sites. We first define a reference period through manual analysis +of images from sites and conversations with site owners and calculate the standard deviation of the NDVI in this period. +The homogeneous periods are then defined as periods where a rolling standard deviation of NDVI is below 1.5 times the reference standard deviation. +Any measurements outside these homogeneous periods are removed. + +.. figure:: NDVI_LOBE.png + :width: 600px + :align: center + :alt: NDVI timeseries for LOBE April 2025 – August 2025 + + (Top) NDVI timeseries for LOBE April 2025 – August 2025. Green dots show NDVI of a measurement; blue line shows rolling average of 100 measurements with rolling standard deviation as shaded region. (Bottom) Rolling standard deviation of NDVI timeseries; red, orange, and yellow lines show 1, 2, and 3 times the reference standard deviation, respectively, where the reference is defined as the standard deviation of a known homogeneous period (here July 2025). + +Unsuitable Viewing Zenith & Azimuth Angle +----------------------------------------- + +Such angles with tolerances can be revealed as part of the offline analysis (e.g. see Figure below), or can be known by the site owner due to expert site knowledge, knowledge about the deployment, etc. +To mask out shadows, there is also the option to specify the relative azimuth angle (shadows typically at 0 degrees) and to select all zenith angles smaller than the solar zenith angle. + +.. figure:: NDVI_JSIT.png + :width: 600px + :align: center + :alt: NDVI Timeseries for JSIT Jan – Jul 2025 coloured by viewing zenith angle + + NDVI Timeseries for JSIT Jan – Jul 2025 coloured by viewing zenith angle, lower NDVI can be seen close to nadir (0 and 5 degrees). Close to nadir the sensor is looking at bare soil next to the mast and this data is removed in L2B. + +Unsuitable Sun Zenith Angle +-------------------------- + +Any measurements with a solar zenith angle above 60 degrees are removed. This value is changed for some high-latitude sites where very low solar zeniths occur; these are defined in offline analysis. + +Poorly Performing Wavelength Ranges +----------------------------------- + +Wavelength ranges with known issues (such as calibration failures) are masked. + +.. figure:: CLEAR_SKY_JSIT.png + :width: 600px + :align: center + :alt: Clear sky check plot for JSIT irradiance measurements + + Clear sky check plot for JSIT irradiance measurements on 2024-05-17 at 9:00 UTC. There is an erroneous peak in the measured irradiance around 585 nm. + +A Note on Clear-Sky Modelling +----------------------------- + +Site-specific clear sky models are generated using the LibRadtran Radiative Transfer software. The surface was set to be Lambertian with reflectance equal to the mean L2A spectral reflectance for the specific site at a viewing zenith angle of 20 degrees. Atmospheric properties were taken from ERA5 reanalysis data averaged over 2023. One set of models was run with no aerosols, and one set of models was run with the median aerosol optical depth taken from CAMS reanalysis data over the site in 2023. BOA irradiance calculations (including direct and diffuse contributions) were then performed using RT for a range of different solar zenith angles (0 to 80 in steps of 10). The altitude for each simulation was also set specifically from the Copernicus DEM, extracted at each site. During analysis, models are interpolated to the solar zenith angle of the measurements. + +Misalignment of the HYPSTAR Sensor +---------------------------------- + +Sequences with erroneous irradiances were spotted with the likely cause being a slight misalignment of the HYPSTAR sensor to the vertical during the irradiance measurements, causing a trend in the ratio between a clear sky model and the irradiance measurements throughout the day. The observed ratio between the clear sky model with median aerosols (described above) and the irradiance measurements is used to fit a modelled ratio. The modelled ratio has the following free parameters: misalignment viewing zenith angle (the tilt from the vertical); misalignment viewing azimuth angle (the azimuthal direction of the tilt with 0 as North); offset (to account for errors in the clear sky model and drift in the calibration). These free parameters are fit using a Markov Chain Monte Carlo (MCMC) process. The data is then normalised using a correction factor so that the average irradiance is the same as prior to the misalignment correction. + +.. figure:: MISALIGNMENT_MODEL.png + :width: 600px + :align: center + :alt: Illustration of the misalignment modelling results + + Illustration of the misalignment modelling results. Data points show data from JAES May-Sep 2023 in bins of time-of-day. Errorbars show standard deviation within the bin. Top: observed ratio of clear sky model and irradiance measurements, which shows some unexpected variability during the day. 2nd panel: modelled ratio, allowing for viewing zenith and azimuth angle to be pointed not exactly at nadir (also includes an offset to account for mismatches between model and observations). 3rd panel: residual between observations and models. 4th panel: normalised corrected ratios (by using the modelled misalignment, but not applying offset). Bottom: solar zenith angle in each bin. + +More Stringent Clear Sky Check +------------------------------ + +The improved clear sky consists of discarding any sequences where the measured irradiances (after correction for misalignment) are either more than 5% higher than the model without aerosols at 550 nm, or more than 10% smaller than the clear sky model with median aerosols. Note that various thresholds and aerosol loads were experimented with, and this combination was found to hold the best trade-off between masking erroneous sequences and wrongfully masking valid sequences. + +.. figure:: CLOUDY_SCENE_LOBE.png + :width: 600px + :align: center + :alt: Example of cloudy scene from LOBE + + Example of cloudy scene from LOBE which passed L2A QC. More stringent L2B site-specific cloud checks correctly identify this as a cloudy scene. + +Unsuitable Reflectances for Specific Site +----------------------------------------- + +Based on offline analysis, realistic upper and lower bounds on the reflectance values are provided in bins along multiple dimensions (typically solar zenith angle, viewing zenith angle, relative azimuth angle, and optionally temporal bins), to allow for angular variability due to BRDF effects (and optionally seasonal effects). The bounds themselves come from binning the data within these bins, and then iteratively removing outliers (as compared to other values in the bin, as well as fits to the overall angular and temporal variability). + +.. figure:: REFLECTANCE_BOUNDS_GHNA.png + :width: 600px + :align: center + :alt: Illustration of the reflectance bounds for GHNA May 2022 – Oct 2023 + + Illustration of the reflectance bounds for GHNA May 2022 – Oct 2023. The bounds are shown in blue, data which pass in green, and data which are removed in red. + +Other Unusual Conditions +------------------------ + +Any remaining issues, due to unusual conditions that occur occasionally, can be omitted by manually specifying the sequence ID. diff --git a/docs/sphinx/content/atbd/processing/processing_config.rst b/docs/sphinx/content/atbd/processing/processing_config.rst index 9e4ff66e..686572d8 100755 --- a/docs/sphinx/content/atbd/processing/processing_config.rst +++ b/docs/sphinx/content/atbd/processing/processing_config.rst @@ -61,7 +61,7 @@ Default configuration files can be changed in `hypernets_processor/hypernets_pro - system longitude - longitude of site * - angle2use - - Ange used to compute the viewing geometry (i.e. pt_ref is the pan and tilt angles with the HYPSTAR as reference - use offset pan and tilt to retrieve viewing geometry with true North) + - Angle used to compute the viewing geometry (i.e. pt_ref is the pan and tilt angles with the HYPSTAR as reference - use offset pan and tilt to retrieve viewing geometry with true North) - pt_ref, pt_ask or pt_abs **Table 2: Processor** @@ -84,19 +84,25 @@ Default configuration files can be changed in `hypernets_processor/hypernets_pro - 0 no uncertainties, suggested > 100 when uncertainties * - max_level - maximum level of processing - - Default: L2A + - Default: L2B * - uncertainty_l1a - uncertainty computation of the level L1A - Default: False - * - bad_wavelenth_ranges + * - bad_wavelength_ranges - wavelength ranges for which uncertainties are expected to be high and ignored when triggering flags and anomalies - Default: 757.5-767.5, 1350-1390 + * - delay_hours + - minimum delay between sequence acquisition and processing (e.g. to allow time for ancillary data download) + - Default: 48 * - verbose - printing warnings and errors in terminal - Default: False * - archive_directory - directory where products will be saved when running automated processing - + * - reprocess_from + - Optional processing level from which to restart reprocessing (overrides defaults if set) + - e.g. L1B **Table 3: Databases** @@ -217,6 +223,12 @@ Default configuration files can be changed in `hypernets_processor/hypernets_pro * - measurement_function_calibrate - measurement function used for the calibration of the radiance and irradiance scans - e.g. StandardMeasurementFunction + * - calibration_interpolation_method + - interpolation method for interpolating between pre-deployment and post-deployment calibration + - e.g. previous or linear + * - calibration_file_version + - version of the calibration file to be used + - e.g. 2.4 **Table 8: Interpolate** @@ -526,7 +538,7 @@ Default configuration files can be changed in `hypernets_processor/hypernets_pro * - uncertainty_l1a - uncertainty computation of the level L1A - Default: False - * - bad_wavelenth_ranges + * - bad_wavelength_ranges - wavelength ranges for which uncertainties are expected to be high and ignored when triggering flags and anomalies - Default: 757.5-767.5, 1350-1390 * - verbose @@ -535,6 +547,12 @@ Default configuration files can be changed in `hypernets_processor/hypernets_pro * - archive_directory - directory where products will be saved when running automated processing - + * - delay_hours + - minimum delay between sequence acquisition and processing (e.g. to allow time for ancillary data download) + - e.g. 1 + * - reprocess_from + - Optional processing level from which to restart reprocessing (overrides defaults if set) + - e.g. L1B **Table 3: Databases** @@ -732,6 +750,12 @@ Default configuration files can be changed in `hypernets_processor/hypernets_pro * - write_l2a - Write output file L2A - Default: True + * - write_l2b + - Write output file L2B + - Default: True + * - write_l1d + - Write output file L1D + - Default: True **Table 12: Plotting** @@ -769,6 +793,12 @@ Default configuration files can be changed in `hypernets_processor/hypernets_pro * - plot_l2a - Plotting L2A data - Default: True + * - plot_l2b + - Plotting L2B data + - Default: True + * - plot_l1d + - Plotting L1D data + - Default: True * - plot_uncertainty - Plotting uncertainties - Default: True @@ -787,4 +817,74 @@ Default configuration files can be changed in `hypernets_processor/hypernets_pro * - plot_polar_max - macimum reflectace in the colourscale for polar plot showing the angular variation in reflectance - + * - plot_polar_ndvi + - Plot NDVI in polar plots + - Default: True + +**Table 13: Site_specific_QC** + +.. list-table:: + :widths: 20 60 20 + :header-rows: 1 + + * - Configuration parameter + - Definition + - Options/example + * - deployment_periods + - List of deployment periods with start/stop dates, instrument serial number, validity, and comments. + - e.g [{"start_date": "2022-05-17", "stop_date": "2023-10-17", "HYPSTAR_SN": 220261, "valid": True, "comments": "v1 instrument"}, ...] + * - bad_dates_period + - List of date ranges to exclude for each period. + - e.g. [None, None, ...] + * - limit_tod_period + - List of time-of-day limits for each period. + - e.g. [["08:30","19:00"], ...] + * - bad_sequences_period + - List of sequence IDs to exclude for each period. + - e.g. [[], [], ...] + * - max_sza_period + - Maximum solar zenith angle allowed for each period. + - e.g. [60, 60, ...] + * - bad_viewing_angles_period + - List of viewing zenith/azimuth angle pairs to mask for each period. + - e.g. [[(0, "all"), (5, "all")], ...] + * - bad_solar_angles_period + - List of solar angle pairs to mask for each period. + - e.g. [[], [], ...] + * - bad_relative_angles_period + - List of relative angles to mask for each period. + - e.g. [[("sza", 0), ("sza", 360)], ...] + * - angle_tolerance + - Tolerance for angle matching for bad angle masking (degrees). + - e.g. 1 + * - raa_angle_tolerance + - Tolerance for relative azimuth angle matching for bad angle masking (degrees). + - e.g. 10 + * - bad_wavelengths_period + - List of wavelength ranges to exclude for each period. + - e.g. [[], [], ...] + * - postprocessing_qc_file_period + - List of filenames for post-processing QC bounds for each period. + - e.g. ["GHNA_2022May_2023Oct_bounds.nc", ...] + * - misalignment_vza + - Misalignment viewing zenith angle for each period. + - e.g. [1.01, 1.96, ...] + * - misalignment_vza_unc + - Uncertainty for misalignment viewing zenith angle. + - e.g. [0.07, 0.06, ...] + * - misalignment_vaa + - Misalignment viewing azimuth angle for each period. + - e.g. [97, -72, ...] + * - misalignment_vaa_unc + - Uncertainty for misalignment viewing azimuth angle. + - e.g. [6, 3, ...] + * - misalignment_corr + - Correction factor for misalignment. + - e.g. [0.99, 1.01, ...] + * - misalignment_corr_unc + - Uncertainty for correction factor. + - e.g. [0.01, 0.02, ...] + * - apply_site_specific_cloud_check + - Enable/disable site-specific cloud checks. + - True/False diff --git a/docs/sphinx/content/atbd/products/product_files.rst b/docs/sphinx/content/atbd/products/product_files.rst index cfd49f32..fdf2c7dc 100755 --- a/docs/sphinx/content/atbd/products/product_files.rst +++ b/docs/sphinx/content/atbd/products/product_files.rst @@ -16,14 +16,6 @@ The naming convention is intended to allow the unique identification of all prod *SYSTEM_NETWORK_SITEID_LEVEL_TYPE_ACQUISITIONDATETIME_PROCESSINGDATETIME_version.nc* -For the HYPSTAR calibration data, the file name is similar except that it includes the system_id and the date and time of the calibration. - -*SYSTEM_NETWORK_SYSTEMID_TYPE_CALIBRATIONDATETIME_version.nc* - -For the RGB images taken during the measurements, the file name is similar except that it also includes the series ID, viewing and azimuth angle. - -*SYSTEM_NETWORK_SITEID_TYPE_ACQUISITIONDATETIME_PROCESSINGDATETIME_SERIESID_ZENITH_AZIMUTH_version.nc* - The files are stored in the NetCDF data format and so have the extension “.nc” (except for the RGB images taken during the measurements by the instrument). The definition of the data fields and their allowed contents is described as follows: **Table 1: File name conventions** @@ -80,6 +72,15 @@ Table 2 defines the abbreviated name convention applicable to the individual Hyp | VEIT | Aqua Alta Oceanographic Tower, Venice, Italy | +---------+----------------------------------------------------------+ +For the HYPSTAR calibration data, the file name is similar except that it includes the system_id and the date and time of the calibration. + +*SYSTEM_NETWORK_SYSTEMID_TYPE_CALIBRATIONDATETIME_version.nc* + +For the RGB images taken during the measurements, the file name is similar except that it also includes the series ID, viewing and azimuth angle. + +*SYSTEM_NETWORK_SITEID_TYPE_ACQUISITIONDATETIME_PROCESSINGDATETIME_SERIESID_ZENITH_AZIMUTH_version.nc* + + Data level ---------- The end-to-end prototype processor takes the data from acquisition (raw data) to application of calibration and quality controls, computation of correction factors (e.g. Fresnel correction for water processing), temporal interpolation to coincident timestamps, processing to surface reflectance and averaging per series. To account for all these steps different data levels have been defined (see Table 3). See :ref:`data_structure` for a detailed explanantion of the terminology used. diff --git a/docs/sphinx/content/atbd/products/table_anomalies.csv b/docs/sphinx/content/atbd/products/table_anomalies.csv index 16acfa42..a46a70a3 100755 --- a/docs/sphinx/content/atbd/products/table_anomalies.csv +++ b/docs/sphinx/content/atbd/products/table_anomalies.csv @@ -9,10 +9,19 @@ ms, series_missing , L & W , L1B , Check if there are any missing series (either u, negative_unc , L & W , L1B , There are negative uncertainties, True , o, half_of_unc_too_big , L & W , L1B , More than 50\% of data has random error above 100\% (likely corrupted or dark data), False , 17 nu, check_valid_irradiance , L & W , L1C , Halt processing if `variable_irradiance' flag was triggered at previous level, True , -in, check_valid_sequence , L & W , L1C , Halt processing if there are no valid series (flagged by `not_enough_dark_scans'; `not_enough_irr_scans'; `not_enough_rad_scans' or `vza_irradiance')., True , +in, check_valid_sequence , L & W , L1C , Halt processing if there are no valid series (not present or flagged by `not_enough_dark_scans'; `not_enough_irr_scans'; `not_enough_rad_scans' or `vza_irradiance')., True , cl, no_clear_sky_sequence , L & W , L1C , If all irradiance series are flagged with the `no_clear_sky_irradiance' flag, False , 19 d, discontinuity_VNIR_SWIR , L , L1B , Checks if the VNIR and SWIR are different by more than 25\%, False, 20 nd, variable_radiance, W , L1C , More than 10\% difference between start and end $L_d$ at 550~nm, True , l, ld_missing, W , L1C , Ld missing for sky reflectance correction, True, -x, unexpected_error, L & W , , unexpected error during processing; catches all errors, True, - +per, not_deploy_period, L , L2B , Sequence not within deployment periods, True, +val, not_valid_period, L , L2B ,sequence within deployment periods that is not valid for satellite validation, True, +tod, bad_time_of_day, L , L2B , Sequence not within selected time of day, True, +hsn, hypstar_not_deploy_period, L , L2B , HYPSTAR_SN does not match deployment periods, True, +scl, no_clear_sky_site_specific, L , L2B , Site-specific clear sky check has failed (irr not within 10% of model), True, +npr, no_pre_files, L , L2B , No pre-existing files available, while config is set up to reprocess from pre-existing files, True, +man, manual_discard, L , L2B , This sequences was manually discarded as unreliable in the config file, True, +wns, wrong_number_series, L2B , Wrong number of series (l2a and l1b rad do not match in number of series). No L1d file is produced, False, +nos, no_series_site_specific, L , L2B , No series was present in the sequences after applying all site-specific QC, True, +hos, half_of_series_site_specific, L, L2B, More than half series were removed by site-specific QC checks (excluding bad viewing angles and bad dates), True, +x, unexpected_error, L & W , , unexpected error during processing; catches all errors, True, \ No newline at end of file diff --git a/docs/sphinx/content/atbd/products/table_flags_common.csv b/docs/sphinx/content/atbd/products/table_flags_common.csv index 829aa512..63bbc10f 100644 --- a/docs/sphinx/content/atbd/products/table_flags_common.csv +++ b/docs/sphinx/content/atbd/products/table_flags_common.csv @@ -1,20 +1,20 @@ Bit #,name,network,level,Description,anomaly raised -1, lon_default, L & W , L0A , Default longitude as given in the configuration file is used (missing Lon in metadata) , -2, lat_default, L & W , L0A , same as `lon_default' but for latitude , -3, pt_ref_invalid , L & W , L0A , No effective pan/tilt is returned and the requested pan/tilt was used instead , -4, bad_pointing , L & W , L0A , Difference between requested and effective pan/tilt angle is >= 3° , `a' -5, outliers, L & W , L1A , If the spectrally integrated signal of the scan is more than 3 sigma or more than 25\% (whichever is largest) removed from the mean; it is masked and will not be used when averaging the series. This process is repeated until convergence and applied to the measured (ir)radiances and to the darks. , -6, L0_threshold, L & W , L1A , Check if any of the spectral pixels are saturated; i.e. digital number DN>=64000 , -7, L0_discontinuity, L & W , L1A , Check if there are missing values in the spectrum; or significant discontinuities (\Delta DN>10^4) , -8, dark_masked, L & W , L1A , If any of the darks have been masked by `outliers'; `L0_threshold'; and/or; `L0_discontinuity' , -9, half_of_scans_masked, L & W , L0B , Less than half of the scans for a series passed quality checks `bad_pointing'; `outliers'; `L0_threshold'; and; `L0_discontinuity' , -10, not_enough_dark_scans, L & W , L0B , Not enough valid dark scans for this series (# valid dark scans < n_valid_dark from the config file) , `nld' -11, not_enough_rad_scans, L & W , L0B , Not enough valid radiance scans for this series (# valid radiance scans < n_valid_rad from the config file) ,`nlu' -12, not_enough_irr_scans, L & W , L0B , Not enough valid irradiance scans for this series (# valid irradiance scans < n_valid_irr from the config file) , `ned' -13, series_missing , L & W , L1B , Check if there are any missing series (either not present or flagged by `not_enough_dark_scans'; `not_enough_irr_scans'; `not_enough_rad_scans' or `vza_irradiance') , `ms' -14, vza_irradiance , L & W , L1B , One of the irradiance measurements did not have theta_v=180° (within 2° tolerance) so has been masked , -15, no_clear_sky_irradiance, L & W , L1B , More than 10% of the wavelength bands have a difference of more than 50% with the clear-sky model , -16, variable_irradiance, L & W , L1B , More than 10\% difference between start and end E_d at 550~nm , -17, half_of_unc_too_big , L & W , L1B , More than 50\% of data has random error above 100% (likely corrupted or dark data) , `o' -18, single_irradiance_used, L & W , L1C , If only one series of irradiance is used for the computation of the reflectance , -19, no_clear_sky_sequence , L & W , L1C , If all irradiance series are flagged with the `no_clear_sky_irradiance' flag , `cl' \ No newline at end of file +0, lon_default, L & W , L0A , Default longitude as given in the configuration file is used (missing Lon in metadata) , +1, lat_default, L & W , L0A , same as `lon_default' but for latitude , +2, pt_ref_invalid , L & W , L0A , No effective pan/tilt is returned and the requested pan/tilt was used instead , +3, bad_pointing , L & W , L0A , Difference between requested and effective pan/tilt angle is >= 3° , `a' +4, outliers, L & W , L1A , If the spectrally integrated signal of the scan is more than 3 sigma or more than 25\% (whichever is largest) removed from the mean; it is masked and will not be used when averaging the series. This process is repeated until convergence and applied to the measured (ir)radiances and to the darks. , +5, L0_threshold, L & W , L1A , Check if any of the spectral pixels are saturated; i.e. digital number DN>=64000 , +6, L0_discontinuity, L & W , L1A , Check if there are missing values in the spectrum; or significant discontinuities (\Delta DN>10^4) , +7, dark_masked, L & W , L1A , If any of the darks have been masked by `outliers'; `L0_threshold'; and/or; `L0_discontinuity' , +8, half_of_scans_masked, L & W , L0B , Less than half of the scans for a series passed quality checks `bad_pointing'; `outliers'; `L0_threshold'; and; `L0_discontinuity' , +9, not_enough_dark_scans, L & W , L0B , Not enough valid dark scans for this series (# valid dark scans < n_valid_dark from the config file) , `nld' +10, not_enough_rad_scans, L & W , L0B , Not enough valid radiance scans for this series (# valid radiance scans < n_valid_rad from the config file) ,`nlu' +11, not_enough_irr_scans, L & W , L0B , Not enough valid irradiance scans for this series (# valid irradiance scans < n_valid_irr from the config file) , `ned' +12, series_missing , L & W , L1B , Check if there are any missing series (either not present or flagged by `not_enough_dark_scans'; `not_enough_irr_scans'; `not_enough_rad_scans' or `vza_irradiance') , `ms' +13, vza_irradiance , L & W , L1B , One of the irradiance measurements did not have theta_v=180° (within 2° tolerance) so has been masked , +14, no_clear_sky_irradiance, L & W , L1B , More than 10% of the wavelength bands have a difference of more than 50% with the clear-sky model , +15, variable_irradiance, L & W , L1B , More than 10\% difference between start and end E_d at 550~nm , +16, half_of_unc_too_big , L & W , L1B , More than 50\% of data has random error above 100% (likely corrupted or dark data) , `o' +17, single_irradiance_used, L & W , L1C , If only one series of irradiance is used for the computation of the reflectance , +18, no_clear_sky_sequence , L & W , L1C , If all irradiance series are flagged with the `no_clear_sky_irradiance' flag , `cl' \ No newline at end of file diff --git a/docs/sphinx/content/atbd/products/table_flags_land.csv b/docs/sphinx/content/atbd/products/table_flags_land.csv index bd201588..a82338f1 100644 --- a/docs/sphinx/content/atbd/products/table_flags_land.csv +++ b/docs/sphinx/content/atbd/products/table_flags_land.csv @@ -1,2 +1,2 @@ Bit #,name,network,level,Description,anomaly raised -20, discontinuity_VNIR_SWIR , L , L1B , Checks if the VNIR and SWIR are different by more than 25% ,`d' \ No newline at end of file +19, discontinuity_VNIR_SWIR , L , L1B , Checks if the VNIR and SWIR are different by more than 25% ,`d' \ No newline at end of file diff --git a/docs/sphinx/content/atbd/products/table_flags_water.csv b/docs/sphinx/content/atbd/products/table_flags_water.csv index a366b83d..9f3d7d2d 100644 --- a/docs/sphinx/content/atbd/products/table_flags_water.csv +++ b/docs/sphinx/content/atbd/products/table_flags_water.csv @@ -1,12 +1,12 @@ Bit #,name,network,level,Description,anomaly raised -20, single_skyradiance_used, W , L1C , If only one series of downwelling radiance is used for the computation of the reflectance , -21, lu_eq_missing, W , L1C , If there is no upwelling and downwelling radiance pair with similar pointing azimuth angles (within 1° tolerance) , `l' -22, rhof_angle_missing, W , L1C , If there is no downwelling radiance scans at the appropriate viewing zenith angle (i.e. 180°-theta_v; within 1° tolerance), `l' -23, rhof_default, W , L1C , If the viewing geometry of the upwelling and downwelling radiance measurements are outside the viewing geometry range of the selected LUT for the `rhof_option' (e.g. Delta \phi > 180° when using the LUT from Mobley et al 1999 a default rho_F is used for the air-water interface correction factor (default: rho_F=0.0256) , -24, temp_variability_irr, W , L1C , If the difference in E_d scans does not exceed a given threshold between two neighbouring scans (default: threshold = 25% for wavelength = 550nm; see also Ruddick et al. (2006)) , -25, temp_variability_rad, W , L1C , If the difference in L_d or L_u scans does not exceed a given threshold between two neighbouring scans (default: threshold = 25% for wavelength = 550nm; Ruddick et al. (2006)) , -26, min_nbred, W , L1C , If the total number of scans not flagged by either `L0_threshold'; `bad_pointing' or `outliers' is less than a given threshold (default: 3) , `ned' -27, min_nbrlu, W , L1C , If the total number of scans not flagged by either `L0_threshold'; `bad_pointing' or `outliers' is less than a given threshold (default: 3) , `nlu' -28, min_nblsky, W , L1C , If the total number of scans not flagged by either `L0_threshold'; `bad_pointing' or `outliers' is less than a given threshold (default: 3) , `nld' -29, def_wind_flag, W , L1C , If a default wind speed is used (by default: wind speed = 2m/s) , -30, simil_fail, W , L1C , If the quality check applied on the NIR similarity spectrum is not verified as suggested by Ruddick et al. (2005) (see Section 3.2 and Fig. 4 in Ruddick et al. (2005)) with default values for the computation of the NIR Similarity being 780 and 870~nm; the reference wavelength 670~nm and the threshold 5% , +19, single_skyradiance_used, W , L1C , If only one series of downwelling radiance is used for the computation of the reflectance , +20, lu_eq_missing, W , L1C , If there is no upwelling and downwelling radiance pair with similar pointing azimuth angles (within 1° tolerance) , `l' +21, rhof_angle_missing, W , L1C , If there is no downwelling radiance scans at the appropriate viewing zenith angle (i.e. 180°-theta_v; within 1° tolerance), `l' +22, rhof_default, W , L1C , If the viewing geometry of the upwelling and downwelling radiance measurements are outside the viewing geometry range of the selected LUT for the `rhof_option' (e.g. Delta \phi > 180° when using the LUT from Mobley et al 1999 a default rho_F is used for the air-water interface correction factor (default: rho_F=0.0256) , +23, temp_variability_irr, W , L1C , If the difference in E_d scans does not exceed a given threshold between two neighbouring scans (default: threshold = 25% for wavelength = 550nm; see also Ruddick et al. (2006)) , +24, temp_variability_rad, W , L1C , If the difference in L_d or L_u scans does not exceed a given threshold between two neighbouring scans (default: threshold = 25% for wavelength = 550nm; Ruddick et al. (2006)) , +25, min_nbred, W , L1C , If the total number of scans not flagged by either `L0_threshold'; `bad_pointing' or `outliers' is less than a given threshold (default: 3) , `ned' +26, min_nbrlu, W , L1C , If the total number of scans not flagged by either `L0_threshold'; `bad_pointing' or `outliers' is less than a given threshold (default: 3) , `nlu' +27, min_nblsky, W , L1C , If the total number of scans not flagged by either `L0_threshold'; `bad_pointing' or `outliers' is less than a given threshold (default: 3) , `nld' +28, def_wind_flag, W , L1C , If a default wind speed is used (by default: wind speed = 2m/s) , +29, simil_fail, W , L1C , If the quality check applied on the NIR similarity spectrum is not verified as suggested by Ruddick et al. (2005) (see Section 3.2 and Fig. 4 in Ruddick et al. (2005)) with default values for the computation of the NIR Similarity being 780 and 870~nm; the reference wavelength 670~nm and the threshold 5% , diff --git a/hypernets_processor/anomaly_handler.py b/hypernets_processor/anomaly_handler.py index 3d1bb26e..194d7a95 100755 --- a/hypernets_processor/anomaly_handler.py +++ b/hypernets_processor/anomaly_handler.py @@ -43,6 +43,7 @@ def __init__( self.anomalies_dict = anomalies_dict self.anomalies_added = [] + self.anomalies_SEQ = [] def add_anomaly(self, anomaly_id, ds=None): """ @@ -53,7 +54,7 @@ def add_anomaly(self, anomaly_id, ds=None): :type ds: xarray.Dataset :param ds: producgt that was being processed when anomaly was raised """ - + print(f"Adding anomaly: {anomaly_id}") # Check anomaly defined if anomaly_id not in self.get_anomaly_ids(): self.context.logger.info( @@ -64,6 +65,9 @@ def add_anomaly(self, anomaly_id, ds=None): # Add anomaly to list of anomalies added during current processing self.anomalies_added.append(anomaly_id) + # Add anomaly to list of anomalies for current sequence + self.anomalies_SEQ.append(anomaly_id) + # Add anomaly to db if self.anomaly_db is not None: self.anomaly_db.add_anomaly(anomaly_id, ds) @@ -87,7 +91,9 @@ def add_x_anomaly(self): if self.get_sequence_crashing_anomalies() == []: self.add_anomaly("x") - + else: + print("Not adding x anomaly as other crashing anomalies already present (%s)" % ",".join(self.get_sequence_crashing_anomalies())) + def get_anomaly_ids(self): """ Returns available anomaly ids @@ -136,10 +142,10 @@ def get_crashing_anomaly_ids(self): :return: crashing anomaly ids """ - crashing_anomalies = [] + crashing_anomalies = ["x"] for anomaly_id in self.get_anomaly_ids(): - if self.get_anomaly_error(anomaly_id) is not None: + if self.get_anomaly_error(anomaly_id) is not None and anomaly_id != "m": crashing_anomalies.append(anomaly_id) return crashing_anomalies diff --git a/hypernets_processor/calibration/calibrate.py b/hypernets_processor/calibration/calibrate.py index b8d75747..9c3204c8 100755 --- a/hypernets_processor/calibration/calibrate.py +++ b/hypernets_processor/calibration/calibrate.py @@ -1,6 +1,7 @@ """ Calibration class """ + import datetime from hypernets_processor.version import __version__ @@ -92,9 +93,9 @@ def calibrate_l1a( store_unc_percent=True, ) - if self.context.get_config_value("bad_wavelenth_ranges"): + if self.context.get_config_value("bad_wavelength_ranges"): for maskrange in self.context.get_config_value( - "bad_wavelenth_ranges" + "bad_wavelength_ranges" ).split(","): start_mask = float(maskrange.split("-")[0]) end_mask = float(maskrange.split("-")[1]) @@ -107,18 +108,10 @@ def calibrate_l1a( ] += 50 else: - measurand = calibrate_function.run(dataset_l0_masked, calibration_data) - dataset_l1a[measurandstring].values = measurand - dataset_l1a[measurandstring].attrs["unc_comps"] = [] - dataset_l1a = dataset_l1a.drop( - [ - "u_rel_random_" + measurandstring, - "u_rel_systematic_indep_" + measurandstring, - "u_rel_systematic_corr_rad_irr_" + measurandstring, - "err_corr_systematic_indep_" + measurandstring, - "err_corr_systematic_corr_rad_irr_" + measurandstring, - ] + measurand = calibrate_function.run_meas_function( + dataset_l0_masked, calibration_data ) + dataset_l1a[measurandstring].values = measurand if self.context.get_config_value("write_l1a"): self.writer.write( @@ -176,26 +169,17 @@ def calibrate_l1b( store_unc_percent=True, ) else: - measurand = calibrate_function.run(dataset_l0b, calibration_data) - dataset_l1b[measurandstring].values = measurand - dataset_l1b[measurandstring].attrs["unc_comps"] = [] - dataset_l1b = dataset_l1b.drop( - [ - "u_rel_random_" + measurandstring, - "u_rel_systematic_indep_" + measurandstring, - "u_rel_systematic_corr_rad_irr_" + measurandstring, - "err_corr_systematic_indep_" + measurandstring, - "err_corr_systematic_corr_rad_irr_" + measurandstring, - ] + measurand = calibrate_function.run_meas_function( + dataset_l0b, calibration_data ) - + dataset_l1b[measurandstring].values = measurand if self.context.get_config_value("mcsteps") > 0: self.qual.perform_quality_check_rand_unc(dataset_l1b, measurandstring) - if self.context.get_config_value("bad_wavelenth_ranges"): + if self.context.get_config_value("bad_wavelength_ranges"): for maskrange in self.context.get_config_value( - "bad_wavelenth_ranges" + "bad_wavelength_ranges" ).split(","): start_mask = float(maskrange.split("-")[0]) end_mask = float(maskrange.split("-")[1]) @@ -397,11 +381,11 @@ def preprocess_l0(self, datasetl0, datasetl0_bla, dataset_calib): datasetl0masked, mask = self.qual.perform_quality_check_L0A( datasetl0masked, series_ids ) - datasetl0masked["quality_flag"][ - np.where(dark_outlier_rad == 1) - ] = DatasetUtil.set_flag( - datasetl0masked["quality_flag"][np.where(dark_outlier_rad == 1)], - "dark_masked", + datasetl0masked["quality_flag"][np.where(dark_outliers_radscans == 1)] = ( + DatasetUtil.set_flag( + datasetl0masked["quality_flag"][np.where(dark_outliers_radscans == 1)], + "dark_masked", + ) ) # for i in range(len(mask))] # calculate and store random uncertainties on radiance/irradiance diff --git a/hypernets_processor/calibration/calibration_converter.py b/hypernets_processor/calibration/calibration_converter.py index 43ee1221..2e4f1706 100755 --- a/hypernets_processor/calibration/calibration_converter.py +++ b/hypernets_processor/calibration/calibration_converter.py @@ -1,3 +1,5 @@ +import warnings + from hypernets_processor.version import __version__ from hypernets_processor.data_io.data_templates import DataTemplates from hypernets_processor.test.test_functions import ( @@ -39,12 +41,53 @@ def __init__(self, context): self.templ = DataTemplates(context) self.writer = HypernetsWriter(context) self.context = context - self.version = self.context.get_config_value("version") - if self.version == 0.0: + self.version = self.context.get_config_value("calibration_file_version") + if self.version is None or self.version == 0.0: self.version = input( "Enter hypernets_processor version for which you are generating calib files:" ) + def convert_coordinates_dt64(self, ds): + ds = ds.assign_coords( + calibrationdates=[ + datetime.strptime(date, "%y%m%dT%H%M%S") + for date in ds["calibrationdates"].values + ] + ) + ds = ds.assign_coords( + nonlineardates=[ + datetime.strptime(date, "%y%m%dT%H%M%S") + for date in ds["nonlineardates"].values + ] + ) + ds = ds.assign_coords( + wavdates=[ + datetime.strptime(date, "%y%m%dT%H%M%S") + for date in ds["wavdates"].values + ] + ) + return ds + + def interpolate_calibration_ds(self, ds, sequence_dt64): + ds = self.convert_coordinates_dt64(ds) + for coord in ["calibrationdates", "nonlineardates", "wavdates"]: + dt64s = ds[coord].values + if self.context.get_config_value( + "calibration_interpolation_method" + ) == "previous" or sequence_dt64 > np.max(dt64s): + i = [x for x, date in enumerate(dt64s) if date < sequence_dt64][-1] + ds = ds.isel(indexers={coord: i}) + elif sequence_dt64 <= np.max(dt64s) and sequence_dt64 >= np.min(dt64s): + ds = ds.interp( + coords={coord: sequence_dt64}, + method=self.context.get_config_value( + "calibration_interpolation_method" + ), + ) + else: + raise NotImplementedError + return ds + def read_calib_files(self, sequence_path): metadata = ConfigParser() @@ -91,89 +134,13 @@ def read_calib_files(self, sequence_path): os.path.join(hypstar_path, name) + " calibration file does not exist" ) - sequence_datetime = parse_sequence_path(sequence_path)["datetime"] - - calibration_data_times = calibration_data_rad["calibrationdates"].values - nonlin_times = calibration_data_rad["nonlineardates"].values - wav_times = calibration_data_rad["wavdates"].values + sequence_dt64 = np.datetime64(parse_sequence_path(sequence_path)["datetime"]) - # print(nonlin_times) - # - # - # calib_i=[x for x, date in enumerate(calibration_data_times) - # if datetime.strptime(date,"%y%m%dT%H%M%S") < sequence_datetime][-1] - # - # if len(nonlin_times)==1: - # nlin_i=0 - # else: - # nlin_i=[x for x, date in enumerate(nonlin_times) if datetime.strptime(date,"%y%m%dT%H%M%S") < sequence_datetime][-1] - - calib_i = [ - x - for x, date in enumerate(calibration_data_times) - if datetime.strptime(date, "%y%m%dT%H%M%S") < sequence_datetime - ][-1] - if len(nonlin_times) == 1: - nlin_i = 0 - else: - nlin_i = [ - x - for x, date in enumerate(nonlin_times) - if datetime.strptime(date, "%y%m%dT%H%M%S") < sequence_datetime - ][-1] - - wav_i = [ - x - for x, date in enumerate(wav_times) - if datetime.strptime(date, "%y%m%dT%H%M%S") < sequence_datetime - ][-1] - - calibration_data_rad = calibration_data_rad.sel( - calibrationdates=calibration_data_times[calib_i] + calibration_data_rad = self.interpolate_calibration_ds( + calibration_data_rad, sequence_dt64 ) - calibration_data_rad = calibration_data_rad.sel( - nonlineardates=nonlin_times[nlin_i] - ) - calibration_data_rad = calibration_data_rad.sel(wavdates=wav_times[wav_i]) - - calibration_data_times_irr = calibration_data_irr["calibrationdates"].values - nonlin_times_irr = calibration_data_irr["nonlineardates"].values - wav_times_irr = calibration_data_irr["wavdates"].values - - calib_i_irr = [ - x - for x, date in enumerate(calibration_data_times_irr) - if datetime.strptime(date, "%y%m%dT%H%M%S") < sequence_datetime - ][-1] - - # if len(nonlin_times)==1: - # nlin_i_irr=0 - # else: - # nlin_i_irr=[x for x, date in enumerate(nonlin_times) if datetime.strptime(date,"%y%m%dT%H%M%S") < sequence_datetime][-1] - # - if len(nonlin_times) == 1: - nlin_i_irr = 0 - else: - nlin_i_irr = [ - x - for x, date in enumerate(nonlin_times_irr) - if datetime.strptime(date, "%y%m%dT%H%M%S") < sequence_datetime - ][-1] - - wav_i_irr = [ - x - for x, date in enumerate(wav_times_irr) - if datetime.strptime(date, "%y%m%dT%H%M%S") < sequence_datetime - ][-1] - - calibration_data_irr = calibration_data_irr.sel( - calibrationdates=calibration_data_times_irr[calib_i_irr] - ) - calibration_data_irr = calibration_data_irr.sel( - nonlineardates=nonlin_times_irr[nlin_i_irr] - ) - calibration_data_irr = calibration_data_irr.sel( - wavdates=wav_times_irr[wav_i_irr] + calibration_data_irr = self.interpolate_calibration_ds( + calibration_data_irr, sequence_dt64 ) if self.context.get_config_value("network") == "l": @@ -211,29 +178,140 @@ def read_calib_files(self, sequence_path): + " calibration file does not exist" ) - calibration_data_times = calibration_data_rad_swir[ - "calibrationdates" - ].values - nonlin_times = calibration_data_rad_swir["nonlineardates"].values - calibration_data_rad_swir = calibration_data_rad_swir.sel( - calibrationdates=calibration_data_times[calib_i] - ) - calibration_data_rad_swir = calibration_data_rad_swir.sel( - nonlineardates=nonlin_times[nlin_i] - ) - calibration_data_rad_swir = calibration_data_rad_swir.sel( - wavdates=wav_times[wav_i] + calibration_data_rad_swir = self.interpolate_calibration_ds( + calibration_data_rad_swir, sequence_dt64 ) - calibration_data_irr_swir = calibration_data_irr_swir.sel( - calibrationdates=calibration_data_times_irr[calib_i_irr] - ) - calibration_data_irr_swir = calibration_data_irr_swir.sel( - nonlineardates=nonlin_times_irr[nlin_i_irr] - ) - calibration_data_irr_swir = calibration_data_irr_swir.sel( - wavdates=wav_times_irr[wav_i_irr] + calibration_data_irr_swir = self.interpolate_calibration_ds( + calibration_data_irr_swir, sequence_dt64 ) + # + # if self.context.get_config_value("calibration_interpolation_method")=="previous" or sequence_datetime>np.max(calibration_data_times): + # calib_i = [ + # x + # for x, date in enumerate(calibration_data_times) + # if date < sequence_datetime + # ][-1] + # if len(nonlin_times) == 1: + # nlin_i = 0 + # else: + # nlin_i = [ + # x + # for x, date in enumerate(nonlin_times) + # if date < sequence_datetime + # ][-1] + # + # wav_i = [ + # x + # for x, date in enumerate(wav_times) + # if date < sequence_datetime + # ][-1] + # + # calibration_data_rad = calibration_data_rad.sel( + # calibrationdates=calibration_data_times[calib_i] + # ) + # calibration_data_rad = calibration_data_rad.sel( + # nonlineardates=nonlin_times[nlin_i] + # ) + # calibration_data_rad = calibration_data_rad.sel(wavdates=wav_times[wav_i]) + # + # + # + # calib_i_irr = [ + # x + # for x, date in enumerate(calibration_data_times_irr) + # if date < sequence_datetime + # ][-1] + # + # if len(nonlin_times) == 1: + # nlin_i_irr = 0 + # else: + # nlin_i_irr = [ + # x + # for x, date in enumerate(nonlin_times_irr) + # if date < sequence_datetime + # ][-1] + # + # wav_i_irr = [ + # x + # for x, date in enumerate(wav_times_irr) + # if date < sequence_datetime + # ][-1] + # + # calibration_data_irr = calibration_data_irr.sel( + # calibrationdates=calibration_data_times_irr[calib_i_irr] + # ) + # calibration_data_irr = calibration_data_irr.sel( + # nonlineardates=nonlin_times_irr[nlin_i_irr] + # ) + # calibration_data_irr = calibration_data_irr.sel( + # wavdates=wav_times_irr[wav_i_irr] + # ) + # + # if self.context.get_config_value("network") == "l": + # + # calibration_data_rad_swir = calibration_data_rad_swir.sel( + # calibrationdates=calibration_data_times_swir[calib_i] + # ) + # calibration_data_rad_swir = calibration_data_rad_swir.sel( + # nonlineardates=nonlin_times_swir[nlin_i] + # ) + # calibration_data_rad_swir = calibration_data_rad_swir.sel( + # wavdates=wav_times_swir[wav_i] + # ) + # calibration_data_irr_swir = calibration_data_irr_swir.sel( + # calibrationdates=calibration_data_times_irr_swir[calib_i_irr] + # ) + # calibration_data_irr_swir = calibration_data_irr_swir.sel( + # nonlineardates=nonlin_times_irr_swir[nlin_i_irr] + # ) + # calibration_data_irr_swir = calibration_data_irr_swir.sel( + # wavdates=wav_times_irr_swir[wav_i_irr] + # ) + # + # else: + # calibration_data_rad = calibration_data_rad.interp(calibrationdates=[sequence_datetime], + # method=self.context.get_config_value( + # "calibration_interpolation_method")) + # calibration_data_rad = calibration_data_rad.interp(nonlineardates=[sequence_datetime], + # method=self.context.get_config_value( + # "calibration_interpolation_method")) + # calibration_data_rad = calibration_data_rad.interp(wavdates=[sequence_datetime], + # method=self.context.get_config_value( + # "calibration_interpolation_method")) + # + # calibration_data_irr = calibration_data_irr.interp(calibrationdates=[sequence_datetime], + # method=self.context.get_config_value( + # "calibration_interpolation_method")) + # calibration_data_irr = calibration_data_irr.interp(nonlineardates=[sequence_datetime], + # method=self.context.get_config_value( + # "calibration_interpolation_method")) + # calibration_data_irr = calibration_data_irr.interp(wavdates=[sequence_datetime], + # method=self.context.get_config_value( + # "calibration_interpolation_method")) + # if self.context.get_config_value("network") == "l": + # + # calibration_data_rad_swir = calibration_data_rad_swir.interp(calibrationdates=[sequence_datetime], + # method=self.context.get_config_value( + # "calibration_interpolation_method")) + # calibration_data_rad_swir = calibration_data_rad_swir.interp(nonlineardates=[sequence_datetime], + # method=self.context.get_config_value( + # "calibration_interpolation_method")) + # calibration_data_rad_swir = calibration_data_rad_swir.interp(wavdates=[sequence_datetime], + # method=self.context.get_config_value( + # "calibration_interpolation_method")) + # + # calibration_data_irr_swir = calibration_data_irr_swir.interp(calibrationdates=[sequence_datetime], + # method=self.context.get_config_value( + # "calibration_interpolation_method")) + # calibration_data_irr_swir = calibration_data_irr_swir.interp(nonlineardates=[sequence_datetime], + # method=self.context.get_config_value( + # "calibration_interpolation_method")) + # calibration_data_irr_swir = calibration_data_irr_swir.interp(wavdates=[sequence_datetime], + # method=self.context.get_config_value( + # "calibration_interpolation_method")) + + if self.context.get_config_value("network") == "l": return ( calibration_data_rad, calibration_data_irr, @@ -312,40 +390,69 @@ def prepare_calibration_data(self, measurandstring, hypstar, swir=False): minwav = 350 for caldatepath in caldatepaths: if measurandstring == "radiance": - calpath = glob.glob( + filelist = glob.glob( os.path.join( caldatepath, "hypstar_" + str(hypstar) + "_radcal_L_*_%s.dat" % (sensortag), ) - )[0] - caldate = calpath[-15:-9] - print(os.path.basename(caldatepath)) - if ("b" in os.path.basename(caldatepath)) or ( - "10C" in os.path.basename(caldatepath) - ): - caldate += "T235959" - elif "a" in os.path.basename(caldatepath): - caldate += "T000000" + ) + if len(filelist) == 1: + calpath = filelist[0] + caldate = calpath[-15:-9] + print(os.path.basename(caldatepath)) + if ("b" in os.path.basename(caldatepath)) or ( + "10C" in os.path.basename(caldatepath) + ): + caldate += "T235959" + elif "a" in os.path.basename(caldatepath): + caldate += "T000000" + else: + caldate += "T120000" + print(caldate) else: - caldate += "T120000" - print(caldate) + warnings.warn( + "the number of files matching %s is not equal to 1: %s" + % ( + os.path.join( + caldatepath, + "hypstar_" + + str(hypstar) + + "_radcal_L_*_%s.dat" % (sensortag), + ), + filelist, + ) + ) else: - calpath = glob.glob( + filelist = glob.glob( os.path.join( caldatepath, "hypstar_" + str(hypstar) + "_radcal_E_*_%s.dat" % (sensortag), ) - )[0] - caldate = calpath[-15:-9] - if ("b" in os.path.basename(caldatepath)) or ( - "10C" in os.path.basename(caldatepath) - ): - caldate += "T235959" - elif "a" in os.path.basename(caldatepath): - caldate += "T000000" + ) + if len(filelist) == 1: + calpath = filelist[0] + caldate = calpath[-15:-9] + if ("b" in os.path.basename(caldatepath)) or ( + "10C" in os.path.basename(caldatepath) + ): + caldate += "T235959" + elif "a" in os.path.basename(caldatepath): + caldate += "T000000" + else: + caldate += "T120000" else: - caldate += "T120000" - + warnings.warn( + "the number of files matching %s is not equal to 1: %s" + % ( + os.path.join( + caldatepath, + "hypstar_" + + str(hypstar) + + "_radcal_E_*_%s.dat" % (sensortag), + ), + filelist, + ) + ) if os.path.exists(calpath): caldates = np.append(caldates, caldate) gains_temp = np.genfromtxt(calpath) @@ -360,44 +467,59 @@ def prepare_calibration_data(self, measurandstring, hypstar, swir=False): for path in glob.glob( os.path.join(directory, "hypstar_" + str(hypstar), "linearity", "*") ) + if ".pdf" not in path ] - if len(lincaldatepaths) == 0: - lincaldatepaths = [ - path - for path in glob.glob( - os.path.join( - directory, "hypstar_" + str(hypstar), "radiometric", "*" - ) + lincaldatepaths += [ + path + for path in glob.glob( + os.path.join( + directory, "hypstar_" + str(hypstar), "radiometric", "*" ) - if len( - glob.glob( - os.path.join( - path, - "hypstar_" + str(hypstar) + "_nonlin_corr_coefs_*.dat", - ) + ) + if len( + glob.glob( + os.path.join( + path, + "hypstar_" + str(hypstar) + "_nonlin_corr_coefs_*.dat", ) ) - > 0 - ] + ) + > 0 + ] nonlindates = [] for lincaldatepath in lincaldatepaths: - nonlinpath = glob.glob( + filelist = glob.glob( os.path.join( lincaldatepath, "hypstar_" + str(hypstar) + "_nonlin_corr_coefs_*.dat", ) - )[0] - lincaldate = nonlinpath[-10:-4] - if ("b" in os.path.basename(lincaldatepath)) or ( - "10C" in os.path.basename(lincaldatepath) - ): - lincaldate += "T235959" - elif "a" in os.path.basename(lincaldatepath): - lincaldate += "T000000" + ) + if len(filelist) == 1: + nonlinpath = filelist[0] + lincaldate = nonlinpath[-10:-4] + if ("b" in os.path.basename(lincaldatepath)) or ( + "10C" in os.path.basename(lincaldatepath) + ): + lincaldate += "T235959" + elif "a" in os.path.basename(lincaldatepath): + lincaldate += "T000000" + else: + lincaldate += "T120000" else: - lincaldate += "T120000" + warnings.warn( + "the number of files matching %s is not equal to 1: %s" + % ( + os.path.join( + caldatepath, + "hypstar_" + + str(hypstar) + + "_radcal_E_*_%s.dat" % (sensortag), + ), + filelist, + ) + ) if os.path.exists(nonlinpath): nonlindates = np.append(nonlindates, lincaldate) @@ -442,12 +564,15 @@ def prepare_calibration_data(self, measurandstring, hypstar, swir=False): i_nonlin = 0 # todo remove [-1] and use all calibrations and interpolate? for lincaldatepath in lincaldatepaths: - nonlinpath = glob.glob( - os.path.join( - lincaldatepath, - "hypstar_" + str(hypstar) + "_nonlin_corr_coefs_*.dat", - ) - )[0] + try: + nonlinpath = glob.glob( + os.path.join( + lincaldatepath, + "hypstar_" + str(hypstar) + "_nonlin_corr_coefs_*.dat", + ) + )[0] + except: + stop if os.path.exists(nonlinpath): if swir: non_linear_cals = np.genfromtxt(nonlinpath)[:, 1] @@ -466,13 +591,13 @@ def prepare_calibration_data(self, measurandstring, hypstar, swir=False): float(line.strip()[8::]) / 2 ) # reading in from comments in non_lin files, and convert to k=1 - calibration_data["non_linearity_coefficients"].values[ - i_nonlin - ] = np.pad( - non_linear_cals, - (0, 13 - len(non_linear_cals)), - "constant", - constant_values=(0, 0), + calibration_data["non_linearity_coefficients"].values[i_nonlin] = ( + np.pad( + non_linear_cals, + (0, 13 - len(non_linear_cals)), + "constant", + constant_values=(0, 0), + ) ) i_nonlin += 1 @@ -502,130 +627,137 @@ def prepare_calibration_data(self, measurandstring, hypstar, swir=False): for caldatepath in caldatepaths: # print(caldatepath,) if measurandstring == "radiance": - calpath = glob.glob( + filelist = glob.glob( os.path.join( caldatepath, "hypstar_" + str(hypstar) + "_radcal_L_*_%s.dat" % (sensortag), ) - )[0] + ) + else: - calpath = glob.glob( + filelist = glob.glob( os.path.join( caldatepath, "hypstar_" + str(hypstar) + "_radcal_E_*_%s.dat" % (sensortag), ) - )[0] + ) - if os.path.exists(calpath): - caldates = np.append(caldates, caldate) - gains = np.genfromtxt(calpath) - wavs = gains[:, 1] - gains = gains[np.where(wavs > 350)[0]] - wavs = wavs[np.where(wavs > 350)[0]] - placeholder_unc = 2 - gainlen = len(calibration_data["wavpix"].values[i_cal, :]) - if True: - # calibration_data["wavelength"].values = gains[:, 1] - calibration_data["wavpix"].values[i_cal, :] = gains[:gainlen, 0] - calibration_data["gains"].values[i_cal, :] = gains[:gainlen, 2] - # calibration_data["u_rel_random_gains"].values = None - - calibration_data["u_rel_systematic_indep_gains"].values[ - i_cal, : - ] = ( - ( - gains[:, 6] ** 2 - + gains[:, 7] ** 2 - + gains[:, 8] ** 2 - + gains[:, 9] ** 2 - + gains[:, 10] ** 2 - + gains[:, 11] ** 2 - + gains[:, 12] ** 2 - + gains[:, 13] ** 2 - + gains[:, 14] ** 2 - + gains[:, 15] ** 2 - + gains[:, 16] ** 2 - + gains[:, 17] ** 2 - + gains[:, 19] ** 2 - + nonlin_unc**2 - + placeholder_unc**2 - ) - ** 0.5 - )[ - :gainlen - ] + if len(filelist) == 1: + calpath = filelist[0] - cov_diag = cm.convert_corr_to_cov( - np.eye(len(gains[:, 2])), gains[:, 2] * (gains[:, 19]) - ) + if os.path.exists(calpath): + caldates = np.append(caldates, caldate) + gains = np.genfromtxt(calpath) + wavs = gains[:, 1] + gains = gains[np.where(wavs > 350)[0]] + wavs = wavs[np.where(wavs > 350)[0]] + placeholder_unc = 2 + gainlen = len(calibration_data["wavpix"].values[i_cal, :]) + if True: + # calibration_data["wavelength"].values = gains[:, 1] + calibration_data["wavpix"].values[i_cal, :] = gains[:gainlen, 0] + calibration_data["gains"].values[i_cal, :] = gains[:gainlen, 2] + # calibration_data["u_rel_random_gains"].values = None + + calibration_data["u_rel_systematic_indep_gains"].values[ + i_cal, : + ] = ( + ( + gains[:, 6] ** 2 + + gains[:, 7] ** 2 + + gains[:, 8] ** 2 + + gains[:, 9] ** 2 + + gains[:, 10] ** 2 + + gains[:, 11] ** 2 + + gains[:, 12] ** 2 + + gains[:, 13] ** 2 + + gains[:, 14] ** 2 + + gains[:, 15] ** 2 + + gains[:, 16] ** 2 + + gains[:, 17] ** 2 + + gains[:, 19] ** 2 + + nonlin_unc**2 + + placeholder_unc**2 + ) + ** 0.5 + )[ + :gainlen + ] + + cov_diag = cm.convert_corr_to_cov( + np.eye(len(gains[:, 2])), gains[:, 2] * (gains[:, 19]) + ) - cov_other = cm.convert_corr_to_cov( - np.eye(len(gains[:, 2])), - gains[:, 2] - * ( - gains[:, 8] ** 2 - + gains[:, 10] ** 2 - + gains[:, 11] ** 2 - + gains[:, 16] ** 2 - + gains[:, 17] ** 2 - + nonlin_unc**2 + cov_other = cm.convert_corr_to_cov( + np.eye(len(gains[:, 2])), + gains[:, 2] + * ( + gains[:, 8] ** 2 + + gains[:, 10] ** 2 + + gains[:, 11] ** 2 + + gains[:, 16] ** 2 + + gains[:, 17] ** 2 + + nonlin_unc**2 + ) + ** 0.5, ) - ** 0.5, - ) - cov_full = cm.convert_corr_to_cov( - np.ones((len(gains[:, 2]), len(gains[:, 2]))), - gains[:, 2] - * ( - gains[:, 7] ** 2 - + gains[:, 9] ** 2 - + gains[:, 12] ** 2 - + gains[:, 13] ** 2 - + gains[:, 14] ** 2 - + gains[:, 15] ** 2 - + placeholder_unc**2 + cov_full = cm.convert_corr_to_cov( + np.ones((len(gains[:, 2]), len(gains[:, 2]))), + gains[:, 2] + * ( + gains[:, 7] ** 2 + + gains[:, 9] ** 2 + + gains[:, 12] ** 2 + + gains[:, 13] ** 2 + + gains[:, 14] ** 2 + + gains[:, 15] ** 2 + + placeholder_unc**2 + ) + ** 0.5, ) - ** 0.5, - ) - cov_filament = cm.convert_corr_to_cov( - np.ones((len(gains[:, 2]), len(gains[:, 2]))), - gains[:, 2] * (gains[:, 6] ** 2) ** 0.5, - ) + cov_filament = cm.convert_corr_to_cov( + np.ones((len(gains[:, 2]), len(gains[:, 2]))), + gains[:, 2] * (gains[:, 6] ** 2) ** 0.5, + ) - calibration_data["err_corr_systematic_indep_gains"].values[ - i_cal, :, : - ] = cm.correlation_from_covariance( - cov_diag + cov_other + cov_full + cov_filament - )[ - :gainlen, :gainlen - ] - - calibration_data["u_rel_systematic_corr_rad_irr_gains"].values[ - i_cal, : - ] = ( - (gains[:, 4] ** 2 + gains[:, 5] ** 2 + gains[:, 18] ** 2) ** 0.5 - )[ - :gainlen - ] - - cov_other = cm.convert_corr_to_cov( - np.eye(len(gains[:, 2])), - gains[:, 2] * (gains[:, 4] ** 2 + gains[:, 18] ** 2) ** 0.5, - ) + calibration_data["err_corr_systematic_indep_gains"].values[ + i_cal, :, : + ] = cm.correlation_from_covariance( + cov_diag + cov_other + cov_full + cov_filament + )[ + :gainlen, :gainlen + ] + + calibration_data["u_rel_systematic_corr_rad_irr_gains"].values[ + i_cal, : + ] = ( + (gains[:, 4] ** 2 + gains[:, 5] ** 2 + gains[:, 18] ** 2) + ** 0.5 + )[ + :gainlen + ] + + cov_other = cm.convert_corr_to_cov( + np.eye(len(gains[:, 2])), + gains[:, 2] * (gains[:, 4] ** 2 + gains[:, 18] ** 2) ** 0.5, + ) - cov_filament = cm.convert_corr_to_cov( - np.ones((len(gains[:, 2]), len(gains[:, 2]))), - gains[:, 2] * (gains[:, 5] ** 2) ** 0.5, - ) + cov_filament = cm.convert_corr_to_cov( + np.ones((len(gains[:, 2]), len(gains[:, 2]))), + gains[:, 2] * (gains[:, 5] ** 2) ** 0.5, + ) - calibration_data["err_corr_systematic_corr_rad_irr_gains"].values[ - i_cal, :, : - ] = cm.correlation_from_covariance(cov_other + cov_filament)[ - :gainlen, :gainlen - ] - # except: - # print(caldatepath, " failed") + calibration_data[ + "err_corr_systematic_corr_rad_irr_gains" + ].values[i_cal, :, :] = cm.correlation_from_covariance( + cov_other + cov_filament + )[ + :gainlen, :gainlen + ] + # except: + # print(caldatepath, " failed") i_cal += 1 return calibration_data diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120241/HYPERNETS_CAL_HYPSTAR_120241_IRR_v2.0.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120241/HYPERNETS_CAL_HYPSTAR_120241_IRR_v2.1.nc similarity index 97% rename from hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120241/HYPERNETS_CAL_HYPSTAR_120241_IRR_v2.0.nc rename to hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120241/HYPERNETS_CAL_HYPSTAR_120241_IRR_v2.1.nc index 48fe7422..7cbfa385 100644 Binary files a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120241/HYPERNETS_CAL_HYPSTAR_120241_IRR_v2.0.nc and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120241/HYPERNETS_CAL_HYPSTAR_120241_IRR_v2.1.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120241/HYPERNETS_CAL_HYPSTAR_120241_IRR_v2.2.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120241/HYPERNETS_CAL_HYPSTAR_120241_IRR_v2.2.nc new file mode 100644 index 00000000..4811b967 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120241/HYPERNETS_CAL_HYPSTAR_120241_IRR_v2.2.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120241/HYPERNETS_CAL_HYPSTAR_120241_IRR_v2.3.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120241/HYPERNETS_CAL_HYPSTAR_120241_IRR_v2.3.nc new file mode 100644 index 00000000..b22eb926 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120241/HYPERNETS_CAL_HYPSTAR_120241_IRR_v2.3.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120241/HYPERNETS_CAL_HYPSTAR_120241_IRR_v2.4.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120241/HYPERNETS_CAL_HYPSTAR_120241_IRR_v2.4.nc new file mode 100644 index 00000000..2930460e Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120241/HYPERNETS_CAL_HYPSTAR_120241_IRR_v2.4.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120241/HYPERNETS_CAL_HYPSTAR_120241_RAD_v2.0.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120241/HYPERNETS_CAL_HYPSTAR_120241_RAD_v2.1.nc similarity index 95% rename from hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120241/HYPERNETS_CAL_HYPSTAR_120241_RAD_v2.0.nc rename to hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120241/HYPERNETS_CAL_HYPSTAR_120241_RAD_v2.1.nc index 94509e24..1de3810b 100644 Binary files a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120241/HYPERNETS_CAL_HYPSTAR_120241_RAD_v2.0.nc and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120241/HYPERNETS_CAL_HYPSTAR_120241_RAD_v2.1.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120241/HYPERNETS_CAL_HYPSTAR_120241_RAD_v2.2.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120241/HYPERNETS_CAL_HYPSTAR_120241_RAD_v2.2.nc new file mode 100644 index 00000000..4fadee8c Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120241/HYPERNETS_CAL_HYPSTAR_120241_RAD_v2.2.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120241/HYPERNETS_CAL_HYPSTAR_120241_RAD_v2.3.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120241/HYPERNETS_CAL_HYPSTAR_120241_RAD_v2.3.nc new file mode 100644 index 00000000..0f47cd8f Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120241/HYPERNETS_CAL_HYPSTAR_120241_RAD_v2.3.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120241/HYPERNETS_CAL_HYPSTAR_120241_RAD_v2.4.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120241/HYPERNETS_CAL_HYPSTAR_120241_RAD_v2.4.nc new file mode 100644 index 00000000..d248d9c0 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120241/HYPERNETS_CAL_HYPSTAR_120241_RAD_v2.4.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120242/HYPERNETS_CAL_HYPSTAR_120242_IRR_v2.0.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120242/HYPERNETS_CAL_HYPSTAR_120242_IRR_v2.0.nc deleted file mode 100644 index 9c9395c9..00000000 Binary files a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120242/HYPERNETS_CAL_HYPSTAR_120242_IRR_v2.0.nc and /dev/null differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120242/HYPERNETS_CAL_HYPSTAR_120242_IRR_v2.1.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120242/HYPERNETS_CAL_HYPSTAR_120242_IRR_v2.1.nc new file mode 100644 index 00000000..f5bb5c0a Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120242/HYPERNETS_CAL_HYPSTAR_120242_IRR_v2.1.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120242/HYPERNETS_CAL_HYPSTAR_120242_IRR_v2.2.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120242/HYPERNETS_CAL_HYPSTAR_120242_IRR_v2.2.nc new file mode 100644 index 00000000..00001623 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120242/HYPERNETS_CAL_HYPSTAR_120242_IRR_v2.2.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120242/HYPERNETS_CAL_HYPSTAR_120242_IRR_v2.3.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120242/HYPERNETS_CAL_HYPSTAR_120242_IRR_v2.3.nc new file mode 100644 index 00000000..22687881 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120242/HYPERNETS_CAL_HYPSTAR_120242_IRR_v2.3.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120242/HYPERNETS_CAL_HYPSTAR_120242_IRR_v2.4.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120242/HYPERNETS_CAL_HYPSTAR_120242_IRR_v2.4.nc new file mode 100644 index 00000000..9c02bbcd Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120242/HYPERNETS_CAL_HYPSTAR_120242_IRR_v2.4.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120242/HYPERNETS_CAL_HYPSTAR_120242_RAD_v2.0.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120242/HYPERNETS_CAL_HYPSTAR_120242_RAD_v2.0.nc deleted file mode 100644 index f6e46ee0..00000000 Binary files a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120242/HYPERNETS_CAL_HYPSTAR_120242_RAD_v2.0.nc and /dev/null differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120242/HYPERNETS_CAL_HYPSTAR_120242_RAD_v2.1.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120242/HYPERNETS_CAL_HYPSTAR_120242_RAD_v2.1.nc new file mode 100644 index 00000000..f5fc0e67 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120242/HYPERNETS_CAL_HYPSTAR_120242_RAD_v2.1.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120242/HYPERNETS_CAL_HYPSTAR_120242_RAD_v2.2.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120242/HYPERNETS_CAL_HYPSTAR_120242_RAD_v2.2.nc new file mode 100644 index 00000000..c233fb30 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120242/HYPERNETS_CAL_HYPSTAR_120242_RAD_v2.2.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120242/HYPERNETS_CAL_HYPSTAR_120242_RAD_v2.3.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120242/HYPERNETS_CAL_HYPSTAR_120242_RAD_v2.3.nc new file mode 100644 index 00000000..cb408784 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120242/HYPERNETS_CAL_HYPSTAR_120242_RAD_v2.3.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120242/HYPERNETS_CAL_HYPSTAR_120242_RAD_v2.4.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120242/HYPERNETS_CAL_HYPSTAR_120242_RAD_v2.4.nc new file mode 100644 index 00000000..16b88654 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120242/HYPERNETS_CAL_HYPSTAR_120242_RAD_v2.4.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120243/HYPERNETS_CAL_HYPSTAR_120243_IRR_v2.0.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120243/HYPERNETS_CAL_HYPSTAR_120243_IRR_v2.1.nc similarity index 98% rename from hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120243/HYPERNETS_CAL_HYPSTAR_120243_IRR_v2.0.nc rename to hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120243/HYPERNETS_CAL_HYPSTAR_120243_IRR_v2.1.nc index 7940c971..fccc16ea 100644 Binary files a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120243/HYPERNETS_CAL_HYPSTAR_120243_IRR_v2.0.nc and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120243/HYPERNETS_CAL_HYPSTAR_120243_IRR_v2.1.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120243/HYPERNETS_CAL_HYPSTAR_120243_IRR_v2.2.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120243/HYPERNETS_CAL_HYPSTAR_120243_IRR_v2.2.nc new file mode 100644 index 00000000..71d0c458 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120243/HYPERNETS_CAL_HYPSTAR_120243_IRR_v2.2.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120243/HYPERNETS_CAL_HYPSTAR_120243_IRR_v2.3.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120243/HYPERNETS_CAL_HYPSTAR_120243_IRR_v2.3.nc new file mode 100644 index 00000000..388a6dcb Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120243/HYPERNETS_CAL_HYPSTAR_120243_IRR_v2.3.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120243/HYPERNETS_CAL_HYPSTAR_120243_IRR_v2.4.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120243/HYPERNETS_CAL_HYPSTAR_120243_IRR_v2.4.nc new file mode 100644 index 00000000..6698478f Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120243/HYPERNETS_CAL_HYPSTAR_120243_IRR_v2.4.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120243/HYPERNETS_CAL_HYPSTAR_120243_RAD_v2.0.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120243/HYPERNETS_CAL_HYPSTAR_120243_RAD_v2.1.nc similarity index 96% rename from hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120243/HYPERNETS_CAL_HYPSTAR_120243_RAD_v2.0.nc rename to hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120243/HYPERNETS_CAL_HYPSTAR_120243_RAD_v2.1.nc index 2e7e80c9..f2ace717 100644 Binary files a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120243/HYPERNETS_CAL_HYPSTAR_120243_RAD_v2.0.nc and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120243/HYPERNETS_CAL_HYPSTAR_120243_RAD_v2.1.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120243/HYPERNETS_CAL_HYPSTAR_120243_RAD_v2.2.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120243/HYPERNETS_CAL_HYPSTAR_120243_RAD_v2.2.nc new file mode 100644 index 00000000..b1cc5813 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120243/HYPERNETS_CAL_HYPSTAR_120243_RAD_v2.2.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120243/HYPERNETS_CAL_HYPSTAR_120243_RAD_v2.3.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120243/HYPERNETS_CAL_HYPSTAR_120243_RAD_v2.3.nc new file mode 100644 index 00000000..6256a756 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120243/HYPERNETS_CAL_HYPSTAR_120243_RAD_v2.3.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120243/HYPERNETS_CAL_HYPSTAR_120243_RAD_v2.4.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120243/HYPERNETS_CAL_HYPSTAR_120243_RAD_v2.4.nc new file mode 100644 index 00000000..7b884fa1 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120243/HYPERNETS_CAL_HYPSTAR_120243_RAD_v2.4.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120251/HYPERNETS_CAL_HYPSTAR_120251_IRR_v2.0.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120251/HYPERNETS_CAL_HYPSTAR_120251_IRR_v2.1.nc similarity index 99% rename from hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120251/HYPERNETS_CAL_HYPSTAR_120251_IRR_v2.0.nc rename to hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120251/HYPERNETS_CAL_HYPSTAR_120251_IRR_v2.1.nc index 0882fb7d..681e1d08 100644 Binary files a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120251/HYPERNETS_CAL_HYPSTAR_120251_IRR_v2.0.nc and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120251/HYPERNETS_CAL_HYPSTAR_120251_IRR_v2.1.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120251/HYPERNETS_CAL_HYPSTAR_120251_IRR_v2.2.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120251/HYPERNETS_CAL_HYPSTAR_120251_IRR_v2.2.nc new file mode 100644 index 00000000..fbc60665 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120251/HYPERNETS_CAL_HYPSTAR_120251_IRR_v2.2.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120251/HYPERNETS_CAL_HYPSTAR_120251_IRR_v2.3.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120251/HYPERNETS_CAL_HYPSTAR_120251_IRR_v2.3.nc new file mode 100644 index 00000000..7e33f232 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120251/HYPERNETS_CAL_HYPSTAR_120251_IRR_v2.3.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120251/HYPERNETS_CAL_HYPSTAR_120251_IRR_v2.4.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120251/HYPERNETS_CAL_HYPSTAR_120251_IRR_v2.4.nc new file mode 100644 index 00000000..299b3606 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120251/HYPERNETS_CAL_HYPSTAR_120251_IRR_v2.4.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120251/HYPERNETS_CAL_HYPSTAR_120251_RAD_v2.0.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120251/HYPERNETS_CAL_HYPSTAR_120251_RAD_v2.1.nc similarity index 98% rename from hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120251/HYPERNETS_CAL_HYPSTAR_120251_RAD_v2.0.nc rename to hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120251/HYPERNETS_CAL_HYPSTAR_120251_RAD_v2.1.nc index 0d7bc956..6c67f9e4 100644 Binary files a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120251/HYPERNETS_CAL_HYPSTAR_120251_RAD_v2.0.nc and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120251/HYPERNETS_CAL_HYPSTAR_120251_RAD_v2.1.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120251/HYPERNETS_CAL_HYPSTAR_120251_RAD_v2.2.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120251/HYPERNETS_CAL_HYPSTAR_120251_RAD_v2.2.nc new file mode 100644 index 00000000..b790ca8c Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120251/HYPERNETS_CAL_HYPSTAR_120251_RAD_v2.2.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120251/HYPERNETS_CAL_HYPSTAR_120251_RAD_v2.3.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120251/HYPERNETS_CAL_HYPSTAR_120251_RAD_v2.3.nc new file mode 100644 index 00000000..dff0e8d3 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120251/HYPERNETS_CAL_HYPSTAR_120251_RAD_v2.3.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120251/HYPERNETS_CAL_HYPSTAR_120251_RAD_v2.4.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120251/HYPERNETS_CAL_HYPSTAR_120251_RAD_v2.4.nc new file mode 100644 index 00000000..580cb1c7 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_120251/HYPERNETS_CAL_HYPSTAR_120251_RAD_v2.4.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_121221/HYPERNETS_CAL_HYPSTAR_121221_IRR_v2.0.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_121221/HYPERNETS_CAL_HYPSTAR_121221_IRR_v2.1.nc similarity index 95% rename from hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_121221/HYPERNETS_CAL_HYPSTAR_121221_IRR_v2.0.nc rename to hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_121221/HYPERNETS_CAL_HYPSTAR_121221_IRR_v2.1.nc index a7fb5279..71de3a46 100644 Binary files a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_121221/HYPERNETS_CAL_HYPSTAR_121221_IRR_v2.0.nc and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_121221/HYPERNETS_CAL_HYPSTAR_121221_IRR_v2.1.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_121221/HYPERNETS_CAL_HYPSTAR_121221_IRR_v2.2.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_121221/HYPERNETS_CAL_HYPSTAR_121221_IRR_v2.2.nc new file mode 100644 index 00000000..c7addd34 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_121221/HYPERNETS_CAL_HYPSTAR_121221_IRR_v2.2.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_121221/HYPERNETS_CAL_HYPSTAR_121221_IRR_v2.3.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_121221/HYPERNETS_CAL_HYPSTAR_121221_IRR_v2.3.nc new file mode 100644 index 00000000..0754e5ec Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_121221/HYPERNETS_CAL_HYPSTAR_121221_IRR_v2.3.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_121221/HYPERNETS_CAL_HYPSTAR_121221_IRR_v2.4.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_121221/HYPERNETS_CAL_HYPSTAR_121221_IRR_v2.4.nc new file mode 100644 index 00000000..2452d82b Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_121221/HYPERNETS_CAL_HYPSTAR_121221_IRR_v2.4.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_121221/HYPERNETS_CAL_HYPSTAR_121221_RAD_v2.0.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_121221/HYPERNETS_CAL_HYPSTAR_121221_RAD_v2.1.nc similarity index 93% rename from hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_121221/HYPERNETS_CAL_HYPSTAR_121221_RAD_v2.0.nc rename to hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_121221/HYPERNETS_CAL_HYPSTAR_121221_RAD_v2.1.nc index 8a3ca3ab..60075132 100644 Binary files a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_121221/HYPERNETS_CAL_HYPSTAR_121221_RAD_v2.0.nc and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_121221/HYPERNETS_CAL_HYPSTAR_121221_RAD_v2.1.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_121221/HYPERNETS_CAL_HYPSTAR_121221_RAD_v2.2.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_121221/HYPERNETS_CAL_HYPSTAR_121221_RAD_v2.2.nc new file mode 100644 index 00000000..5fa61165 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_121221/HYPERNETS_CAL_HYPSTAR_121221_RAD_v2.2.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_121221/HYPERNETS_CAL_HYPSTAR_121221_RAD_v2.3.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_121221/HYPERNETS_CAL_HYPSTAR_121221_RAD_v2.3.nc new file mode 100644 index 00000000..5c7802e0 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_121221/HYPERNETS_CAL_HYPSTAR_121221_RAD_v2.3.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_121221/HYPERNETS_CAL_HYPSTAR_121221_RAD_v2.4.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_121221/HYPERNETS_CAL_HYPSTAR_121221_RAD_v2.4.nc new file mode 100644 index 00000000..ca231596 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_121221/HYPERNETS_CAL_HYPSTAR_121221_RAD_v2.4.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_121222/HYPERNETS_CAL_HYPSTAR_121222_IRR_v2.0.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_121222/HYPERNETS_CAL_HYPSTAR_121222_IRR_v2.1.nc similarity index 96% rename from hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_121222/HYPERNETS_CAL_HYPSTAR_121222_IRR_v2.0.nc rename to hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_121222/HYPERNETS_CAL_HYPSTAR_121222_IRR_v2.1.nc index 037567ce..cecb8762 100644 Binary files a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_121222/HYPERNETS_CAL_HYPSTAR_121222_IRR_v2.0.nc and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_121222/HYPERNETS_CAL_HYPSTAR_121222_IRR_v2.1.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_121222/HYPERNETS_CAL_HYPSTAR_121222_IRR_v2.2.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_121222/HYPERNETS_CAL_HYPSTAR_121222_IRR_v2.2.nc new file mode 100644 index 00000000..d737f059 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_121222/HYPERNETS_CAL_HYPSTAR_121222_IRR_v2.2.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_121222/HYPERNETS_CAL_HYPSTAR_121222_IRR_v2.3.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_121222/HYPERNETS_CAL_HYPSTAR_121222_IRR_v2.3.nc new file mode 100644 index 00000000..b68c89d4 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_121222/HYPERNETS_CAL_HYPSTAR_121222_IRR_v2.3.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_121222/HYPERNETS_CAL_HYPSTAR_121222_IRR_v2.4.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_121222/HYPERNETS_CAL_HYPSTAR_121222_IRR_v2.4.nc new file mode 100644 index 00000000..54590640 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_121222/HYPERNETS_CAL_HYPSTAR_121222_IRR_v2.4.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_121222/HYPERNETS_CAL_HYPSTAR_121222_RAD_v2.0.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_121222/HYPERNETS_CAL_HYPSTAR_121222_RAD_v2.1.nc similarity index 93% rename from hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_121222/HYPERNETS_CAL_HYPSTAR_121222_RAD_v2.0.nc rename to hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_121222/HYPERNETS_CAL_HYPSTAR_121222_RAD_v2.1.nc index 32aa3cd5..508c33da 100644 Binary files a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_121222/HYPERNETS_CAL_HYPSTAR_121222_RAD_v2.0.nc and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_121222/HYPERNETS_CAL_HYPSTAR_121222_RAD_v2.1.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_121222/HYPERNETS_CAL_HYPSTAR_121222_RAD_v2.2.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_121222/HYPERNETS_CAL_HYPSTAR_121222_RAD_v2.2.nc new file mode 100644 index 00000000..d6ad56b1 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_121222/HYPERNETS_CAL_HYPSTAR_121222_RAD_v2.2.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_121222/HYPERNETS_CAL_HYPSTAR_121222_RAD_v2.3.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_121222/HYPERNETS_CAL_HYPSTAR_121222_RAD_v2.3.nc new file mode 100644 index 00000000..c7af138f Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_121222/HYPERNETS_CAL_HYPSTAR_121222_RAD_v2.3.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_121222/HYPERNETS_CAL_HYPSTAR_121222_RAD_v2.4.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_121222/HYPERNETS_CAL_HYPSTAR_121222_RAD_v2.4.nc new file mode 100644 index 00000000..017e47bb Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_121222/HYPERNETS_CAL_HYPSTAR_121222_RAD_v2.4.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_121231/HYPERNETS_CAL_HYPSTAR_121231_IRR_v2.0.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_121231/HYPERNETS_CAL_HYPSTAR_121231_IRR_v2.1.nc similarity index 95% rename from hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_121231/HYPERNETS_CAL_HYPSTAR_121231_IRR_v2.0.nc rename to hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_121231/HYPERNETS_CAL_HYPSTAR_121231_IRR_v2.1.nc index 3d92379b..3f448e3b 100644 Binary files a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_121231/HYPERNETS_CAL_HYPSTAR_121231_IRR_v2.0.nc and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_121231/HYPERNETS_CAL_HYPSTAR_121231_IRR_v2.1.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_121231/HYPERNETS_CAL_HYPSTAR_121231_IRR_v2.2.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_121231/HYPERNETS_CAL_HYPSTAR_121231_IRR_v2.2.nc new file mode 100644 index 00000000..a59c2706 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_121231/HYPERNETS_CAL_HYPSTAR_121231_IRR_v2.2.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_121231/HYPERNETS_CAL_HYPSTAR_121231_IRR_v2.3.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_121231/HYPERNETS_CAL_HYPSTAR_121231_IRR_v2.3.nc new file mode 100644 index 00000000..fbb5086e Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_121231/HYPERNETS_CAL_HYPSTAR_121231_IRR_v2.3.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_121231/HYPERNETS_CAL_HYPSTAR_121231_IRR_v2.4.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_121231/HYPERNETS_CAL_HYPSTAR_121231_IRR_v2.4.nc new file mode 100644 index 00000000..dff1b27e Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_121231/HYPERNETS_CAL_HYPSTAR_121231_IRR_v2.4.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_121231/HYPERNETS_CAL_HYPSTAR_121231_RAD_v2.0.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_121231/HYPERNETS_CAL_HYPSTAR_121231_RAD_v2.1.nc similarity index 93% rename from hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_121231/HYPERNETS_CAL_HYPSTAR_121231_RAD_v2.0.nc rename to hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_121231/HYPERNETS_CAL_HYPSTAR_121231_RAD_v2.1.nc index 64f9d512..15330118 100644 Binary files a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_121231/HYPERNETS_CAL_HYPSTAR_121231_RAD_v2.0.nc and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_121231/HYPERNETS_CAL_HYPSTAR_121231_RAD_v2.1.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_121231/HYPERNETS_CAL_HYPSTAR_121231_RAD_v2.2.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_121231/HYPERNETS_CAL_HYPSTAR_121231_RAD_v2.2.nc new file mode 100644 index 00000000..201fcd3f Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_121231/HYPERNETS_CAL_HYPSTAR_121231_RAD_v2.2.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_121231/HYPERNETS_CAL_HYPSTAR_121231_RAD_v2.3.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_121231/HYPERNETS_CAL_HYPSTAR_121231_RAD_v2.3.nc new file mode 100644 index 00000000..5e167620 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_121231/HYPERNETS_CAL_HYPSTAR_121231_RAD_v2.3.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_121231/HYPERNETS_CAL_HYPSTAR_121231_RAD_v2.4.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_121231/HYPERNETS_CAL_HYPSTAR_121231_RAD_v2.4.nc new file mode 100644 index 00000000..041577c7 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_121231/HYPERNETS_CAL_HYPSTAR_121231_RAD_v2.4.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122241/HYPERNETS_CAL_HYPSTAR_122241_IRR_v2.0.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122241/HYPERNETS_CAL_HYPSTAR_122241_IRR_v2.0.nc deleted file mode 100644 index 9c4d18bb..00000000 Binary files a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122241/HYPERNETS_CAL_HYPSTAR_122241_IRR_v2.0.nc and /dev/null differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122241/HYPERNETS_CAL_HYPSTAR_122241_IRR_v2.1.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122241/HYPERNETS_CAL_HYPSTAR_122241_IRR_v2.1.nc new file mode 100644 index 00000000..56697a6e Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122241/HYPERNETS_CAL_HYPSTAR_122241_IRR_v2.1.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122241/HYPERNETS_CAL_HYPSTAR_122241_IRR_v2.2.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122241/HYPERNETS_CAL_HYPSTAR_122241_IRR_v2.2.nc new file mode 100644 index 00000000..aac08421 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122241/HYPERNETS_CAL_HYPSTAR_122241_IRR_v2.2.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122241/HYPERNETS_CAL_HYPSTAR_122241_IRR_v2.3.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122241/HYPERNETS_CAL_HYPSTAR_122241_IRR_v2.3.nc new file mode 100644 index 00000000..b8296e74 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122241/HYPERNETS_CAL_HYPSTAR_122241_IRR_v2.3.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122241/HYPERNETS_CAL_HYPSTAR_122241_IRR_v2.4.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122241/HYPERNETS_CAL_HYPSTAR_122241_IRR_v2.4.nc new file mode 100644 index 00000000..762624ce Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122241/HYPERNETS_CAL_HYPSTAR_122241_IRR_v2.4.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122241/HYPERNETS_CAL_HYPSTAR_122241_RAD_v2.0.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122241/HYPERNETS_CAL_HYPSTAR_122241_RAD_v2.0.nc deleted file mode 100644 index 665f5ab4..00000000 Binary files a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122241/HYPERNETS_CAL_HYPSTAR_122241_RAD_v2.0.nc and /dev/null differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122241/HYPERNETS_CAL_HYPSTAR_122241_RAD_v2.1.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122241/HYPERNETS_CAL_HYPSTAR_122241_RAD_v2.1.nc new file mode 100644 index 00000000..06cf0bfc Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122241/HYPERNETS_CAL_HYPSTAR_122241_RAD_v2.1.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122241/HYPERNETS_CAL_HYPSTAR_122241_RAD_v2.2.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122241/HYPERNETS_CAL_HYPSTAR_122241_RAD_v2.2.nc new file mode 100644 index 00000000..d6e3ae56 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122241/HYPERNETS_CAL_HYPSTAR_122241_RAD_v2.2.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122241/HYPERNETS_CAL_HYPSTAR_122241_RAD_v2.3.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122241/HYPERNETS_CAL_HYPSTAR_122241_RAD_v2.3.nc new file mode 100644 index 00000000..18390301 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122241/HYPERNETS_CAL_HYPSTAR_122241_RAD_v2.3.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122241/HYPERNETS_CAL_HYPSTAR_122241_RAD_v2.4.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122241/HYPERNETS_CAL_HYPSTAR_122241_RAD_v2.4.nc new file mode 100644 index 00000000..62254cbe Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122241/HYPERNETS_CAL_HYPSTAR_122241_RAD_v2.4.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122242/HYPERNETS_CAL_HYPSTAR_122242_IRR_v2.0.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122242/HYPERNETS_CAL_HYPSTAR_122242_IRR_v2.1.nc similarity index 97% rename from hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122242/HYPERNETS_CAL_HYPSTAR_122242_IRR_v2.0.nc rename to hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122242/HYPERNETS_CAL_HYPSTAR_122242_IRR_v2.1.nc index 2c518ab3..67342aa8 100644 Binary files a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122242/HYPERNETS_CAL_HYPSTAR_122242_IRR_v2.0.nc and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122242/HYPERNETS_CAL_HYPSTAR_122242_IRR_v2.1.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122242/HYPERNETS_CAL_HYPSTAR_122242_IRR_v2.2.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122242/HYPERNETS_CAL_HYPSTAR_122242_IRR_v2.2.nc new file mode 100644 index 00000000..2d3864a4 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122242/HYPERNETS_CAL_HYPSTAR_122242_IRR_v2.2.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122242/HYPERNETS_CAL_HYPSTAR_122242_IRR_v2.3.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122242/HYPERNETS_CAL_HYPSTAR_122242_IRR_v2.3.nc new file mode 100644 index 00000000..95f87481 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122242/HYPERNETS_CAL_HYPSTAR_122242_IRR_v2.3.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122242/HYPERNETS_CAL_HYPSTAR_122242_IRR_v2.4.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122242/HYPERNETS_CAL_HYPSTAR_122242_IRR_v2.4.nc new file mode 100644 index 00000000..dbb6bf16 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122242/HYPERNETS_CAL_HYPSTAR_122242_IRR_v2.4.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122242/HYPERNETS_CAL_HYPSTAR_122242_RAD_v2.0.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122242/HYPERNETS_CAL_HYPSTAR_122242_RAD_v2.1.nc similarity index 96% rename from hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122242/HYPERNETS_CAL_HYPSTAR_122242_RAD_v2.0.nc rename to hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122242/HYPERNETS_CAL_HYPSTAR_122242_RAD_v2.1.nc index 93f7dff3..259612e3 100644 Binary files a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122242/HYPERNETS_CAL_HYPSTAR_122242_RAD_v2.0.nc and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122242/HYPERNETS_CAL_HYPSTAR_122242_RAD_v2.1.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122242/HYPERNETS_CAL_HYPSTAR_122242_RAD_v2.2.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122242/HYPERNETS_CAL_HYPSTAR_122242_RAD_v2.2.nc new file mode 100644 index 00000000..53c930ec Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122242/HYPERNETS_CAL_HYPSTAR_122242_RAD_v2.2.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122242/HYPERNETS_CAL_HYPSTAR_122242_RAD_v2.3.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122242/HYPERNETS_CAL_HYPSTAR_122242_RAD_v2.3.nc new file mode 100644 index 00000000..2462cbed Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122242/HYPERNETS_CAL_HYPSTAR_122242_RAD_v2.3.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122242/HYPERNETS_CAL_HYPSTAR_122242_RAD_v2.4.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122242/HYPERNETS_CAL_HYPSTAR_122242_RAD_v2.4.nc new file mode 100644 index 00000000..23195d2c Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122242/HYPERNETS_CAL_HYPSTAR_122242_RAD_v2.4.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122301/HYPERNETS_CAL_HYPSTAR_122301_IRR_v2.0.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122301/HYPERNETS_CAL_HYPSTAR_122301_IRR_v2.1.nc similarity index 97% rename from hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122301/HYPERNETS_CAL_HYPSTAR_122301_IRR_v2.0.nc rename to hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122301/HYPERNETS_CAL_HYPSTAR_122301_IRR_v2.1.nc index a1dddbdf..0f0fb4d2 100644 Binary files a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122301/HYPERNETS_CAL_HYPSTAR_122301_IRR_v2.0.nc and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122301/HYPERNETS_CAL_HYPSTAR_122301_IRR_v2.1.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122301/HYPERNETS_CAL_HYPSTAR_122301_IRR_v2.2.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122301/HYPERNETS_CAL_HYPSTAR_122301_IRR_v2.2.nc new file mode 100644 index 00000000..ef4c4de8 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122301/HYPERNETS_CAL_HYPSTAR_122301_IRR_v2.2.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122301/HYPERNETS_CAL_HYPSTAR_122301_IRR_v2.3.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122301/HYPERNETS_CAL_HYPSTAR_122301_IRR_v2.3.nc new file mode 100644 index 00000000..de0b38be Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122301/HYPERNETS_CAL_HYPSTAR_122301_IRR_v2.3.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122301/HYPERNETS_CAL_HYPSTAR_122301_IRR_v2.4.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122301/HYPERNETS_CAL_HYPSTAR_122301_IRR_v2.4.nc new file mode 100644 index 00000000..a00e33ab Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122301/HYPERNETS_CAL_HYPSTAR_122301_IRR_v2.4.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122301/HYPERNETS_CAL_HYPSTAR_122301_RAD_v2.0.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122301/HYPERNETS_CAL_HYPSTAR_122301_RAD_v2.1.nc similarity index 96% rename from hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122301/HYPERNETS_CAL_HYPSTAR_122301_RAD_v2.0.nc rename to hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122301/HYPERNETS_CAL_HYPSTAR_122301_RAD_v2.1.nc index b7eeacb8..45ff776a 100644 Binary files a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122301/HYPERNETS_CAL_HYPSTAR_122301_RAD_v2.0.nc and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122301/HYPERNETS_CAL_HYPSTAR_122301_RAD_v2.1.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122301/HYPERNETS_CAL_HYPSTAR_122301_RAD_v2.2.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122301/HYPERNETS_CAL_HYPSTAR_122301_RAD_v2.2.nc new file mode 100644 index 00000000..646f2899 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122301/HYPERNETS_CAL_HYPSTAR_122301_RAD_v2.2.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122301/HYPERNETS_CAL_HYPSTAR_122301_RAD_v2.3.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122301/HYPERNETS_CAL_HYPSTAR_122301_RAD_v2.3.nc new file mode 100644 index 00000000..30901280 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122301/HYPERNETS_CAL_HYPSTAR_122301_RAD_v2.3.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122301/HYPERNETS_CAL_HYPSTAR_122301_RAD_v2.4.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122301/HYPERNETS_CAL_HYPSTAR_122301_RAD_v2.4.nc new file mode 100644 index 00000000..5a12f79c Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122301/HYPERNETS_CAL_HYPSTAR_122301_RAD_v2.4.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122302/HYPERNETS_CAL_HYPSTAR_122302_IRR_v2.0.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122302/HYPERNETS_CAL_HYPSTAR_122302_IRR_v2.1.nc similarity index 97% rename from hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122302/HYPERNETS_CAL_HYPSTAR_122302_IRR_v2.0.nc rename to hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122302/HYPERNETS_CAL_HYPSTAR_122302_IRR_v2.1.nc index 4e9b9815..18a2a828 100644 Binary files a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122302/HYPERNETS_CAL_HYPSTAR_122302_IRR_v2.0.nc and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122302/HYPERNETS_CAL_HYPSTAR_122302_IRR_v2.1.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122302/HYPERNETS_CAL_HYPSTAR_122302_IRR_v2.2.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122302/HYPERNETS_CAL_HYPSTAR_122302_IRR_v2.2.nc new file mode 100644 index 00000000..3803f090 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122302/HYPERNETS_CAL_HYPSTAR_122302_IRR_v2.2.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122302/HYPERNETS_CAL_HYPSTAR_122302_IRR_v2.3.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122302/HYPERNETS_CAL_HYPSTAR_122302_IRR_v2.3.nc new file mode 100644 index 00000000..2d428d76 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122302/HYPERNETS_CAL_HYPSTAR_122302_IRR_v2.3.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122302/HYPERNETS_CAL_HYPSTAR_122302_IRR_v2.4.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122302/HYPERNETS_CAL_HYPSTAR_122302_IRR_v2.4.nc new file mode 100644 index 00000000..d5f823a8 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122302/HYPERNETS_CAL_HYPSTAR_122302_IRR_v2.4.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122302/HYPERNETS_CAL_HYPSTAR_122302_RAD_v2.0.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122302/HYPERNETS_CAL_HYPSTAR_122302_RAD_v2.1.nc similarity index 96% rename from hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122302/HYPERNETS_CAL_HYPSTAR_122302_RAD_v2.0.nc rename to hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122302/HYPERNETS_CAL_HYPSTAR_122302_RAD_v2.1.nc index ab35480a..c27c7fc0 100644 Binary files a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122302/HYPERNETS_CAL_HYPSTAR_122302_RAD_v2.0.nc and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122302/HYPERNETS_CAL_HYPSTAR_122302_RAD_v2.1.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122302/HYPERNETS_CAL_HYPSTAR_122302_RAD_v2.2.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122302/HYPERNETS_CAL_HYPSTAR_122302_RAD_v2.2.nc new file mode 100644 index 00000000..dd9736c4 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122302/HYPERNETS_CAL_HYPSTAR_122302_RAD_v2.2.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122302/HYPERNETS_CAL_HYPSTAR_122302_RAD_v2.3.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122302/HYPERNETS_CAL_HYPSTAR_122302_RAD_v2.3.nc new file mode 100644 index 00000000..5ed72a57 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122302/HYPERNETS_CAL_HYPSTAR_122302_RAD_v2.3.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122302/HYPERNETS_CAL_HYPSTAR_122302_RAD_v2.4.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122302/HYPERNETS_CAL_HYPSTAR_122302_RAD_v2.4.nc new file mode 100644 index 00000000..a53501d3 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122302/HYPERNETS_CAL_HYPSTAR_122302_RAD_v2.4.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122303/HYPERNETS_CAL_HYPSTAR_122303_IRR_v2.0.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122303/HYPERNETS_CAL_HYPSTAR_122303_IRR_v2.1.nc similarity index 97% rename from hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122303/HYPERNETS_CAL_HYPSTAR_122303_IRR_v2.0.nc rename to hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122303/HYPERNETS_CAL_HYPSTAR_122303_IRR_v2.1.nc index 6265778c..cd7d1c50 100644 Binary files a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122303/HYPERNETS_CAL_HYPSTAR_122303_IRR_v2.0.nc and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122303/HYPERNETS_CAL_HYPSTAR_122303_IRR_v2.1.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122303/HYPERNETS_CAL_HYPSTAR_122303_IRR_v2.2.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122303/HYPERNETS_CAL_HYPSTAR_122303_IRR_v2.2.nc new file mode 100644 index 00000000..ed1a961b Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122303/HYPERNETS_CAL_HYPSTAR_122303_IRR_v2.2.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122303/HYPERNETS_CAL_HYPSTAR_122303_IRR_v2.3.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122303/HYPERNETS_CAL_HYPSTAR_122303_IRR_v2.3.nc new file mode 100644 index 00000000..06ed712b Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122303/HYPERNETS_CAL_HYPSTAR_122303_IRR_v2.3.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122303/HYPERNETS_CAL_HYPSTAR_122303_IRR_v2.4.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122303/HYPERNETS_CAL_HYPSTAR_122303_IRR_v2.4.nc new file mode 100644 index 00000000..8caf5214 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122303/HYPERNETS_CAL_HYPSTAR_122303_IRR_v2.4.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122303/HYPERNETS_CAL_HYPSTAR_122303_RAD_v2.0.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122303/HYPERNETS_CAL_HYPSTAR_122303_RAD_v2.1.nc similarity index 96% rename from hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122303/HYPERNETS_CAL_HYPSTAR_122303_RAD_v2.0.nc rename to hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122303/HYPERNETS_CAL_HYPSTAR_122303_RAD_v2.1.nc index 91694e9c..530d312e 100644 Binary files a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122303/HYPERNETS_CAL_HYPSTAR_122303_RAD_v2.0.nc and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122303/HYPERNETS_CAL_HYPSTAR_122303_RAD_v2.1.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122303/HYPERNETS_CAL_HYPSTAR_122303_RAD_v2.2.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122303/HYPERNETS_CAL_HYPSTAR_122303_RAD_v2.2.nc new file mode 100644 index 00000000..60736ca7 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122303/HYPERNETS_CAL_HYPSTAR_122303_RAD_v2.2.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122303/HYPERNETS_CAL_HYPSTAR_122303_RAD_v2.3.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122303/HYPERNETS_CAL_HYPSTAR_122303_RAD_v2.3.nc new file mode 100644 index 00000000..d9a8f59d Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122303/HYPERNETS_CAL_HYPSTAR_122303_RAD_v2.3.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122303/HYPERNETS_CAL_HYPSTAR_122303_RAD_v2.4.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122303/HYPERNETS_CAL_HYPSTAR_122303_RAD_v2.4.nc new file mode 100644 index 00000000..4571141a Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122303/HYPERNETS_CAL_HYPSTAR_122303_RAD_v2.4.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122304/HYPERNETS_CAL_HYPSTAR_122304_IRR_v2.0.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122304/HYPERNETS_CAL_HYPSTAR_122304_IRR_v2.1.nc similarity index 97% rename from hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122304/HYPERNETS_CAL_HYPSTAR_122304_IRR_v2.0.nc rename to hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122304/HYPERNETS_CAL_HYPSTAR_122304_IRR_v2.1.nc index ca6c3e87..efc551c7 100644 Binary files a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122304/HYPERNETS_CAL_HYPSTAR_122304_IRR_v2.0.nc and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122304/HYPERNETS_CAL_HYPSTAR_122304_IRR_v2.1.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122304/HYPERNETS_CAL_HYPSTAR_122304_IRR_v2.2.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122304/HYPERNETS_CAL_HYPSTAR_122304_IRR_v2.2.nc new file mode 100644 index 00000000..16cf4896 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122304/HYPERNETS_CAL_HYPSTAR_122304_IRR_v2.2.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122304/HYPERNETS_CAL_HYPSTAR_122304_IRR_v2.3.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122304/HYPERNETS_CAL_HYPSTAR_122304_IRR_v2.3.nc new file mode 100644 index 00000000..78d2206d Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122304/HYPERNETS_CAL_HYPSTAR_122304_IRR_v2.3.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122304/HYPERNETS_CAL_HYPSTAR_122304_IRR_v2.4.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122304/HYPERNETS_CAL_HYPSTAR_122304_IRR_v2.4.nc new file mode 100644 index 00000000..14f38d4b Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122304/HYPERNETS_CAL_HYPSTAR_122304_IRR_v2.4.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122304/HYPERNETS_CAL_HYPSTAR_122304_RAD_v2.0.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122304/HYPERNETS_CAL_HYPSTAR_122304_RAD_v2.1.nc similarity index 96% rename from hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122304/HYPERNETS_CAL_HYPSTAR_122304_RAD_v2.0.nc rename to hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122304/HYPERNETS_CAL_HYPSTAR_122304_RAD_v2.1.nc index c67fc34f..3704e426 100644 Binary files a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122304/HYPERNETS_CAL_HYPSTAR_122304_RAD_v2.0.nc and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122304/HYPERNETS_CAL_HYPSTAR_122304_RAD_v2.1.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122304/HYPERNETS_CAL_HYPSTAR_122304_RAD_v2.2.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122304/HYPERNETS_CAL_HYPSTAR_122304_RAD_v2.2.nc new file mode 100644 index 00000000..cd715cb9 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122304/HYPERNETS_CAL_HYPSTAR_122304_RAD_v2.2.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122304/HYPERNETS_CAL_HYPSTAR_122304_RAD_v2.3.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122304/HYPERNETS_CAL_HYPSTAR_122304_RAD_v2.3.nc new file mode 100644 index 00000000..93e1e57c Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122304/HYPERNETS_CAL_HYPSTAR_122304_RAD_v2.3.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122304/HYPERNETS_CAL_HYPSTAR_122304_RAD_v2.4.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122304/HYPERNETS_CAL_HYPSTAR_122304_RAD_v2.4.nc new file mode 100644 index 00000000..46039104 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122304/HYPERNETS_CAL_HYPSTAR_122304_RAD_v2.4.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122305/HYPERNETS_CAL_HYPSTAR_122305_IRR_v2.0.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122305/HYPERNETS_CAL_HYPSTAR_122305_IRR_v2.1.nc similarity index 97% rename from hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122305/HYPERNETS_CAL_HYPSTAR_122305_IRR_v2.0.nc rename to hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122305/HYPERNETS_CAL_HYPSTAR_122305_IRR_v2.1.nc index ff917c95..97429571 100644 Binary files a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122305/HYPERNETS_CAL_HYPSTAR_122305_IRR_v2.0.nc and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122305/HYPERNETS_CAL_HYPSTAR_122305_IRR_v2.1.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122305/HYPERNETS_CAL_HYPSTAR_122305_IRR_v2.2.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122305/HYPERNETS_CAL_HYPSTAR_122305_IRR_v2.2.nc new file mode 100644 index 00000000..275ac980 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122305/HYPERNETS_CAL_HYPSTAR_122305_IRR_v2.2.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122305/HYPERNETS_CAL_HYPSTAR_122305_IRR_v2.3.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122305/HYPERNETS_CAL_HYPSTAR_122305_IRR_v2.3.nc new file mode 100644 index 00000000..f245ae92 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122305/HYPERNETS_CAL_HYPSTAR_122305_IRR_v2.3.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122305/HYPERNETS_CAL_HYPSTAR_122305_IRR_v2.4.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122305/HYPERNETS_CAL_HYPSTAR_122305_IRR_v2.4.nc new file mode 100644 index 00000000..484db2f8 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122305/HYPERNETS_CAL_HYPSTAR_122305_IRR_v2.4.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122305/HYPERNETS_CAL_HYPSTAR_122305_RAD_v2.0.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122305/HYPERNETS_CAL_HYPSTAR_122305_RAD_v2.1.nc similarity index 96% rename from hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122305/HYPERNETS_CAL_HYPSTAR_122305_RAD_v2.0.nc rename to hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122305/HYPERNETS_CAL_HYPSTAR_122305_RAD_v2.1.nc index 01e6c25f..3404a7fb 100644 Binary files a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122305/HYPERNETS_CAL_HYPSTAR_122305_RAD_v2.0.nc and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122305/HYPERNETS_CAL_HYPSTAR_122305_RAD_v2.1.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122305/HYPERNETS_CAL_HYPSTAR_122305_RAD_v2.2.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122305/HYPERNETS_CAL_HYPSTAR_122305_RAD_v2.2.nc new file mode 100644 index 00000000..b9d3ee32 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122305/HYPERNETS_CAL_HYPSTAR_122305_RAD_v2.2.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122305/HYPERNETS_CAL_HYPSTAR_122305_RAD_v2.3.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122305/HYPERNETS_CAL_HYPSTAR_122305_RAD_v2.3.nc new file mode 100644 index 00000000..d98dd514 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122305/HYPERNETS_CAL_HYPSTAR_122305_RAD_v2.3.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122305/HYPERNETS_CAL_HYPSTAR_122305_RAD_v2.4.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122305/HYPERNETS_CAL_HYPSTAR_122305_RAD_v2.4.nc new file mode 100644 index 00000000..b3d90092 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122305/HYPERNETS_CAL_HYPSTAR_122305_RAD_v2.4.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122306/HYPERNETS_CAL_HYPSTAR_122306_IRR_v2.0.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122306/HYPERNETS_CAL_HYPSTAR_122306_IRR_v2.1.nc similarity index 97% rename from hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122306/HYPERNETS_CAL_HYPSTAR_122306_IRR_v2.0.nc rename to hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122306/HYPERNETS_CAL_HYPSTAR_122306_IRR_v2.1.nc index e98be4b9..60a688dd 100644 Binary files a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122306/HYPERNETS_CAL_HYPSTAR_122306_IRR_v2.0.nc and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122306/HYPERNETS_CAL_HYPSTAR_122306_IRR_v2.1.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122306/HYPERNETS_CAL_HYPSTAR_122306_IRR_v2.2.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122306/HYPERNETS_CAL_HYPSTAR_122306_IRR_v2.2.nc new file mode 100644 index 00000000..05939aa0 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122306/HYPERNETS_CAL_HYPSTAR_122306_IRR_v2.2.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122306/HYPERNETS_CAL_HYPSTAR_122306_IRR_v2.3.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122306/HYPERNETS_CAL_HYPSTAR_122306_IRR_v2.3.nc new file mode 100644 index 00000000..12225626 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122306/HYPERNETS_CAL_HYPSTAR_122306_IRR_v2.3.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122306/HYPERNETS_CAL_HYPSTAR_122306_IRR_v2.4.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122306/HYPERNETS_CAL_HYPSTAR_122306_IRR_v2.4.nc new file mode 100644 index 00000000..4242a395 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122306/HYPERNETS_CAL_HYPSTAR_122306_IRR_v2.4.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122306/HYPERNETS_CAL_HYPSTAR_122306_RAD_v2.0.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122306/HYPERNETS_CAL_HYPSTAR_122306_RAD_v2.1.nc similarity index 96% rename from hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122306/HYPERNETS_CAL_HYPSTAR_122306_RAD_v2.0.nc rename to hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122306/HYPERNETS_CAL_HYPSTAR_122306_RAD_v2.1.nc index ce941bf4..5e377da9 100644 Binary files a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122306/HYPERNETS_CAL_HYPSTAR_122306_RAD_v2.0.nc and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122306/HYPERNETS_CAL_HYPSTAR_122306_RAD_v2.1.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122306/HYPERNETS_CAL_HYPSTAR_122306_RAD_v2.2.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122306/HYPERNETS_CAL_HYPSTAR_122306_RAD_v2.2.nc new file mode 100644 index 00000000..43639559 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122306/HYPERNETS_CAL_HYPSTAR_122306_RAD_v2.2.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122306/HYPERNETS_CAL_HYPSTAR_122306_RAD_v2.3.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122306/HYPERNETS_CAL_HYPSTAR_122306_RAD_v2.3.nc new file mode 100644 index 00000000..b2e72719 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122306/HYPERNETS_CAL_HYPSTAR_122306_RAD_v2.3.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122306/HYPERNETS_CAL_HYPSTAR_122306_RAD_v2.4.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122306/HYPERNETS_CAL_HYPSTAR_122306_RAD_v2.4.nc new file mode 100644 index 00000000..0e2527b3 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122306/HYPERNETS_CAL_HYPSTAR_122306_RAD_v2.4.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122307/HYPERNETS_CAL_HYPSTAR_122307_IRR_v2.0.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122307/HYPERNETS_CAL_HYPSTAR_122307_IRR_v2.1.nc similarity index 97% rename from hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122307/HYPERNETS_CAL_HYPSTAR_122307_IRR_v2.0.nc rename to hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122307/HYPERNETS_CAL_HYPSTAR_122307_IRR_v2.1.nc index 33a694ae..475f02f9 100644 Binary files a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122307/HYPERNETS_CAL_HYPSTAR_122307_IRR_v2.0.nc and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122307/HYPERNETS_CAL_HYPSTAR_122307_IRR_v2.1.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122307/HYPERNETS_CAL_HYPSTAR_122307_IRR_v2.2.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122307/HYPERNETS_CAL_HYPSTAR_122307_IRR_v2.2.nc new file mode 100644 index 00000000..8b79d8e2 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122307/HYPERNETS_CAL_HYPSTAR_122307_IRR_v2.2.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122307/HYPERNETS_CAL_HYPSTAR_122307_IRR_v2.3.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122307/HYPERNETS_CAL_HYPSTAR_122307_IRR_v2.3.nc new file mode 100644 index 00000000..ed2b2227 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122307/HYPERNETS_CAL_HYPSTAR_122307_IRR_v2.3.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122307/HYPERNETS_CAL_HYPSTAR_122307_IRR_v2.4.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122307/HYPERNETS_CAL_HYPSTAR_122307_IRR_v2.4.nc new file mode 100644 index 00000000..dfe809e4 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122307/HYPERNETS_CAL_HYPSTAR_122307_IRR_v2.4.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122307/HYPERNETS_CAL_HYPSTAR_122307_RAD_v2.0.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122307/HYPERNETS_CAL_HYPSTAR_122307_RAD_v2.1.nc similarity index 96% rename from hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122307/HYPERNETS_CAL_HYPSTAR_122307_RAD_v2.0.nc rename to hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122307/HYPERNETS_CAL_HYPSTAR_122307_RAD_v2.1.nc index 077db344..ba2faaab 100644 Binary files a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122307/HYPERNETS_CAL_HYPSTAR_122307_RAD_v2.0.nc and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122307/HYPERNETS_CAL_HYPSTAR_122307_RAD_v2.1.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122307/HYPERNETS_CAL_HYPSTAR_122307_RAD_v2.2.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122307/HYPERNETS_CAL_HYPSTAR_122307_RAD_v2.2.nc new file mode 100644 index 00000000..1d03e8e0 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122307/HYPERNETS_CAL_HYPSTAR_122307_RAD_v2.2.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122307/HYPERNETS_CAL_HYPSTAR_122307_RAD_v2.3.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122307/HYPERNETS_CAL_HYPSTAR_122307_RAD_v2.3.nc new file mode 100644 index 00000000..ce2c613c Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122307/HYPERNETS_CAL_HYPSTAR_122307_RAD_v2.3.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122307/HYPERNETS_CAL_HYPSTAR_122307_RAD_v2.4.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122307/HYPERNETS_CAL_HYPSTAR_122307_RAD_v2.4.nc new file mode 100644 index 00000000..96cb9394 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_122307/HYPERNETS_CAL_HYPSTAR_122307_RAD_v2.4.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_125111/HYPERNETS_CAL_HYPSTAR_125111_IRR_v2.3.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_125111/HYPERNETS_CAL_HYPSTAR_125111_IRR_v2.3.nc new file mode 100644 index 00000000..32746ab3 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_125111/HYPERNETS_CAL_HYPSTAR_125111_IRR_v2.3.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_125111/HYPERNETS_CAL_HYPSTAR_125111_IRR_v2.4.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_125111/HYPERNETS_CAL_HYPSTAR_125111_IRR_v2.4.nc new file mode 100644 index 00000000..eb3f61fa Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_125111/HYPERNETS_CAL_HYPSTAR_125111_IRR_v2.4.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_125111/HYPERNETS_CAL_HYPSTAR_125111_RAD_v2.3.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_125111/HYPERNETS_CAL_HYPSTAR_125111_RAD_v2.3.nc new file mode 100644 index 00000000..32355cb9 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_125111/HYPERNETS_CAL_HYPSTAR_125111_RAD_v2.3.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_125111/HYPERNETS_CAL_HYPSTAR_125111_RAD_v2.4.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_125111/HYPERNETS_CAL_HYPSTAR_125111_RAD_v2.4.nc new file mode 100644 index 00000000..adc0534c Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_125111/HYPERNETS_CAL_HYPSTAR_125111_RAD_v2.4.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_125112/HYPERNETS_CAL_HYPSTAR_125112_IRR_v2.3.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_125112/HYPERNETS_CAL_HYPSTAR_125112_IRR_v2.3.nc new file mode 100644 index 00000000..a4e2bd8d Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_125112/HYPERNETS_CAL_HYPSTAR_125112_IRR_v2.3.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_125112/HYPERNETS_CAL_HYPSTAR_125112_IRR_v2.4.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_125112/HYPERNETS_CAL_HYPSTAR_125112_IRR_v2.4.nc new file mode 100644 index 00000000..197becbd Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_125112/HYPERNETS_CAL_HYPSTAR_125112_IRR_v2.4.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_125112/HYPERNETS_CAL_HYPSTAR_125112_RAD_v2.3.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_125112/HYPERNETS_CAL_HYPSTAR_125112_RAD_v2.3.nc new file mode 100644 index 00000000..f1aceb77 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_125112/HYPERNETS_CAL_HYPSTAR_125112_RAD_v2.3.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_125112/HYPERNETS_CAL_HYPSTAR_125112_RAD_v2.4.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_125112/HYPERNETS_CAL_HYPSTAR_125112_RAD_v2.4.nc new file mode 100644 index 00000000..df2d83a7 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_125112/HYPERNETS_CAL_HYPSTAR_125112_RAD_v2.4.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_125113/HYPERNETS_CAL_HYPSTAR_125113_IRR_v2.3.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_125113/HYPERNETS_CAL_HYPSTAR_125113_IRR_v2.3.nc new file mode 100644 index 00000000..1eb135d0 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_125113/HYPERNETS_CAL_HYPSTAR_125113_IRR_v2.3.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_125113/HYPERNETS_CAL_HYPSTAR_125113_IRR_v2.4.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_125113/HYPERNETS_CAL_HYPSTAR_125113_IRR_v2.4.nc new file mode 100644 index 00000000..e1daea6e Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_125113/HYPERNETS_CAL_HYPSTAR_125113_IRR_v2.4.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_125113/HYPERNETS_CAL_HYPSTAR_125113_RAD_v2.3.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_125113/HYPERNETS_CAL_HYPSTAR_125113_RAD_v2.3.nc new file mode 100644 index 00000000..14bd3abd Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_125113/HYPERNETS_CAL_HYPSTAR_125113_RAD_v2.3.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_125113/HYPERNETS_CAL_HYPSTAR_125113_RAD_v2.4.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_125113/HYPERNETS_CAL_HYPSTAR_125113_RAD_v2.4.nc new file mode 100644 index 00000000..df68947e Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_125113/HYPERNETS_CAL_HYPSTAR_125113_RAD_v2.4.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_125114/HYPERNETS_CAL_HYPSTAR_125114_IRR_v2.3.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_125114/HYPERNETS_CAL_HYPSTAR_125114_IRR_v2.3.nc new file mode 100644 index 00000000..c0ea5f24 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_125114/HYPERNETS_CAL_HYPSTAR_125114_IRR_v2.3.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_125114/HYPERNETS_CAL_HYPSTAR_125114_IRR_v2.4.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_125114/HYPERNETS_CAL_HYPSTAR_125114_IRR_v2.4.nc new file mode 100644 index 00000000..0cb0e21d Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_125114/HYPERNETS_CAL_HYPSTAR_125114_IRR_v2.4.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_125114/HYPERNETS_CAL_HYPSTAR_125114_RAD_v2.3.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_125114/HYPERNETS_CAL_HYPSTAR_125114_RAD_v2.3.nc new file mode 100644 index 00000000..acd31bb7 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_125114/HYPERNETS_CAL_HYPSTAR_125114_RAD_v2.3.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_125114/HYPERNETS_CAL_HYPSTAR_125114_RAD_v2.4.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_125114/HYPERNETS_CAL_HYPSTAR_125114_RAD_v2.4.nc new file mode 100644 index 00000000..1053390c Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_125114/HYPERNETS_CAL_HYPSTAR_125114_RAD_v2.4.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220241/HYPERNETS_CAL_HYPSTAR_220241_IRR_SWIR_v2.0.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220241/HYPERNETS_CAL_HYPSTAR_220241_IRR_SWIR_v2.1.nc similarity index 65% rename from hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220241/HYPERNETS_CAL_HYPSTAR_220241_IRR_SWIR_v2.0.nc rename to hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220241/HYPERNETS_CAL_HYPSTAR_220241_IRR_SWIR_v2.1.nc index f537088d..20ac4052 100644 Binary files a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220241/HYPERNETS_CAL_HYPSTAR_220241_IRR_SWIR_v2.0.nc and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220241/HYPERNETS_CAL_HYPSTAR_220241_IRR_SWIR_v2.1.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220241/HYPERNETS_CAL_HYPSTAR_220241_IRR_SWIR_v2.2.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220241/HYPERNETS_CAL_HYPSTAR_220241_IRR_SWIR_v2.2.nc new file mode 100644 index 00000000..1b653542 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220241/HYPERNETS_CAL_HYPSTAR_220241_IRR_SWIR_v2.2.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220241/HYPERNETS_CAL_HYPSTAR_220241_IRR_SWIR_v2.3.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220241/HYPERNETS_CAL_HYPSTAR_220241_IRR_SWIR_v2.3.nc new file mode 100644 index 00000000..c64298cb Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220241/HYPERNETS_CAL_HYPSTAR_220241_IRR_SWIR_v2.3.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220241/HYPERNETS_CAL_HYPSTAR_220241_IRR_SWIR_v2.4.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220241/HYPERNETS_CAL_HYPSTAR_220241_IRR_SWIR_v2.4.nc new file mode 100644 index 00000000..1f12e328 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220241/HYPERNETS_CAL_HYPSTAR_220241_IRR_SWIR_v2.4.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220241/HYPERNETS_CAL_HYPSTAR_220241_IRR_v2.0.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220241/HYPERNETS_CAL_HYPSTAR_220241_IRR_v2.1.nc similarity index 97% rename from hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220241/HYPERNETS_CAL_HYPSTAR_220241_IRR_v2.0.nc rename to hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220241/HYPERNETS_CAL_HYPSTAR_220241_IRR_v2.1.nc index 158031e7..27bc4910 100644 Binary files a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220241/HYPERNETS_CAL_HYPSTAR_220241_IRR_v2.0.nc and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220241/HYPERNETS_CAL_HYPSTAR_220241_IRR_v2.1.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220241/HYPERNETS_CAL_HYPSTAR_220241_IRR_v2.2.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220241/HYPERNETS_CAL_HYPSTAR_220241_IRR_v2.2.nc new file mode 100644 index 00000000..7e3a31c4 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220241/HYPERNETS_CAL_HYPSTAR_220241_IRR_v2.2.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220241/HYPERNETS_CAL_HYPSTAR_220241_IRR_v2.3.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220241/HYPERNETS_CAL_HYPSTAR_220241_IRR_v2.3.nc new file mode 100644 index 00000000..3e7fd3e0 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220241/HYPERNETS_CAL_HYPSTAR_220241_IRR_v2.3.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220241/HYPERNETS_CAL_HYPSTAR_220241_IRR_v2.4.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220241/HYPERNETS_CAL_HYPSTAR_220241_IRR_v2.4.nc new file mode 100644 index 00000000..fd9d33a7 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220241/HYPERNETS_CAL_HYPSTAR_220241_IRR_v2.4.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220241/HYPERNETS_CAL_HYPSTAR_220241_RAD_SWIR_v2.0.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220241/HYPERNETS_CAL_HYPSTAR_220241_RAD_SWIR_v2.1.nc similarity index 65% rename from hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220241/HYPERNETS_CAL_HYPSTAR_220241_RAD_SWIR_v2.0.nc rename to hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220241/HYPERNETS_CAL_HYPSTAR_220241_RAD_SWIR_v2.1.nc index abd92ef3..c61d736e 100644 Binary files a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220241/HYPERNETS_CAL_HYPSTAR_220241_RAD_SWIR_v2.0.nc and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220241/HYPERNETS_CAL_HYPSTAR_220241_RAD_SWIR_v2.1.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220241/HYPERNETS_CAL_HYPSTAR_220241_RAD_SWIR_v2.2.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220241/HYPERNETS_CAL_HYPSTAR_220241_RAD_SWIR_v2.2.nc new file mode 100644 index 00000000..c008c22a Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220241/HYPERNETS_CAL_HYPSTAR_220241_RAD_SWIR_v2.2.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220241/HYPERNETS_CAL_HYPSTAR_220241_RAD_SWIR_v2.3.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220241/HYPERNETS_CAL_HYPSTAR_220241_RAD_SWIR_v2.3.nc new file mode 100644 index 00000000..8927309c Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220241/HYPERNETS_CAL_HYPSTAR_220241_RAD_SWIR_v2.3.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220241/HYPERNETS_CAL_HYPSTAR_220241_RAD_SWIR_v2.4.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220241/HYPERNETS_CAL_HYPSTAR_220241_RAD_SWIR_v2.4.nc new file mode 100644 index 00000000..a5a20c14 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220241/HYPERNETS_CAL_HYPSTAR_220241_RAD_SWIR_v2.4.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220241/HYPERNETS_CAL_HYPSTAR_220241_RAD_v2.0.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220241/HYPERNETS_CAL_HYPSTAR_220241_RAD_v2.1.nc similarity index 95% rename from hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220241/HYPERNETS_CAL_HYPSTAR_220241_RAD_v2.0.nc rename to hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220241/HYPERNETS_CAL_HYPSTAR_220241_RAD_v2.1.nc index 0d116219..83e0ff03 100644 Binary files a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220241/HYPERNETS_CAL_HYPSTAR_220241_RAD_v2.0.nc and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220241/HYPERNETS_CAL_HYPSTAR_220241_RAD_v2.1.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220241/HYPERNETS_CAL_HYPSTAR_220241_RAD_v2.2.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220241/HYPERNETS_CAL_HYPSTAR_220241_RAD_v2.2.nc new file mode 100644 index 00000000..041f2cdc Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220241/HYPERNETS_CAL_HYPSTAR_220241_RAD_v2.2.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220241/HYPERNETS_CAL_HYPSTAR_220241_RAD_v2.3.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220241/HYPERNETS_CAL_HYPSTAR_220241_RAD_v2.3.nc new file mode 100644 index 00000000..04e9c055 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220241/HYPERNETS_CAL_HYPSTAR_220241_RAD_v2.3.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220241/HYPERNETS_CAL_HYPSTAR_220241_RAD_v2.4.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220241/HYPERNETS_CAL_HYPSTAR_220241_RAD_v2.4.nc new file mode 100644 index 00000000..c7fac0de Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220241/HYPERNETS_CAL_HYPSTAR_220241_RAD_v2.4.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220251/HYPERNETS_CAL_HYPSTAR_220251_IRR_SWIR_v2.0.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220251/HYPERNETS_CAL_HYPSTAR_220251_IRR_SWIR_v2.0.nc deleted file mode 100644 index 312170ff..00000000 Binary files a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220251/HYPERNETS_CAL_HYPSTAR_220251_IRR_SWIR_v2.0.nc and /dev/null differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220251/HYPERNETS_CAL_HYPSTAR_220251_IRR_SWIR_v2.1.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220251/HYPERNETS_CAL_HYPSTAR_220251_IRR_SWIR_v2.1.nc new file mode 100644 index 00000000..dd1a1c8e Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220251/HYPERNETS_CAL_HYPSTAR_220251_IRR_SWIR_v2.1.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220251/HYPERNETS_CAL_HYPSTAR_220251_IRR_SWIR_v2.2.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220251/HYPERNETS_CAL_HYPSTAR_220251_IRR_SWIR_v2.2.nc new file mode 100644 index 00000000..2f34f9de Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220251/HYPERNETS_CAL_HYPSTAR_220251_IRR_SWIR_v2.2.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220251/HYPERNETS_CAL_HYPSTAR_220251_IRR_SWIR_v2.3.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220251/HYPERNETS_CAL_HYPSTAR_220251_IRR_SWIR_v2.3.nc new file mode 100644 index 00000000..89ecd9be Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220251/HYPERNETS_CAL_HYPSTAR_220251_IRR_SWIR_v2.3.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220251/HYPERNETS_CAL_HYPSTAR_220251_IRR_SWIR_v2.4.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220251/HYPERNETS_CAL_HYPSTAR_220251_IRR_SWIR_v2.4.nc new file mode 100644 index 00000000..3c7ca666 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220251/HYPERNETS_CAL_HYPSTAR_220251_IRR_SWIR_v2.4.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220251/HYPERNETS_CAL_HYPSTAR_220251_IRR_v2.0.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220251/HYPERNETS_CAL_HYPSTAR_220251_IRR_v2.0.nc deleted file mode 100644 index f42c0a9b..00000000 Binary files a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220251/HYPERNETS_CAL_HYPSTAR_220251_IRR_v2.0.nc and /dev/null differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220251/HYPERNETS_CAL_HYPSTAR_220251_IRR_v2.1.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220251/HYPERNETS_CAL_HYPSTAR_220251_IRR_v2.1.nc new file mode 100644 index 00000000..994dcf36 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220251/HYPERNETS_CAL_HYPSTAR_220251_IRR_v2.1.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220251/HYPERNETS_CAL_HYPSTAR_220251_IRR_v2.2.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220251/HYPERNETS_CAL_HYPSTAR_220251_IRR_v2.2.nc new file mode 100644 index 00000000..be0a3dda Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220251/HYPERNETS_CAL_HYPSTAR_220251_IRR_v2.2.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220251/HYPERNETS_CAL_HYPSTAR_220251_IRR_v2.3.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220251/HYPERNETS_CAL_HYPSTAR_220251_IRR_v2.3.nc new file mode 100644 index 00000000..9b61946e Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220251/HYPERNETS_CAL_HYPSTAR_220251_IRR_v2.3.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220251/HYPERNETS_CAL_HYPSTAR_220251_IRR_v2.4.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220251/HYPERNETS_CAL_HYPSTAR_220251_IRR_v2.4.nc new file mode 100644 index 00000000..f981bed5 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220251/HYPERNETS_CAL_HYPSTAR_220251_IRR_v2.4.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220251/HYPERNETS_CAL_HYPSTAR_220251_RAD_SWIR_v2.0.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220251/HYPERNETS_CAL_HYPSTAR_220251_RAD_SWIR_v2.0.nc deleted file mode 100644 index 0b52df39..00000000 Binary files a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220251/HYPERNETS_CAL_HYPSTAR_220251_RAD_SWIR_v2.0.nc and /dev/null differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220251/HYPERNETS_CAL_HYPSTAR_220251_RAD_SWIR_v2.1.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220251/HYPERNETS_CAL_HYPSTAR_220251_RAD_SWIR_v2.1.nc new file mode 100644 index 00000000..fff83391 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220251/HYPERNETS_CAL_HYPSTAR_220251_RAD_SWIR_v2.1.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220251/HYPERNETS_CAL_HYPSTAR_220251_RAD_SWIR_v2.2.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220251/HYPERNETS_CAL_HYPSTAR_220251_RAD_SWIR_v2.2.nc new file mode 100644 index 00000000..33150853 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220251/HYPERNETS_CAL_HYPSTAR_220251_RAD_SWIR_v2.2.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220251/HYPERNETS_CAL_HYPSTAR_220251_RAD_SWIR_v2.3.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220251/HYPERNETS_CAL_HYPSTAR_220251_RAD_SWIR_v2.3.nc new file mode 100644 index 00000000..faf54902 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220251/HYPERNETS_CAL_HYPSTAR_220251_RAD_SWIR_v2.3.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220251/HYPERNETS_CAL_HYPSTAR_220251_RAD_SWIR_v2.4.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220251/HYPERNETS_CAL_HYPSTAR_220251_RAD_SWIR_v2.4.nc new file mode 100644 index 00000000..0cb57d4f Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220251/HYPERNETS_CAL_HYPSTAR_220251_RAD_SWIR_v2.4.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220251/HYPERNETS_CAL_HYPSTAR_220251_RAD_v2.0.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220251/HYPERNETS_CAL_HYPSTAR_220251_RAD_v2.0.nc deleted file mode 100644 index c7a30947..00000000 Binary files a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220251/HYPERNETS_CAL_HYPSTAR_220251_RAD_v2.0.nc and /dev/null differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220251/HYPERNETS_CAL_HYPSTAR_220251_RAD_v2.1.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220251/HYPERNETS_CAL_HYPSTAR_220251_RAD_v2.1.nc new file mode 100644 index 00000000..a2b21b97 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220251/HYPERNETS_CAL_HYPSTAR_220251_RAD_v2.1.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220251/HYPERNETS_CAL_HYPSTAR_220251_RAD_v2.2.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220251/HYPERNETS_CAL_HYPSTAR_220251_RAD_v2.2.nc new file mode 100644 index 00000000..2af3ec55 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220251/HYPERNETS_CAL_HYPSTAR_220251_RAD_v2.2.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220251/HYPERNETS_CAL_HYPSTAR_220251_RAD_v2.3.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220251/HYPERNETS_CAL_HYPSTAR_220251_RAD_v2.3.nc new file mode 100644 index 00000000..347fd38a Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220251/HYPERNETS_CAL_HYPSTAR_220251_RAD_v2.3.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220251/HYPERNETS_CAL_HYPSTAR_220251_RAD_v2.4.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220251/HYPERNETS_CAL_HYPSTAR_220251_RAD_v2.4.nc new file mode 100644 index 00000000..b51d0332 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220251/HYPERNETS_CAL_HYPSTAR_220251_RAD_v2.4.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220261/HYPERNETS_CAL_HYPSTAR_220261_IRR_SWIR_v2.0.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220261/HYPERNETS_CAL_HYPSTAR_220261_IRR_SWIR_v2.0.nc deleted file mode 100644 index 1400937f..00000000 Binary files a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220261/HYPERNETS_CAL_HYPSTAR_220261_IRR_SWIR_v2.0.nc and /dev/null differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220261/HYPERNETS_CAL_HYPSTAR_220261_IRR_SWIR_v2.1.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220261/HYPERNETS_CAL_HYPSTAR_220261_IRR_SWIR_v2.1.nc new file mode 100644 index 00000000..170d21ca Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220261/HYPERNETS_CAL_HYPSTAR_220261_IRR_SWIR_v2.1.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220261/HYPERNETS_CAL_HYPSTAR_220261_IRR_SWIR_v2.2.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220261/HYPERNETS_CAL_HYPSTAR_220261_IRR_SWIR_v2.2.nc new file mode 100644 index 00000000..9cd9fbb2 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220261/HYPERNETS_CAL_HYPSTAR_220261_IRR_SWIR_v2.2.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220261/HYPERNETS_CAL_HYPSTAR_220261_IRR_SWIR_v2.3.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220261/HYPERNETS_CAL_HYPSTAR_220261_IRR_SWIR_v2.3.nc new file mode 100644 index 00000000..e921469c Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220261/HYPERNETS_CAL_HYPSTAR_220261_IRR_SWIR_v2.3.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220261/HYPERNETS_CAL_HYPSTAR_220261_IRR_SWIR_v2.4.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220261/HYPERNETS_CAL_HYPSTAR_220261_IRR_SWIR_v2.4.nc new file mode 100644 index 00000000..b3fa1da4 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220261/HYPERNETS_CAL_HYPSTAR_220261_IRR_SWIR_v2.4.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220261/HYPERNETS_CAL_HYPSTAR_220261_IRR_v2.0.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220261/HYPERNETS_CAL_HYPSTAR_220261_IRR_v2.0.nc deleted file mode 100644 index 23892386..00000000 Binary files a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220261/HYPERNETS_CAL_HYPSTAR_220261_IRR_v2.0.nc and /dev/null differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220261/HYPERNETS_CAL_HYPSTAR_220261_IRR_v2.1.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220261/HYPERNETS_CAL_HYPSTAR_220261_IRR_v2.1.nc new file mode 100644 index 00000000..6a22e73e Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220261/HYPERNETS_CAL_HYPSTAR_220261_IRR_v2.1.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220261/HYPERNETS_CAL_HYPSTAR_220261_IRR_v2.2.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220261/HYPERNETS_CAL_HYPSTAR_220261_IRR_v2.2.nc new file mode 100644 index 00000000..d8e1026e Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220261/HYPERNETS_CAL_HYPSTAR_220261_IRR_v2.2.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220261/HYPERNETS_CAL_HYPSTAR_220261_IRR_v2.3.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220261/HYPERNETS_CAL_HYPSTAR_220261_IRR_v2.3.nc new file mode 100644 index 00000000..972a892e Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220261/HYPERNETS_CAL_HYPSTAR_220261_IRR_v2.3.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220261/HYPERNETS_CAL_HYPSTAR_220261_IRR_v2.4.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220261/HYPERNETS_CAL_HYPSTAR_220261_IRR_v2.4.nc new file mode 100644 index 00000000..f437be09 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220261/HYPERNETS_CAL_HYPSTAR_220261_IRR_v2.4.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220261/HYPERNETS_CAL_HYPSTAR_220261_RAD_SWIR_v2.0.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220261/HYPERNETS_CAL_HYPSTAR_220261_RAD_SWIR_v2.0.nc deleted file mode 100644 index 2cd01b7b..00000000 Binary files a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220261/HYPERNETS_CAL_HYPSTAR_220261_RAD_SWIR_v2.0.nc and /dev/null differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220261/HYPERNETS_CAL_HYPSTAR_220261_RAD_SWIR_v2.1.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220261/HYPERNETS_CAL_HYPSTAR_220261_RAD_SWIR_v2.1.nc new file mode 100644 index 00000000..a5086302 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220261/HYPERNETS_CAL_HYPSTAR_220261_RAD_SWIR_v2.1.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220261/HYPERNETS_CAL_HYPSTAR_220261_RAD_SWIR_v2.2.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220261/HYPERNETS_CAL_HYPSTAR_220261_RAD_SWIR_v2.2.nc new file mode 100644 index 00000000..5b51a4ce Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220261/HYPERNETS_CAL_HYPSTAR_220261_RAD_SWIR_v2.2.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220261/HYPERNETS_CAL_HYPSTAR_220261_RAD_SWIR_v2.3.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220261/HYPERNETS_CAL_HYPSTAR_220261_RAD_SWIR_v2.3.nc new file mode 100644 index 00000000..cab4635b Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220261/HYPERNETS_CAL_HYPSTAR_220261_RAD_SWIR_v2.3.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220261/HYPERNETS_CAL_HYPSTAR_220261_RAD_SWIR_v2.4.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220261/HYPERNETS_CAL_HYPSTAR_220261_RAD_SWIR_v2.4.nc new file mode 100644 index 00000000..5a3299f0 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220261/HYPERNETS_CAL_HYPSTAR_220261_RAD_SWIR_v2.4.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220261/HYPERNETS_CAL_HYPSTAR_220261_RAD_v2.0.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220261/HYPERNETS_CAL_HYPSTAR_220261_RAD_v2.0.nc deleted file mode 100644 index 9c0f7bcc..00000000 Binary files a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220261/HYPERNETS_CAL_HYPSTAR_220261_RAD_v2.0.nc and /dev/null differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220261/HYPERNETS_CAL_HYPSTAR_220261_RAD_v2.1.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220261/HYPERNETS_CAL_HYPSTAR_220261_RAD_v2.1.nc new file mode 100644 index 00000000..2cb8d605 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220261/HYPERNETS_CAL_HYPSTAR_220261_RAD_v2.1.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220261/HYPERNETS_CAL_HYPSTAR_220261_RAD_v2.2.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220261/HYPERNETS_CAL_HYPSTAR_220261_RAD_v2.2.nc new file mode 100644 index 00000000..610af059 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220261/HYPERNETS_CAL_HYPSTAR_220261_RAD_v2.2.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220261/HYPERNETS_CAL_HYPSTAR_220261_RAD_v2.3.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220261/HYPERNETS_CAL_HYPSTAR_220261_RAD_v2.3.nc new file mode 100644 index 00000000..c81bb8c5 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220261/HYPERNETS_CAL_HYPSTAR_220261_RAD_v2.3.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220261/HYPERNETS_CAL_HYPSTAR_220261_RAD_v2.4.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220261/HYPERNETS_CAL_HYPSTAR_220261_RAD_v2.4.nc new file mode 100644 index 00000000..bc4baedb Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_220261/HYPERNETS_CAL_HYPSTAR_220261_RAD_v2.4.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221211/HYPERNETS_CAL_HYPSTAR_221211_IRR_SWIR_v2.0.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221211/HYPERNETS_CAL_HYPSTAR_221211_IRR_SWIR_v2.1.nc similarity index 73% rename from hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221211/HYPERNETS_CAL_HYPSTAR_221211_IRR_SWIR_v2.0.nc rename to hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221211/HYPERNETS_CAL_HYPSTAR_221211_IRR_SWIR_v2.1.nc index cd280581..9dab60f4 100644 Binary files a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221211/HYPERNETS_CAL_HYPSTAR_221211_IRR_SWIR_v2.0.nc and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221211/HYPERNETS_CAL_HYPSTAR_221211_IRR_SWIR_v2.1.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221211/HYPERNETS_CAL_HYPSTAR_221211_IRR_SWIR_v2.2.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221211/HYPERNETS_CAL_HYPSTAR_221211_IRR_SWIR_v2.2.nc new file mode 100644 index 00000000..ba57affe Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221211/HYPERNETS_CAL_HYPSTAR_221211_IRR_SWIR_v2.2.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221211/HYPERNETS_CAL_HYPSTAR_221211_IRR_SWIR_v2.3.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221211/HYPERNETS_CAL_HYPSTAR_221211_IRR_SWIR_v2.3.nc new file mode 100644 index 00000000..113ed267 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221211/HYPERNETS_CAL_HYPSTAR_221211_IRR_SWIR_v2.3.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221211/HYPERNETS_CAL_HYPSTAR_221211_IRR_SWIR_v2.4.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221211/HYPERNETS_CAL_HYPSTAR_221211_IRR_SWIR_v2.4.nc new file mode 100644 index 00000000..f38ecde7 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221211/HYPERNETS_CAL_HYPSTAR_221211_IRR_SWIR_v2.4.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221211/HYPERNETS_CAL_HYPSTAR_221211_IRR_v2.0.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221211/HYPERNETS_CAL_HYPSTAR_221211_IRR_v2.1.nc similarity index 99% rename from hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221211/HYPERNETS_CAL_HYPSTAR_221211_IRR_v2.0.nc rename to hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221211/HYPERNETS_CAL_HYPSTAR_221211_IRR_v2.1.nc index 151620f6..b4155a05 100644 Binary files a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221211/HYPERNETS_CAL_HYPSTAR_221211_IRR_v2.0.nc and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221211/HYPERNETS_CAL_HYPSTAR_221211_IRR_v2.1.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221211/HYPERNETS_CAL_HYPSTAR_221211_IRR_v2.2.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221211/HYPERNETS_CAL_HYPSTAR_221211_IRR_v2.2.nc new file mode 100644 index 00000000..d16be7dc Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221211/HYPERNETS_CAL_HYPSTAR_221211_IRR_v2.2.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221211/HYPERNETS_CAL_HYPSTAR_221211_IRR_v2.3.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221211/HYPERNETS_CAL_HYPSTAR_221211_IRR_v2.3.nc new file mode 100644 index 00000000..bdea2d64 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221211/HYPERNETS_CAL_HYPSTAR_221211_IRR_v2.3.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221211/HYPERNETS_CAL_HYPSTAR_221211_IRR_v2.4.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221211/HYPERNETS_CAL_HYPSTAR_221211_IRR_v2.4.nc new file mode 100644 index 00000000..c87eed41 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221211/HYPERNETS_CAL_HYPSTAR_221211_IRR_v2.4.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221211/HYPERNETS_CAL_HYPSTAR_221211_RAD_SWIR_v2.0.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221211/HYPERNETS_CAL_HYPSTAR_221211_RAD_SWIR_v2.1.nc similarity index 73% rename from hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221211/HYPERNETS_CAL_HYPSTAR_221211_RAD_SWIR_v2.0.nc rename to hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221211/HYPERNETS_CAL_HYPSTAR_221211_RAD_SWIR_v2.1.nc index a4d3fdb3..3836d8b4 100644 Binary files a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221211/HYPERNETS_CAL_HYPSTAR_221211_RAD_SWIR_v2.0.nc and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221211/HYPERNETS_CAL_HYPSTAR_221211_RAD_SWIR_v2.1.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221211/HYPERNETS_CAL_HYPSTAR_221211_RAD_SWIR_v2.2.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221211/HYPERNETS_CAL_HYPSTAR_221211_RAD_SWIR_v2.2.nc new file mode 100644 index 00000000..f8e4adf3 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221211/HYPERNETS_CAL_HYPSTAR_221211_RAD_SWIR_v2.2.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221211/HYPERNETS_CAL_HYPSTAR_221211_RAD_SWIR_v2.3.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221211/HYPERNETS_CAL_HYPSTAR_221211_RAD_SWIR_v2.3.nc new file mode 100644 index 00000000..238da728 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221211/HYPERNETS_CAL_HYPSTAR_221211_RAD_SWIR_v2.3.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221211/HYPERNETS_CAL_HYPSTAR_221211_RAD_SWIR_v2.4.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221211/HYPERNETS_CAL_HYPSTAR_221211_RAD_SWIR_v2.4.nc new file mode 100644 index 00000000..84b5f243 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221211/HYPERNETS_CAL_HYPSTAR_221211_RAD_SWIR_v2.4.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221211/HYPERNETS_CAL_HYPSTAR_221211_RAD_v2.0.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221211/HYPERNETS_CAL_HYPSTAR_221211_RAD_v2.1.nc similarity index 99% rename from hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221211/HYPERNETS_CAL_HYPSTAR_221211_RAD_v2.0.nc rename to hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221211/HYPERNETS_CAL_HYPSTAR_221211_RAD_v2.1.nc index a52fb966..c810e80f 100644 Binary files a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221211/HYPERNETS_CAL_HYPSTAR_221211_RAD_v2.0.nc and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221211/HYPERNETS_CAL_HYPSTAR_221211_RAD_v2.1.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221211/HYPERNETS_CAL_HYPSTAR_221211_RAD_v2.2.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221211/HYPERNETS_CAL_HYPSTAR_221211_RAD_v2.2.nc new file mode 100644 index 00000000..df4bc352 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221211/HYPERNETS_CAL_HYPSTAR_221211_RAD_v2.2.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221211/HYPERNETS_CAL_HYPSTAR_221211_RAD_v2.3.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221211/HYPERNETS_CAL_HYPSTAR_221211_RAD_v2.3.nc new file mode 100644 index 00000000..ac52ec7c Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221211/HYPERNETS_CAL_HYPSTAR_221211_RAD_v2.3.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221211/HYPERNETS_CAL_HYPSTAR_221211_RAD_v2.4.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221211/HYPERNETS_CAL_HYPSTAR_221211_RAD_v2.4.nc new file mode 100644 index 00000000..937f7d5b Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221211/HYPERNETS_CAL_HYPSTAR_221211_RAD_v2.4.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221231/HYPERNETS_CAL_HYPSTAR_221231_IRR_SWIR_v2.0.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221231/HYPERNETS_CAL_HYPSTAR_221231_IRR_SWIR_v2.1.nc similarity index 62% rename from hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221231/HYPERNETS_CAL_HYPSTAR_221231_IRR_SWIR_v2.0.nc rename to hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221231/HYPERNETS_CAL_HYPSTAR_221231_IRR_SWIR_v2.1.nc index 58fc4836..05d6e130 100644 Binary files a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221231/HYPERNETS_CAL_HYPSTAR_221231_IRR_SWIR_v2.0.nc and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221231/HYPERNETS_CAL_HYPSTAR_221231_IRR_SWIR_v2.1.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221231/HYPERNETS_CAL_HYPSTAR_221231_IRR_SWIR_v2.2.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221231/HYPERNETS_CAL_HYPSTAR_221231_IRR_SWIR_v2.2.nc new file mode 100644 index 00000000..8d0a86d3 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221231/HYPERNETS_CAL_HYPSTAR_221231_IRR_SWIR_v2.2.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221231/HYPERNETS_CAL_HYPSTAR_221231_IRR_SWIR_v2.3.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221231/HYPERNETS_CAL_HYPSTAR_221231_IRR_SWIR_v2.3.nc new file mode 100644 index 00000000..08a3e782 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221231/HYPERNETS_CAL_HYPSTAR_221231_IRR_SWIR_v2.3.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221231/HYPERNETS_CAL_HYPSTAR_221231_IRR_SWIR_v2.4.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221231/HYPERNETS_CAL_HYPSTAR_221231_IRR_SWIR_v2.4.nc new file mode 100644 index 00000000..0a24777d Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221231/HYPERNETS_CAL_HYPSTAR_221231_IRR_SWIR_v2.4.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221231/HYPERNETS_CAL_HYPSTAR_221231_IRR_v2.0.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221231/HYPERNETS_CAL_HYPSTAR_221231_IRR_v2.1.nc similarity index 96% rename from hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221231/HYPERNETS_CAL_HYPSTAR_221231_IRR_v2.0.nc rename to hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221231/HYPERNETS_CAL_HYPSTAR_221231_IRR_v2.1.nc index f2dfdf94..119330b6 100644 Binary files a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221231/HYPERNETS_CAL_HYPSTAR_221231_IRR_v2.0.nc and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221231/HYPERNETS_CAL_HYPSTAR_221231_IRR_v2.1.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221231/HYPERNETS_CAL_HYPSTAR_221231_IRR_v2.2.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221231/HYPERNETS_CAL_HYPSTAR_221231_IRR_v2.2.nc new file mode 100644 index 00000000..7b8900aa Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221231/HYPERNETS_CAL_HYPSTAR_221231_IRR_v2.2.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221231/HYPERNETS_CAL_HYPSTAR_221231_IRR_v2.3.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221231/HYPERNETS_CAL_HYPSTAR_221231_IRR_v2.3.nc new file mode 100644 index 00000000..09c9c695 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221231/HYPERNETS_CAL_HYPSTAR_221231_IRR_v2.3.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221231/HYPERNETS_CAL_HYPSTAR_221231_IRR_v2.4.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221231/HYPERNETS_CAL_HYPSTAR_221231_IRR_v2.4.nc new file mode 100644 index 00000000..2c9eb962 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221231/HYPERNETS_CAL_HYPSTAR_221231_IRR_v2.4.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221231/HYPERNETS_CAL_HYPSTAR_221231_RAD_SWIR_v2.0.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221231/HYPERNETS_CAL_HYPSTAR_221231_RAD_SWIR_v2.1.nc similarity index 63% rename from hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221231/HYPERNETS_CAL_HYPSTAR_221231_RAD_SWIR_v2.0.nc rename to hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221231/HYPERNETS_CAL_HYPSTAR_221231_RAD_SWIR_v2.1.nc index e5646eb8..031ee036 100644 Binary files a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221231/HYPERNETS_CAL_HYPSTAR_221231_RAD_SWIR_v2.0.nc and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221231/HYPERNETS_CAL_HYPSTAR_221231_RAD_SWIR_v2.1.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221231/HYPERNETS_CAL_HYPSTAR_221231_RAD_SWIR_v2.2.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221231/HYPERNETS_CAL_HYPSTAR_221231_RAD_SWIR_v2.2.nc new file mode 100644 index 00000000..a53de88c Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221231/HYPERNETS_CAL_HYPSTAR_221231_RAD_SWIR_v2.2.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221231/HYPERNETS_CAL_HYPSTAR_221231_RAD_SWIR_v2.3.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221231/HYPERNETS_CAL_HYPSTAR_221231_RAD_SWIR_v2.3.nc new file mode 100644 index 00000000..a54de895 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221231/HYPERNETS_CAL_HYPSTAR_221231_RAD_SWIR_v2.3.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221231/HYPERNETS_CAL_HYPSTAR_221231_RAD_SWIR_v2.4.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221231/HYPERNETS_CAL_HYPSTAR_221231_RAD_SWIR_v2.4.nc new file mode 100644 index 00000000..33c64366 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221231/HYPERNETS_CAL_HYPSTAR_221231_RAD_SWIR_v2.4.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221231/HYPERNETS_CAL_HYPSTAR_221231_RAD_v2.0.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221231/HYPERNETS_CAL_HYPSTAR_221231_RAD_v2.1.nc similarity index 94% rename from hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221231/HYPERNETS_CAL_HYPSTAR_221231_RAD_v2.0.nc rename to hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221231/HYPERNETS_CAL_HYPSTAR_221231_RAD_v2.1.nc index cb178c53..8a43d3fc 100644 Binary files a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221231/HYPERNETS_CAL_HYPSTAR_221231_RAD_v2.0.nc and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221231/HYPERNETS_CAL_HYPSTAR_221231_RAD_v2.1.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221231/HYPERNETS_CAL_HYPSTAR_221231_RAD_v2.2.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221231/HYPERNETS_CAL_HYPSTAR_221231_RAD_v2.2.nc new file mode 100644 index 00000000..70920412 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221231/HYPERNETS_CAL_HYPSTAR_221231_RAD_v2.2.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221231/HYPERNETS_CAL_HYPSTAR_221231_RAD_v2.3.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221231/HYPERNETS_CAL_HYPSTAR_221231_RAD_v2.3.nc new file mode 100644 index 00000000..85cd1982 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221231/HYPERNETS_CAL_HYPSTAR_221231_RAD_v2.3.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221231/HYPERNETS_CAL_HYPSTAR_221231_RAD_v2.4.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221231/HYPERNETS_CAL_HYPSTAR_221231_RAD_v2.4.nc new file mode 100644 index 00000000..9d0125a2 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221231/HYPERNETS_CAL_HYPSTAR_221231_RAD_v2.4.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221232/HYPERNETS_CAL_HYPSTAR_221232_IRR_SWIR_v2.0.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221232/HYPERNETS_CAL_HYPSTAR_221232_IRR_SWIR_v2.1.nc similarity index 63% rename from hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221232/HYPERNETS_CAL_HYPSTAR_221232_IRR_SWIR_v2.0.nc rename to hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221232/HYPERNETS_CAL_HYPSTAR_221232_IRR_SWIR_v2.1.nc index e133c9d5..9b8d59d7 100644 Binary files a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221232/HYPERNETS_CAL_HYPSTAR_221232_IRR_SWIR_v2.0.nc and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221232/HYPERNETS_CAL_HYPSTAR_221232_IRR_SWIR_v2.1.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221232/HYPERNETS_CAL_HYPSTAR_221232_IRR_SWIR_v2.2.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221232/HYPERNETS_CAL_HYPSTAR_221232_IRR_SWIR_v2.2.nc new file mode 100644 index 00000000..990225f9 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221232/HYPERNETS_CAL_HYPSTAR_221232_IRR_SWIR_v2.2.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221232/HYPERNETS_CAL_HYPSTAR_221232_IRR_SWIR_v2.3.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221232/HYPERNETS_CAL_HYPSTAR_221232_IRR_SWIR_v2.3.nc new file mode 100644 index 00000000..f5919c40 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221232/HYPERNETS_CAL_HYPSTAR_221232_IRR_SWIR_v2.3.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221232/HYPERNETS_CAL_HYPSTAR_221232_IRR_SWIR_v2.4.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221232/HYPERNETS_CAL_HYPSTAR_221232_IRR_SWIR_v2.4.nc new file mode 100644 index 00000000..4c44d3f2 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221232/HYPERNETS_CAL_HYPSTAR_221232_IRR_SWIR_v2.4.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221232/HYPERNETS_CAL_HYPSTAR_221232_IRR_v2.0.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221232/HYPERNETS_CAL_HYPSTAR_221232_IRR_v2.1.nc similarity index 96% rename from hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221232/HYPERNETS_CAL_HYPSTAR_221232_IRR_v2.0.nc rename to hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221232/HYPERNETS_CAL_HYPSTAR_221232_IRR_v2.1.nc index ce0c9002..2aa67f5d 100644 Binary files a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221232/HYPERNETS_CAL_HYPSTAR_221232_IRR_v2.0.nc and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221232/HYPERNETS_CAL_HYPSTAR_221232_IRR_v2.1.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221232/HYPERNETS_CAL_HYPSTAR_221232_IRR_v2.2.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221232/HYPERNETS_CAL_HYPSTAR_221232_IRR_v2.2.nc new file mode 100644 index 00000000..b5ea396b Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221232/HYPERNETS_CAL_HYPSTAR_221232_IRR_v2.2.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221232/HYPERNETS_CAL_HYPSTAR_221232_IRR_v2.3.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221232/HYPERNETS_CAL_HYPSTAR_221232_IRR_v2.3.nc new file mode 100644 index 00000000..56aad9aa Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221232/HYPERNETS_CAL_HYPSTAR_221232_IRR_v2.3.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221232/HYPERNETS_CAL_HYPSTAR_221232_IRR_v2.4.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221232/HYPERNETS_CAL_HYPSTAR_221232_IRR_v2.4.nc new file mode 100644 index 00000000..239ac434 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221232/HYPERNETS_CAL_HYPSTAR_221232_IRR_v2.4.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221232/HYPERNETS_CAL_HYPSTAR_221232_RAD_SWIR_v2.0.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221232/HYPERNETS_CAL_HYPSTAR_221232_RAD_SWIR_v2.1.nc similarity index 63% rename from hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221232/HYPERNETS_CAL_HYPSTAR_221232_RAD_SWIR_v2.0.nc rename to hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221232/HYPERNETS_CAL_HYPSTAR_221232_RAD_SWIR_v2.1.nc index e6ed291f..aef03e6e 100644 Binary files a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221232/HYPERNETS_CAL_HYPSTAR_221232_RAD_SWIR_v2.0.nc and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221232/HYPERNETS_CAL_HYPSTAR_221232_RAD_SWIR_v2.1.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221232/HYPERNETS_CAL_HYPSTAR_221232_RAD_SWIR_v2.2.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221232/HYPERNETS_CAL_HYPSTAR_221232_RAD_SWIR_v2.2.nc new file mode 100644 index 00000000..1ed9f097 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221232/HYPERNETS_CAL_HYPSTAR_221232_RAD_SWIR_v2.2.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221232/HYPERNETS_CAL_HYPSTAR_221232_RAD_SWIR_v2.3.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221232/HYPERNETS_CAL_HYPSTAR_221232_RAD_SWIR_v2.3.nc new file mode 100644 index 00000000..7a32b660 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221232/HYPERNETS_CAL_HYPSTAR_221232_RAD_SWIR_v2.3.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221232/HYPERNETS_CAL_HYPSTAR_221232_RAD_SWIR_v2.4.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221232/HYPERNETS_CAL_HYPSTAR_221232_RAD_SWIR_v2.4.nc new file mode 100644 index 00000000..ce34cc43 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221232/HYPERNETS_CAL_HYPSTAR_221232_RAD_SWIR_v2.4.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221232/HYPERNETS_CAL_HYPSTAR_221232_RAD_v2.0.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221232/HYPERNETS_CAL_HYPSTAR_221232_RAD_v2.1.nc similarity index 93% rename from hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221232/HYPERNETS_CAL_HYPSTAR_221232_RAD_v2.0.nc rename to hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221232/HYPERNETS_CAL_HYPSTAR_221232_RAD_v2.1.nc index 222f8731..601d2d6c 100644 Binary files a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221232/HYPERNETS_CAL_HYPSTAR_221232_RAD_v2.0.nc and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221232/HYPERNETS_CAL_HYPSTAR_221232_RAD_v2.1.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221232/HYPERNETS_CAL_HYPSTAR_221232_RAD_v2.2.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221232/HYPERNETS_CAL_HYPSTAR_221232_RAD_v2.2.nc new file mode 100644 index 00000000..77e7b6a7 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221232/HYPERNETS_CAL_HYPSTAR_221232_RAD_v2.2.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221232/HYPERNETS_CAL_HYPSTAR_221232_RAD_v2.3.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221232/HYPERNETS_CAL_HYPSTAR_221232_RAD_v2.3.nc new file mode 100644 index 00000000..7eb1808b Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221232/HYPERNETS_CAL_HYPSTAR_221232_RAD_v2.3.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221232/HYPERNETS_CAL_HYPSTAR_221232_RAD_v2.4.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221232/HYPERNETS_CAL_HYPSTAR_221232_RAD_v2.4.nc new file mode 100644 index 00000000..3bd6b47e Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221232/HYPERNETS_CAL_HYPSTAR_221232_RAD_v2.4.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221233/HYPERNETS_CAL_HYPSTAR_221233_IRR_SWIR_v2.0.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221233/HYPERNETS_CAL_HYPSTAR_221233_IRR_SWIR_v2.0.nc deleted file mode 100644 index b7f0b330..00000000 Binary files a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221233/HYPERNETS_CAL_HYPSTAR_221233_IRR_SWIR_v2.0.nc and /dev/null differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221233/HYPERNETS_CAL_HYPSTAR_221233_IRR_SWIR_v2.1.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221233/HYPERNETS_CAL_HYPSTAR_221233_IRR_SWIR_v2.1.nc new file mode 100644 index 00000000..7e216b0a Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221233/HYPERNETS_CAL_HYPSTAR_221233_IRR_SWIR_v2.1.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221233/HYPERNETS_CAL_HYPSTAR_221233_IRR_SWIR_v2.2.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221233/HYPERNETS_CAL_HYPSTAR_221233_IRR_SWIR_v2.2.nc new file mode 100644 index 00000000..6443a7e7 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221233/HYPERNETS_CAL_HYPSTAR_221233_IRR_SWIR_v2.2.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221233/HYPERNETS_CAL_HYPSTAR_221233_IRR_SWIR_v2.3.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221233/HYPERNETS_CAL_HYPSTAR_221233_IRR_SWIR_v2.3.nc new file mode 100644 index 00000000..275f7377 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221233/HYPERNETS_CAL_HYPSTAR_221233_IRR_SWIR_v2.3.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221233/HYPERNETS_CAL_HYPSTAR_221233_IRR_SWIR_v2.4.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221233/HYPERNETS_CAL_HYPSTAR_221233_IRR_SWIR_v2.4.nc new file mode 100644 index 00000000..21f04f39 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221233/HYPERNETS_CAL_HYPSTAR_221233_IRR_SWIR_v2.4.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221233/HYPERNETS_CAL_HYPSTAR_221233_IRR_v2.0.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221233/HYPERNETS_CAL_HYPSTAR_221233_IRR_v2.0.nc deleted file mode 100644 index 03ba254b..00000000 Binary files a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221233/HYPERNETS_CAL_HYPSTAR_221233_IRR_v2.0.nc and /dev/null differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221233/HYPERNETS_CAL_HYPSTAR_221233_IRR_v2.1.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221233/HYPERNETS_CAL_HYPSTAR_221233_IRR_v2.1.nc new file mode 100644 index 00000000..d20139a7 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221233/HYPERNETS_CAL_HYPSTAR_221233_IRR_v2.1.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221233/HYPERNETS_CAL_HYPSTAR_221233_IRR_v2.2.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221233/HYPERNETS_CAL_HYPSTAR_221233_IRR_v2.2.nc new file mode 100644 index 00000000..8bea81cc Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221233/HYPERNETS_CAL_HYPSTAR_221233_IRR_v2.2.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221233/HYPERNETS_CAL_HYPSTAR_221233_IRR_v2.3.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221233/HYPERNETS_CAL_HYPSTAR_221233_IRR_v2.3.nc new file mode 100644 index 00000000..9d7fc4b8 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221233/HYPERNETS_CAL_HYPSTAR_221233_IRR_v2.3.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221233/HYPERNETS_CAL_HYPSTAR_221233_IRR_v2.4.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221233/HYPERNETS_CAL_HYPSTAR_221233_IRR_v2.4.nc new file mode 100644 index 00000000..deede2d7 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221233/HYPERNETS_CAL_HYPSTAR_221233_IRR_v2.4.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221233/HYPERNETS_CAL_HYPSTAR_221233_RAD_SWIR_v2.0.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221233/HYPERNETS_CAL_HYPSTAR_221233_RAD_SWIR_v2.0.nc deleted file mode 100644 index 4398b5c1..00000000 Binary files a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221233/HYPERNETS_CAL_HYPSTAR_221233_RAD_SWIR_v2.0.nc and /dev/null differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221233/HYPERNETS_CAL_HYPSTAR_221233_RAD_SWIR_v2.1.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221233/HYPERNETS_CAL_HYPSTAR_221233_RAD_SWIR_v2.1.nc new file mode 100644 index 00000000..64acf6ac Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221233/HYPERNETS_CAL_HYPSTAR_221233_RAD_SWIR_v2.1.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221233/HYPERNETS_CAL_HYPSTAR_221233_RAD_SWIR_v2.2.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221233/HYPERNETS_CAL_HYPSTAR_221233_RAD_SWIR_v2.2.nc new file mode 100644 index 00000000..84c206dc Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221233/HYPERNETS_CAL_HYPSTAR_221233_RAD_SWIR_v2.2.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221233/HYPERNETS_CAL_HYPSTAR_221233_RAD_SWIR_v2.3.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221233/HYPERNETS_CAL_HYPSTAR_221233_RAD_SWIR_v2.3.nc new file mode 100644 index 00000000..c224498f Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221233/HYPERNETS_CAL_HYPSTAR_221233_RAD_SWIR_v2.3.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221233/HYPERNETS_CAL_HYPSTAR_221233_RAD_SWIR_v2.4.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221233/HYPERNETS_CAL_HYPSTAR_221233_RAD_SWIR_v2.4.nc new file mode 100644 index 00000000..bede9ab7 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221233/HYPERNETS_CAL_HYPSTAR_221233_RAD_SWIR_v2.4.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221233/HYPERNETS_CAL_HYPSTAR_221233_RAD_v2.0.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221233/HYPERNETS_CAL_HYPSTAR_221233_RAD_v2.0.nc deleted file mode 100644 index d293245b..00000000 Binary files a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221233/HYPERNETS_CAL_HYPSTAR_221233_RAD_v2.0.nc and /dev/null differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221233/HYPERNETS_CAL_HYPSTAR_221233_RAD_v2.1.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221233/HYPERNETS_CAL_HYPSTAR_221233_RAD_v2.1.nc new file mode 100644 index 00000000..a0e688d8 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221233/HYPERNETS_CAL_HYPSTAR_221233_RAD_v2.1.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221233/HYPERNETS_CAL_HYPSTAR_221233_RAD_v2.2.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221233/HYPERNETS_CAL_HYPSTAR_221233_RAD_v2.2.nc new file mode 100644 index 00000000..d66fb1b3 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221233/HYPERNETS_CAL_HYPSTAR_221233_RAD_v2.2.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221233/HYPERNETS_CAL_HYPSTAR_221233_RAD_v2.3.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221233/HYPERNETS_CAL_HYPSTAR_221233_RAD_v2.3.nc new file mode 100644 index 00000000..bdeab3b0 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221233/HYPERNETS_CAL_HYPSTAR_221233_RAD_v2.3.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221233/HYPERNETS_CAL_HYPSTAR_221233_RAD_v2.4.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221233/HYPERNETS_CAL_HYPSTAR_221233_RAD_v2.4.nc new file mode 100644 index 00000000..021d6aa8 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221233/HYPERNETS_CAL_HYPSTAR_221233_RAD_v2.4.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221461/HYPERNETS_CAL_HYPSTAR_221461_IRR_SWIR_v2.0.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221461/HYPERNETS_CAL_HYPSTAR_221461_IRR_SWIR_v2.0.nc deleted file mode 100644 index 37d07ada..00000000 Binary files a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221461/HYPERNETS_CAL_HYPSTAR_221461_IRR_SWIR_v2.0.nc and /dev/null differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221461/HYPERNETS_CAL_HYPSTAR_221461_IRR_SWIR_v2.1.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221461/HYPERNETS_CAL_HYPSTAR_221461_IRR_SWIR_v2.1.nc new file mode 100644 index 00000000..45a0f9cf Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221461/HYPERNETS_CAL_HYPSTAR_221461_IRR_SWIR_v2.1.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221461/HYPERNETS_CAL_HYPSTAR_221461_IRR_SWIR_v2.2.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221461/HYPERNETS_CAL_HYPSTAR_221461_IRR_SWIR_v2.2.nc new file mode 100644 index 00000000..5031b288 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221461/HYPERNETS_CAL_HYPSTAR_221461_IRR_SWIR_v2.2.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221461/HYPERNETS_CAL_HYPSTAR_221461_IRR_SWIR_v2.3.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221461/HYPERNETS_CAL_HYPSTAR_221461_IRR_SWIR_v2.3.nc new file mode 100644 index 00000000..13d4b6da Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221461/HYPERNETS_CAL_HYPSTAR_221461_IRR_SWIR_v2.3.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221461/HYPERNETS_CAL_HYPSTAR_221461_IRR_SWIR_v2.4.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221461/HYPERNETS_CAL_HYPSTAR_221461_IRR_SWIR_v2.4.nc new file mode 100644 index 00000000..5001ea9d Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221461/HYPERNETS_CAL_HYPSTAR_221461_IRR_SWIR_v2.4.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221461/HYPERNETS_CAL_HYPSTAR_221461_IRR_v2.0.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221461/HYPERNETS_CAL_HYPSTAR_221461_IRR_v2.0.nc deleted file mode 100644 index 162fb51a..00000000 Binary files a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221461/HYPERNETS_CAL_HYPSTAR_221461_IRR_v2.0.nc and /dev/null differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221461/HYPERNETS_CAL_HYPSTAR_221461_IRR_v2.1.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221461/HYPERNETS_CAL_HYPSTAR_221461_IRR_v2.1.nc new file mode 100644 index 00000000..1a8a251a Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221461/HYPERNETS_CAL_HYPSTAR_221461_IRR_v2.1.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221461/HYPERNETS_CAL_HYPSTAR_221461_IRR_v2.2.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221461/HYPERNETS_CAL_HYPSTAR_221461_IRR_v2.2.nc new file mode 100644 index 00000000..316e1ead Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221461/HYPERNETS_CAL_HYPSTAR_221461_IRR_v2.2.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221461/HYPERNETS_CAL_HYPSTAR_221461_IRR_v2.3.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221461/HYPERNETS_CAL_HYPSTAR_221461_IRR_v2.3.nc new file mode 100644 index 00000000..395736e5 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221461/HYPERNETS_CAL_HYPSTAR_221461_IRR_v2.3.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221461/HYPERNETS_CAL_HYPSTAR_221461_IRR_v2.4.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221461/HYPERNETS_CAL_HYPSTAR_221461_IRR_v2.4.nc new file mode 100644 index 00000000..4c78f230 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221461/HYPERNETS_CAL_HYPSTAR_221461_IRR_v2.4.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221461/HYPERNETS_CAL_HYPSTAR_221461_RAD_SWIR_v2.0.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221461/HYPERNETS_CAL_HYPSTAR_221461_RAD_SWIR_v2.0.nc deleted file mode 100644 index 65e3a415..00000000 Binary files a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221461/HYPERNETS_CAL_HYPSTAR_221461_RAD_SWIR_v2.0.nc and /dev/null differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221461/HYPERNETS_CAL_HYPSTAR_221461_RAD_SWIR_v2.1.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221461/HYPERNETS_CAL_HYPSTAR_221461_RAD_SWIR_v2.1.nc new file mode 100644 index 00000000..0cf993fe Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221461/HYPERNETS_CAL_HYPSTAR_221461_RAD_SWIR_v2.1.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221461/HYPERNETS_CAL_HYPSTAR_221461_RAD_SWIR_v2.2.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221461/HYPERNETS_CAL_HYPSTAR_221461_RAD_SWIR_v2.2.nc new file mode 100644 index 00000000..9b7b94c2 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221461/HYPERNETS_CAL_HYPSTAR_221461_RAD_SWIR_v2.2.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221461/HYPERNETS_CAL_HYPSTAR_221461_RAD_SWIR_v2.3.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221461/HYPERNETS_CAL_HYPSTAR_221461_RAD_SWIR_v2.3.nc new file mode 100644 index 00000000..55f8a0b0 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221461/HYPERNETS_CAL_HYPSTAR_221461_RAD_SWIR_v2.3.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221461/HYPERNETS_CAL_HYPSTAR_221461_RAD_SWIR_v2.4.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221461/HYPERNETS_CAL_HYPSTAR_221461_RAD_SWIR_v2.4.nc new file mode 100644 index 00000000..0c39fdda Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221461/HYPERNETS_CAL_HYPSTAR_221461_RAD_SWIR_v2.4.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221461/HYPERNETS_CAL_HYPSTAR_221461_RAD_v2.0.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221461/HYPERNETS_CAL_HYPSTAR_221461_RAD_v2.0.nc deleted file mode 100644 index 1780d29e..00000000 Binary files a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221461/HYPERNETS_CAL_HYPSTAR_221461_RAD_v2.0.nc and /dev/null differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221461/HYPERNETS_CAL_HYPSTAR_221461_RAD_v2.1.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221461/HYPERNETS_CAL_HYPSTAR_221461_RAD_v2.1.nc new file mode 100644 index 00000000..fefe2fcf Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221461/HYPERNETS_CAL_HYPSTAR_221461_RAD_v2.1.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221461/HYPERNETS_CAL_HYPSTAR_221461_RAD_v2.2.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221461/HYPERNETS_CAL_HYPSTAR_221461_RAD_v2.2.nc new file mode 100644 index 00000000..e99d2c48 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221461/HYPERNETS_CAL_HYPSTAR_221461_RAD_v2.2.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221461/HYPERNETS_CAL_HYPSTAR_221461_RAD_v2.3.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221461/HYPERNETS_CAL_HYPSTAR_221461_RAD_v2.3.nc new file mode 100644 index 00000000..366f4654 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221461/HYPERNETS_CAL_HYPSTAR_221461_RAD_v2.3.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221461/HYPERNETS_CAL_HYPSTAR_221461_RAD_v2.4.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221461/HYPERNETS_CAL_HYPSTAR_221461_RAD_v2.4.nc new file mode 100644 index 00000000..93fa77d0 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_221461/HYPERNETS_CAL_HYPSTAR_221461_RAD_v2.4.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222311/HYPERNETS_CAL_HYPSTAR_222311_IRR_SWIR_v2.0.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222311/HYPERNETS_CAL_HYPSTAR_222311_IRR_SWIR_v2.1.nc similarity index 64% rename from hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222311/HYPERNETS_CAL_HYPSTAR_222311_IRR_SWIR_v2.0.nc rename to hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222311/HYPERNETS_CAL_HYPSTAR_222311_IRR_SWIR_v2.1.nc index 7e4df303..0ef93385 100644 Binary files a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222311/HYPERNETS_CAL_HYPSTAR_222311_IRR_SWIR_v2.0.nc and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222311/HYPERNETS_CAL_HYPSTAR_222311_IRR_SWIR_v2.1.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222311/HYPERNETS_CAL_HYPSTAR_222311_IRR_SWIR_v2.2.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222311/HYPERNETS_CAL_HYPSTAR_222311_IRR_SWIR_v2.2.nc new file mode 100644 index 00000000..897c2081 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222311/HYPERNETS_CAL_HYPSTAR_222311_IRR_SWIR_v2.2.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222311/HYPERNETS_CAL_HYPSTAR_222311_IRR_SWIR_v2.3.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222311/HYPERNETS_CAL_HYPSTAR_222311_IRR_SWIR_v2.3.nc new file mode 100644 index 00000000..1502f6f4 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222311/HYPERNETS_CAL_HYPSTAR_222311_IRR_SWIR_v2.3.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222311/HYPERNETS_CAL_HYPSTAR_222311_IRR_SWIR_v2.4.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222311/HYPERNETS_CAL_HYPSTAR_222311_IRR_SWIR_v2.4.nc new file mode 100644 index 00000000..92a1c0b2 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222311/HYPERNETS_CAL_HYPSTAR_222311_IRR_SWIR_v2.4.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222311/HYPERNETS_CAL_HYPSTAR_222311_IRR_v2.0.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222311/HYPERNETS_CAL_HYPSTAR_222311_IRR_v2.1.nc similarity index 97% rename from hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222311/HYPERNETS_CAL_HYPSTAR_222311_IRR_v2.0.nc rename to hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222311/HYPERNETS_CAL_HYPSTAR_222311_IRR_v2.1.nc index 32b6129e..7e045bd0 100644 Binary files a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222311/HYPERNETS_CAL_HYPSTAR_222311_IRR_v2.0.nc and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222311/HYPERNETS_CAL_HYPSTAR_222311_IRR_v2.1.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222311/HYPERNETS_CAL_HYPSTAR_222311_IRR_v2.2.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222311/HYPERNETS_CAL_HYPSTAR_222311_IRR_v2.2.nc new file mode 100644 index 00000000..27df6382 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222311/HYPERNETS_CAL_HYPSTAR_222311_IRR_v2.2.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222311/HYPERNETS_CAL_HYPSTAR_222311_IRR_v2.3.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222311/HYPERNETS_CAL_HYPSTAR_222311_IRR_v2.3.nc new file mode 100644 index 00000000..240af46c Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222311/HYPERNETS_CAL_HYPSTAR_222311_IRR_v2.3.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222311/HYPERNETS_CAL_HYPSTAR_222311_IRR_v2.4.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222311/HYPERNETS_CAL_HYPSTAR_222311_IRR_v2.4.nc new file mode 100644 index 00000000..51b0324e Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222311/HYPERNETS_CAL_HYPSTAR_222311_IRR_v2.4.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222311/HYPERNETS_CAL_HYPSTAR_222311_RAD_SWIR_v2.0.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222311/HYPERNETS_CAL_HYPSTAR_222311_RAD_SWIR_v2.1.nc similarity index 64% rename from hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222311/HYPERNETS_CAL_HYPSTAR_222311_RAD_SWIR_v2.0.nc rename to hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222311/HYPERNETS_CAL_HYPSTAR_222311_RAD_SWIR_v2.1.nc index c7864482..b93a24c1 100644 Binary files a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222311/HYPERNETS_CAL_HYPSTAR_222311_RAD_SWIR_v2.0.nc and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222311/HYPERNETS_CAL_HYPSTAR_222311_RAD_SWIR_v2.1.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222311/HYPERNETS_CAL_HYPSTAR_222311_RAD_SWIR_v2.2.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222311/HYPERNETS_CAL_HYPSTAR_222311_RAD_SWIR_v2.2.nc new file mode 100644 index 00000000..f8310705 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222311/HYPERNETS_CAL_HYPSTAR_222311_RAD_SWIR_v2.2.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222311/HYPERNETS_CAL_HYPSTAR_222311_RAD_SWIR_v2.3.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222311/HYPERNETS_CAL_HYPSTAR_222311_RAD_SWIR_v2.3.nc new file mode 100644 index 00000000..976b270f Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222311/HYPERNETS_CAL_HYPSTAR_222311_RAD_SWIR_v2.3.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222311/HYPERNETS_CAL_HYPSTAR_222311_RAD_SWIR_v2.4.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222311/HYPERNETS_CAL_HYPSTAR_222311_RAD_SWIR_v2.4.nc new file mode 100644 index 00000000..1ac61be5 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222311/HYPERNETS_CAL_HYPSTAR_222311_RAD_SWIR_v2.4.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222311/HYPERNETS_CAL_HYPSTAR_222311_RAD_v2.0.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222311/HYPERNETS_CAL_HYPSTAR_222311_RAD_v2.1.nc similarity index 96% rename from hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222311/HYPERNETS_CAL_HYPSTAR_222311_RAD_v2.0.nc rename to hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222311/HYPERNETS_CAL_HYPSTAR_222311_RAD_v2.1.nc index e3411897..44a02108 100644 Binary files a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222311/HYPERNETS_CAL_HYPSTAR_222311_RAD_v2.0.nc and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222311/HYPERNETS_CAL_HYPSTAR_222311_RAD_v2.1.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222311/HYPERNETS_CAL_HYPSTAR_222311_RAD_v2.2.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222311/HYPERNETS_CAL_HYPSTAR_222311_RAD_v2.2.nc new file mode 100644 index 00000000..91dad514 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222311/HYPERNETS_CAL_HYPSTAR_222311_RAD_v2.2.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222311/HYPERNETS_CAL_HYPSTAR_222311_RAD_v2.3.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222311/HYPERNETS_CAL_HYPSTAR_222311_RAD_v2.3.nc new file mode 100644 index 00000000..f23afb71 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222311/HYPERNETS_CAL_HYPSTAR_222311_RAD_v2.3.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222311/HYPERNETS_CAL_HYPSTAR_222311_RAD_v2.4.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222311/HYPERNETS_CAL_HYPSTAR_222311_RAD_v2.4.nc new file mode 100644 index 00000000..1440e6c7 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222311/HYPERNETS_CAL_HYPSTAR_222311_RAD_v2.4.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222312/HYPERNETS_CAL_HYPSTAR_222312_IRR_SWIR_v2.0.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222312/HYPERNETS_CAL_HYPSTAR_222312_IRR_SWIR_v2.1.nc similarity index 62% rename from hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222312/HYPERNETS_CAL_HYPSTAR_222312_IRR_SWIR_v2.0.nc rename to hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222312/HYPERNETS_CAL_HYPSTAR_222312_IRR_SWIR_v2.1.nc index e9846bef..9ba8d55e 100644 Binary files a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222312/HYPERNETS_CAL_HYPSTAR_222312_IRR_SWIR_v2.0.nc and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222312/HYPERNETS_CAL_HYPSTAR_222312_IRR_SWIR_v2.1.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222312/HYPERNETS_CAL_HYPSTAR_222312_IRR_SWIR_v2.2.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222312/HYPERNETS_CAL_HYPSTAR_222312_IRR_SWIR_v2.2.nc new file mode 100644 index 00000000..3049d665 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222312/HYPERNETS_CAL_HYPSTAR_222312_IRR_SWIR_v2.2.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222312/HYPERNETS_CAL_HYPSTAR_222312_IRR_SWIR_v2.3.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222312/HYPERNETS_CAL_HYPSTAR_222312_IRR_SWIR_v2.3.nc new file mode 100644 index 00000000..c1b8dd1c Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222312/HYPERNETS_CAL_HYPSTAR_222312_IRR_SWIR_v2.3.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222312/HYPERNETS_CAL_HYPSTAR_222312_IRR_SWIR_v2.4.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222312/HYPERNETS_CAL_HYPSTAR_222312_IRR_SWIR_v2.4.nc new file mode 100644 index 00000000..000f7513 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222312/HYPERNETS_CAL_HYPSTAR_222312_IRR_SWIR_v2.4.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222312/HYPERNETS_CAL_HYPSTAR_222312_IRR_v2.0.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222312/HYPERNETS_CAL_HYPSTAR_222312_IRR_v2.1.nc similarity index 97% rename from hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222312/HYPERNETS_CAL_HYPSTAR_222312_IRR_v2.0.nc rename to hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222312/HYPERNETS_CAL_HYPSTAR_222312_IRR_v2.1.nc index 40aeae3e..186215d8 100644 Binary files a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222312/HYPERNETS_CAL_HYPSTAR_222312_IRR_v2.0.nc and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222312/HYPERNETS_CAL_HYPSTAR_222312_IRR_v2.1.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222312/HYPERNETS_CAL_HYPSTAR_222312_IRR_v2.2.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222312/HYPERNETS_CAL_HYPSTAR_222312_IRR_v2.2.nc new file mode 100644 index 00000000..cc5d5e03 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222312/HYPERNETS_CAL_HYPSTAR_222312_IRR_v2.2.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222312/HYPERNETS_CAL_HYPSTAR_222312_IRR_v2.3.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222312/HYPERNETS_CAL_HYPSTAR_222312_IRR_v2.3.nc new file mode 100644 index 00000000..7b04bc9e Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222312/HYPERNETS_CAL_HYPSTAR_222312_IRR_v2.3.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222312/HYPERNETS_CAL_HYPSTAR_222312_IRR_v2.4.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222312/HYPERNETS_CAL_HYPSTAR_222312_IRR_v2.4.nc new file mode 100644 index 00000000..fffe4652 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222312/HYPERNETS_CAL_HYPSTAR_222312_IRR_v2.4.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222312/HYPERNETS_CAL_HYPSTAR_222312_RAD_SWIR_v2.0.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222312/HYPERNETS_CAL_HYPSTAR_222312_RAD_SWIR_v2.1.nc similarity index 63% rename from hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222312/HYPERNETS_CAL_HYPSTAR_222312_RAD_SWIR_v2.0.nc rename to hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222312/HYPERNETS_CAL_HYPSTAR_222312_RAD_SWIR_v2.1.nc index b5309d3f..e25d9bc2 100644 Binary files a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222312/HYPERNETS_CAL_HYPSTAR_222312_RAD_SWIR_v2.0.nc and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222312/HYPERNETS_CAL_HYPSTAR_222312_RAD_SWIR_v2.1.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222312/HYPERNETS_CAL_HYPSTAR_222312_RAD_SWIR_v2.2.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222312/HYPERNETS_CAL_HYPSTAR_222312_RAD_SWIR_v2.2.nc new file mode 100644 index 00000000..115d40fd Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222312/HYPERNETS_CAL_HYPSTAR_222312_RAD_SWIR_v2.2.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222312/HYPERNETS_CAL_HYPSTAR_222312_RAD_SWIR_v2.3.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222312/HYPERNETS_CAL_HYPSTAR_222312_RAD_SWIR_v2.3.nc new file mode 100644 index 00000000..7ea43ab8 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222312/HYPERNETS_CAL_HYPSTAR_222312_RAD_SWIR_v2.3.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222312/HYPERNETS_CAL_HYPSTAR_222312_RAD_SWIR_v2.4.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222312/HYPERNETS_CAL_HYPSTAR_222312_RAD_SWIR_v2.4.nc new file mode 100644 index 00000000..5b7fc7e1 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222312/HYPERNETS_CAL_HYPSTAR_222312_RAD_SWIR_v2.4.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222312/HYPERNETS_CAL_HYPSTAR_222312_RAD_v2.0.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222312/HYPERNETS_CAL_HYPSTAR_222312_RAD_v2.1.nc similarity index 96% rename from hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222312/HYPERNETS_CAL_HYPSTAR_222312_RAD_v2.0.nc rename to hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222312/HYPERNETS_CAL_HYPSTAR_222312_RAD_v2.1.nc index 417ff1ca..93646359 100644 Binary files a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222312/HYPERNETS_CAL_HYPSTAR_222312_RAD_v2.0.nc and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222312/HYPERNETS_CAL_HYPSTAR_222312_RAD_v2.1.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222312/HYPERNETS_CAL_HYPSTAR_222312_RAD_v2.2.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222312/HYPERNETS_CAL_HYPSTAR_222312_RAD_v2.2.nc new file mode 100644 index 00000000..53c61ec4 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222312/HYPERNETS_CAL_HYPSTAR_222312_RAD_v2.2.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222312/HYPERNETS_CAL_HYPSTAR_222312_RAD_v2.3.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222312/HYPERNETS_CAL_HYPSTAR_222312_RAD_v2.3.nc new file mode 100644 index 00000000..7869acd2 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222312/HYPERNETS_CAL_HYPSTAR_222312_RAD_v2.3.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222312/HYPERNETS_CAL_HYPSTAR_222312_RAD_v2.4.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222312/HYPERNETS_CAL_HYPSTAR_222312_RAD_v2.4.nc new file mode 100644 index 00000000..10190487 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222312/HYPERNETS_CAL_HYPSTAR_222312_RAD_v2.4.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222313/HYPERNETS_CAL_HYPSTAR_222313_IRR_SWIR_v2.0.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222313/HYPERNETS_CAL_HYPSTAR_222313_IRR_SWIR_v2.1.nc similarity index 64% rename from hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222313/HYPERNETS_CAL_HYPSTAR_222313_IRR_SWIR_v2.0.nc rename to hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222313/HYPERNETS_CAL_HYPSTAR_222313_IRR_SWIR_v2.1.nc index 28d544a8..dfbb3114 100644 Binary files a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222313/HYPERNETS_CAL_HYPSTAR_222313_IRR_SWIR_v2.0.nc and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222313/HYPERNETS_CAL_HYPSTAR_222313_IRR_SWIR_v2.1.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222313/HYPERNETS_CAL_HYPSTAR_222313_IRR_SWIR_v2.2.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222313/HYPERNETS_CAL_HYPSTAR_222313_IRR_SWIR_v2.2.nc new file mode 100644 index 00000000..ab3ca02b Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222313/HYPERNETS_CAL_HYPSTAR_222313_IRR_SWIR_v2.2.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222313/HYPERNETS_CAL_HYPSTAR_222313_IRR_SWIR_v2.3.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222313/HYPERNETS_CAL_HYPSTAR_222313_IRR_SWIR_v2.3.nc new file mode 100644 index 00000000..46a64e18 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222313/HYPERNETS_CAL_HYPSTAR_222313_IRR_SWIR_v2.3.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222313/HYPERNETS_CAL_HYPSTAR_222313_IRR_SWIR_v2.4.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222313/HYPERNETS_CAL_HYPSTAR_222313_IRR_SWIR_v2.4.nc new file mode 100644 index 00000000..bd596eb6 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222313/HYPERNETS_CAL_HYPSTAR_222313_IRR_SWIR_v2.4.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222313/HYPERNETS_CAL_HYPSTAR_222313_IRR_v2.0.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222313/HYPERNETS_CAL_HYPSTAR_222313_IRR_v2.1.nc similarity index 97% rename from hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222313/HYPERNETS_CAL_HYPSTAR_222313_IRR_v2.0.nc rename to hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222313/HYPERNETS_CAL_HYPSTAR_222313_IRR_v2.1.nc index 37661754..d3470cbc 100644 Binary files a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222313/HYPERNETS_CAL_HYPSTAR_222313_IRR_v2.0.nc and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222313/HYPERNETS_CAL_HYPSTAR_222313_IRR_v2.1.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222313/HYPERNETS_CAL_HYPSTAR_222313_IRR_v2.2.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222313/HYPERNETS_CAL_HYPSTAR_222313_IRR_v2.2.nc new file mode 100644 index 00000000..c494a5f9 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222313/HYPERNETS_CAL_HYPSTAR_222313_IRR_v2.2.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222313/HYPERNETS_CAL_HYPSTAR_222313_IRR_v2.3.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222313/HYPERNETS_CAL_HYPSTAR_222313_IRR_v2.3.nc new file mode 100644 index 00000000..03e57fa6 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222313/HYPERNETS_CAL_HYPSTAR_222313_IRR_v2.3.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222313/HYPERNETS_CAL_HYPSTAR_222313_IRR_v2.4.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222313/HYPERNETS_CAL_HYPSTAR_222313_IRR_v2.4.nc new file mode 100644 index 00000000..88f41a74 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222313/HYPERNETS_CAL_HYPSTAR_222313_IRR_v2.4.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222313/HYPERNETS_CAL_HYPSTAR_222313_RAD_SWIR_v2.0.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222313/HYPERNETS_CAL_HYPSTAR_222313_RAD_SWIR_v2.1.nc similarity index 63% rename from hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222313/HYPERNETS_CAL_HYPSTAR_222313_RAD_SWIR_v2.0.nc rename to hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222313/HYPERNETS_CAL_HYPSTAR_222313_RAD_SWIR_v2.1.nc index c83c2d92..54d5e4de 100644 Binary files a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222313/HYPERNETS_CAL_HYPSTAR_222313_RAD_SWIR_v2.0.nc and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222313/HYPERNETS_CAL_HYPSTAR_222313_RAD_SWIR_v2.1.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222313/HYPERNETS_CAL_HYPSTAR_222313_RAD_SWIR_v2.2.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222313/HYPERNETS_CAL_HYPSTAR_222313_RAD_SWIR_v2.2.nc new file mode 100644 index 00000000..a493c5e2 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222313/HYPERNETS_CAL_HYPSTAR_222313_RAD_SWIR_v2.2.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222313/HYPERNETS_CAL_HYPSTAR_222313_RAD_SWIR_v2.3.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222313/HYPERNETS_CAL_HYPSTAR_222313_RAD_SWIR_v2.3.nc new file mode 100644 index 00000000..9b6ca251 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222313/HYPERNETS_CAL_HYPSTAR_222313_RAD_SWIR_v2.3.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222313/HYPERNETS_CAL_HYPSTAR_222313_RAD_SWIR_v2.4.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222313/HYPERNETS_CAL_HYPSTAR_222313_RAD_SWIR_v2.4.nc new file mode 100644 index 00000000..2905f7f1 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222313/HYPERNETS_CAL_HYPSTAR_222313_RAD_SWIR_v2.4.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222313/HYPERNETS_CAL_HYPSTAR_222313_RAD_v2.0.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222313/HYPERNETS_CAL_HYPSTAR_222313_RAD_v2.1.nc similarity index 97% rename from hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222313/HYPERNETS_CAL_HYPSTAR_222313_RAD_v2.0.nc rename to hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222313/HYPERNETS_CAL_HYPSTAR_222313_RAD_v2.1.nc index 112ab2ca..622eb21c 100644 Binary files a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222313/HYPERNETS_CAL_HYPSTAR_222313_RAD_v2.0.nc and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222313/HYPERNETS_CAL_HYPSTAR_222313_RAD_v2.1.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222313/HYPERNETS_CAL_HYPSTAR_222313_RAD_v2.2.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222313/HYPERNETS_CAL_HYPSTAR_222313_RAD_v2.2.nc new file mode 100644 index 00000000..435f9948 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222313/HYPERNETS_CAL_HYPSTAR_222313_RAD_v2.2.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222313/HYPERNETS_CAL_HYPSTAR_222313_RAD_v2.3.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222313/HYPERNETS_CAL_HYPSTAR_222313_RAD_v2.3.nc new file mode 100644 index 00000000..420a6849 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222313/HYPERNETS_CAL_HYPSTAR_222313_RAD_v2.3.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222313/HYPERNETS_CAL_HYPSTAR_222313_RAD_v2.4.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222313/HYPERNETS_CAL_HYPSTAR_222313_RAD_v2.4.nc new file mode 100644 index 00000000..1321df44 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222313/HYPERNETS_CAL_HYPSTAR_222313_RAD_v2.4.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222314/HYPERNETS_CAL_HYPSTAR_222314_IRR_SWIR_v2.0.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222314/HYPERNETS_CAL_HYPSTAR_222314_IRR_SWIR_v2.1.nc similarity index 66% rename from hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222314/HYPERNETS_CAL_HYPSTAR_222314_IRR_SWIR_v2.0.nc rename to hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222314/HYPERNETS_CAL_HYPSTAR_222314_IRR_SWIR_v2.1.nc index eef93a08..9084a3fc 100644 Binary files a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222314/HYPERNETS_CAL_HYPSTAR_222314_IRR_SWIR_v2.0.nc and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222314/HYPERNETS_CAL_HYPSTAR_222314_IRR_SWIR_v2.1.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222314/HYPERNETS_CAL_HYPSTAR_222314_IRR_SWIR_v2.2.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222314/HYPERNETS_CAL_HYPSTAR_222314_IRR_SWIR_v2.2.nc new file mode 100644 index 00000000..4bb76bf5 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222314/HYPERNETS_CAL_HYPSTAR_222314_IRR_SWIR_v2.2.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222314/HYPERNETS_CAL_HYPSTAR_222314_IRR_SWIR_v2.3.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222314/HYPERNETS_CAL_HYPSTAR_222314_IRR_SWIR_v2.3.nc new file mode 100644 index 00000000..27606c27 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222314/HYPERNETS_CAL_HYPSTAR_222314_IRR_SWIR_v2.3.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222314/HYPERNETS_CAL_HYPSTAR_222314_IRR_SWIR_v2.4.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222314/HYPERNETS_CAL_HYPSTAR_222314_IRR_SWIR_v2.4.nc new file mode 100644 index 00000000..95a9b8c1 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222314/HYPERNETS_CAL_HYPSTAR_222314_IRR_SWIR_v2.4.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222314/HYPERNETS_CAL_HYPSTAR_222314_IRR_v2.0.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222314/HYPERNETS_CAL_HYPSTAR_222314_IRR_v2.1.nc similarity index 98% rename from hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222314/HYPERNETS_CAL_HYPSTAR_222314_IRR_v2.0.nc rename to hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222314/HYPERNETS_CAL_HYPSTAR_222314_IRR_v2.1.nc index e5d98b8b..168aeeff 100644 Binary files a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222314/HYPERNETS_CAL_HYPSTAR_222314_IRR_v2.0.nc and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222314/HYPERNETS_CAL_HYPSTAR_222314_IRR_v2.1.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222314/HYPERNETS_CAL_HYPSTAR_222314_IRR_v2.2.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222314/HYPERNETS_CAL_HYPSTAR_222314_IRR_v2.2.nc new file mode 100644 index 00000000..e4781495 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222314/HYPERNETS_CAL_HYPSTAR_222314_IRR_v2.2.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222314/HYPERNETS_CAL_HYPSTAR_222314_IRR_v2.3.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222314/HYPERNETS_CAL_HYPSTAR_222314_IRR_v2.3.nc new file mode 100644 index 00000000..1aef0d44 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222314/HYPERNETS_CAL_HYPSTAR_222314_IRR_v2.3.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222314/HYPERNETS_CAL_HYPSTAR_222314_IRR_v2.4.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222314/HYPERNETS_CAL_HYPSTAR_222314_IRR_v2.4.nc new file mode 100644 index 00000000..4884e688 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222314/HYPERNETS_CAL_HYPSTAR_222314_IRR_v2.4.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222314/HYPERNETS_CAL_HYPSTAR_222314_RAD_SWIR_v2.0.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222314/HYPERNETS_CAL_HYPSTAR_222314_RAD_SWIR_v2.1.nc similarity index 66% rename from hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222314/HYPERNETS_CAL_HYPSTAR_222314_RAD_SWIR_v2.0.nc rename to hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222314/HYPERNETS_CAL_HYPSTAR_222314_RAD_SWIR_v2.1.nc index f0b309d7..102a4859 100644 Binary files a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222314/HYPERNETS_CAL_HYPSTAR_222314_RAD_SWIR_v2.0.nc and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222314/HYPERNETS_CAL_HYPSTAR_222314_RAD_SWIR_v2.1.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222314/HYPERNETS_CAL_HYPSTAR_222314_RAD_SWIR_v2.2.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222314/HYPERNETS_CAL_HYPSTAR_222314_RAD_SWIR_v2.2.nc new file mode 100644 index 00000000..13395b67 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222314/HYPERNETS_CAL_HYPSTAR_222314_RAD_SWIR_v2.2.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222314/HYPERNETS_CAL_HYPSTAR_222314_RAD_SWIR_v2.3.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222314/HYPERNETS_CAL_HYPSTAR_222314_RAD_SWIR_v2.3.nc new file mode 100644 index 00000000..27bbd831 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222314/HYPERNETS_CAL_HYPSTAR_222314_RAD_SWIR_v2.3.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222314/HYPERNETS_CAL_HYPSTAR_222314_RAD_SWIR_v2.4.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222314/HYPERNETS_CAL_HYPSTAR_222314_RAD_SWIR_v2.4.nc new file mode 100644 index 00000000..61c231c0 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222314/HYPERNETS_CAL_HYPSTAR_222314_RAD_SWIR_v2.4.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222314/HYPERNETS_CAL_HYPSTAR_222314_RAD_v2.0.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222314/HYPERNETS_CAL_HYPSTAR_222314_RAD_v2.1.nc similarity index 98% rename from hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222314/HYPERNETS_CAL_HYPSTAR_222314_RAD_v2.0.nc rename to hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222314/HYPERNETS_CAL_HYPSTAR_222314_RAD_v2.1.nc index ac6fe3c1..e6675fff 100644 Binary files a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222314/HYPERNETS_CAL_HYPSTAR_222314_RAD_v2.0.nc and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222314/HYPERNETS_CAL_HYPSTAR_222314_RAD_v2.1.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222314/HYPERNETS_CAL_HYPSTAR_222314_RAD_v2.2.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222314/HYPERNETS_CAL_HYPSTAR_222314_RAD_v2.2.nc new file mode 100644 index 00000000..6df61c4b Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222314/HYPERNETS_CAL_HYPSTAR_222314_RAD_v2.2.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222314/HYPERNETS_CAL_HYPSTAR_222314_RAD_v2.3.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222314/HYPERNETS_CAL_HYPSTAR_222314_RAD_v2.3.nc new file mode 100644 index 00000000..d8a74d8f Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222314/HYPERNETS_CAL_HYPSTAR_222314_RAD_v2.3.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222314/HYPERNETS_CAL_HYPSTAR_222314_RAD_v2.4.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222314/HYPERNETS_CAL_HYPSTAR_222314_RAD_v2.4.nc new file mode 100644 index 00000000..690bbf0c Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222314/HYPERNETS_CAL_HYPSTAR_222314_RAD_v2.4.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222315/HYPERNETS_CAL_HYPSTAR_222315_IRR_SWIR_v2.0.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222315/HYPERNETS_CAL_HYPSTAR_222315_IRR_SWIR_v2.1.nc similarity index 64% rename from hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222315/HYPERNETS_CAL_HYPSTAR_222315_IRR_SWIR_v2.0.nc rename to hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222315/HYPERNETS_CAL_HYPSTAR_222315_IRR_SWIR_v2.1.nc index 678efdd0..50692c4d 100644 Binary files a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222315/HYPERNETS_CAL_HYPSTAR_222315_IRR_SWIR_v2.0.nc and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222315/HYPERNETS_CAL_HYPSTAR_222315_IRR_SWIR_v2.1.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222315/HYPERNETS_CAL_HYPSTAR_222315_IRR_SWIR_v2.2.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222315/HYPERNETS_CAL_HYPSTAR_222315_IRR_SWIR_v2.2.nc new file mode 100644 index 00000000..c4a97d8d Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222315/HYPERNETS_CAL_HYPSTAR_222315_IRR_SWIR_v2.2.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222315/HYPERNETS_CAL_HYPSTAR_222315_IRR_SWIR_v2.3.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222315/HYPERNETS_CAL_HYPSTAR_222315_IRR_SWIR_v2.3.nc new file mode 100644 index 00000000..6eef09fe Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222315/HYPERNETS_CAL_HYPSTAR_222315_IRR_SWIR_v2.3.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222315/HYPERNETS_CAL_HYPSTAR_222315_IRR_SWIR_v2.4.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222315/HYPERNETS_CAL_HYPSTAR_222315_IRR_SWIR_v2.4.nc new file mode 100644 index 00000000..43f463e8 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222315/HYPERNETS_CAL_HYPSTAR_222315_IRR_SWIR_v2.4.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222315/HYPERNETS_CAL_HYPSTAR_222315_IRR_v2.0.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222315/HYPERNETS_CAL_HYPSTAR_222315_IRR_v2.1.nc similarity index 97% rename from hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222315/HYPERNETS_CAL_HYPSTAR_222315_IRR_v2.0.nc rename to hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222315/HYPERNETS_CAL_HYPSTAR_222315_IRR_v2.1.nc index eb6644f0..edb8967e 100644 Binary files a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222315/HYPERNETS_CAL_HYPSTAR_222315_IRR_v2.0.nc and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222315/HYPERNETS_CAL_HYPSTAR_222315_IRR_v2.1.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222315/HYPERNETS_CAL_HYPSTAR_222315_IRR_v2.2.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222315/HYPERNETS_CAL_HYPSTAR_222315_IRR_v2.2.nc new file mode 100644 index 00000000..69619239 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222315/HYPERNETS_CAL_HYPSTAR_222315_IRR_v2.2.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222315/HYPERNETS_CAL_HYPSTAR_222315_IRR_v2.3.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222315/HYPERNETS_CAL_HYPSTAR_222315_IRR_v2.3.nc new file mode 100644 index 00000000..4180f7ab Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222315/HYPERNETS_CAL_HYPSTAR_222315_IRR_v2.3.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222315/HYPERNETS_CAL_HYPSTAR_222315_IRR_v2.4.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222315/HYPERNETS_CAL_HYPSTAR_222315_IRR_v2.4.nc new file mode 100644 index 00000000..c7b4bf38 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222315/HYPERNETS_CAL_HYPSTAR_222315_IRR_v2.4.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222315/HYPERNETS_CAL_HYPSTAR_222315_RAD_SWIR_v2.0.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222315/HYPERNETS_CAL_HYPSTAR_222315_RAD_SWIR_v2.1.nc similarity index 64% rename from hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222315/HYPERNETS_CAL_HYPSTAR_222315_RAD_SWIR_v2.0.nc rename to hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222315/HYPERNETS_CAL_HYPSTAR_222315_RAD_SWIR_v2.1.nc index bd7d1e9d..fc86eefa 100644 Binary files a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222315/HYPERNETS_CAL_HYPSTAR_222315_RAD_SWIR_v2.0.nc and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222315/HYPERNETS_CAL_HYPSTAR_222315_RAD_SWIR_v2.1.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222315/HYPERNETS_CAL_HYPSTAR_222315_RAD_SWIR_v2.2.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222315/HYPERNETS_CAL_HYPSTAR_222315_RAD_SWIR_v2.2.nc new file mode 100644 index 00000000..009a93ab Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222315/HYPERNETS_CAL_HYPSTAR_222315_RAD_SWIR_v2.2.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222315/HYPERNETS_CAL_HYPSTAR_222315_RAD_SWIR_v2.3.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222315/HYPERNETS_CAL_HYPSTAR_222315_RAD_SWIR_v2.3.nc new file mode 100644 index 00000000..0246cded Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222315/HYPERNETS_CAL_HYPSTAR_222315_RAD_SWIR_v2.3.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222315/HYPERNETS_CAL_HYPSTAR_222315_RAD_SWIR_v2.4.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222315/HYPERNETS_CAL_HYPSTAR_222315_RAD_SWIR_v2.4.nc new file mode 100644 index 00000000..6750d600 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222315/HYPERNETS_CAL_HYPSTAR_222315_RAD_SWIR_v2.4.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222315/HYPERNETS_CAL_HYPSTAR_222315_RAD_v2.0.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222315/HYPERNETS_CAL_HYPSTAR_222315_RAD_v2.1.nc similarity index 96% rename from hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222315/HYPERNETS_CAL_HYPSTAR_222315_RAD_v2.0.nc rename to hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222315/HYPERNETS_CAL_HYPSTAR_222315_RAD_v2.1.nc index fce99371..68a3fc59 100644 Binary files a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222315/HYPERNETS_CAL_HYPSTAR_222315_RAD_v2.0.nc and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222315/HYPERNETS_CAL_HYPSTAR_222315_RAD_v2.1.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222315/HYPERNETS_CAL_HYPSTAR_222315_RAD_v2.2.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222315/HYPERNETS_CAL_HYPSTAR_222315_RAD_v2.2.nc new file mode 100644 index 00000000..2e8fcbbe Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222315/HYPERNETS_CAL_HYPSTAR_222315_RAD_v2.2.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222315/HYPERNETS_CAL_HYPSTAR_222315_RAD_v2.3.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222315/HYPERNETS_CAL_HYPSTAR_222315_RAD_v2.3.nc new file mode 100644 index 00000000..e4d83ac0 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222315/HYPERNETS_CAL_HYPSTAR_222315_RAD_v2.3.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222315/HYPERNETS_CAL_HYPSTAR_222315_RAD_v2.4.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222315/HYPERNETS_CAL_HYPSTAR_222315_RAD_v2.4.nc new file mode 100644 index 00000000..27b07ef3 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222315/HYPERNETS_CAL_HYPSTAR_222315_RAD_v2.4.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222316/HYPERNETS_CAL_HYPSTAR_222316_IRR_SWIR_v2.0.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222316/HYPERNETS_CAL_HYPSTAR_222316_IRR_SWIR_v2.1.nc similarity index 62% rename from hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222316/HYPERNETS_CAL_HYPSTAR_222316_IRR_SWIR_v2.0.nc rename to hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222316/HYPERNETS_CAL_HYPSTAR_222316_IRR_SWIR_v2.1.nc index f02d3d38..c47123c6 100644 Binary files a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222316/HYPERNETS_CAL_HYPSTAR_222316_IRR_SWIR_v2.0.nc and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222316/HYPERNETS_CAL_HYPSTAR_222316_IRR_SWIR_v2.1.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222316/HYPERNETS_CAL_HYPSTAR_222316_IRR_SWIR_v2.2.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222316/HYPERNETS_CAL_HYPSTAR_222316_IRR_SWIR_v2.2.nc new file mode 100644 index 00000000..cbcdedb0 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222316/HYPERNETS_CAL_HYPSTAR_222316_IRR_SWIR_v2.2.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222316/HYPERNETS_CAL_HYPSTAR_222316_IRR_SWIR_v2.3.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222316/HYPERNETS_CAL_HYPSTAR_222316_IRR_SWIR_v2.3.nc new file mode 100644 index 00000000..7c6abfd9 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222316/HYPERNETS_CAL_HYPSTAR_222316_IRR_SWIR_v2.3.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222316/HYPERNETS_CAL_HYPSTAR_222316_IRR_SWIR_v2.4.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222316/HYPERNETS_CAL_HYPSTAR_222316_IRR_SWIR_v2.4.nc new file mode 100644 index 00000000..3ddc3dee Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222316/HYPERNETS_CAL_HYPSTAR_222316_IRR_SWIR_v2.4.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222316/HYPERNETS_CAL_HYPSTAR_222316_IRR_v2.0.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222316/HYPERNETS_CAL_HYPSTAR_222316_IRR_v2.1.nc similarity index 97% rename from hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222316/HYPERNETS_CAL_HYPSTAR_222316_IRR_v2.0.nc rename to hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222316/HYPERNETS_CAL_HYPSTAR_222316_IRR_v2.1.nc index 0f86e9cb..da4da329 100644 Binary files a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222316/HYPERNETS_CAL_HYPSTAR_222316_IRR_v2.0.nc and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222316/HYPERNETS_CAL_HYPSTAR_222316_IRR_v2.1.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222316/HYPERNETS_CAL_HYPSTAR_222316_IRR_v2.2.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222316/HYPERNETS_CAL_HYPSTAR_222316_IRR_v2.2.nc new file mode 100644 index 00000000..f4b48b64 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222316/HYPERNETS_CAL_HYPSTAR_222316_IRR_v2.2.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222316/HYPERNETS_CAL_HYPSTAR_222316_IRR_v2.3.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222316/HYPERNETS_CAL_HYPSTAR_222316_IRR_v2.3.nc new file mode 100644 index 00000000..921df2d5 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222316/HYPERNETS_CAL_HYPSTAR_222316_IRR_v2.3.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222316/HYPERNETS_CAL_HYPSTAR_222316_IRR_v2.4.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222316/HYPERNETS_CAL_HYPSTAR_222316_IRR_v2.4.nc new file mode 100644 index 00000000..cf56d12c Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222316/HYPERNETS_CAL_HYPSTAR_222316_IRR_v2.4.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222316/HYPERNETS_CAL_HYPSTAR_222316_RAD_SWIR_v2.0.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222316/HYPERNETS_CAL_HYPSTAR_222316_RAD_SWIR_v2.1.nc similarity index 63% rename from hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222316/HYPERNETS_CAL_HYPSTAR_222316_RAD_SWIR_v2.0.nc rename to hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222316/HYPERNETS_CAL_HYPSTAR_222316_RAD_SWIR_v2.1.nc index 12e00945..105cb72a 100644 Binary files a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222316/HYPERNETS_CAL_HYPSTAR_222316_RAD_SWIR_v2.0.nc and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222316/HYPERNETS_CAL_HYPSTAR_222316_RAD_SWIR_v2.1.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222316/HYPERNETS_CAL_HYPSTAR_222316_RAD_SWIR_v2.2.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222316/HYPERNETS_CAL_HYPSTAR_222316_RAD_SWIR_v2.2.nc new file mode 100644 index 00000000..f2025e61 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222316/HYPERNETS_CAL_HYPSTAR_222316_RAD_SWIR_v2.2.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222316/HYPERNETS_CAL_HYPSTAR_222316_RAD_SWIR_v2.3.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222316/HYPERNETS_CAL_HYPSTAR_222316_RAD_SWIR_v2.3.nc new file mode 100644 index 00000000..4955db85 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222316/HYPERNETS_CAL_HYPSTAR_222316_RAD_SWIR_v2.3.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222316/HYPERNETS_CAL_HYPSTAR_222316_RAD_SWIR_v2.4.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222316/HYPERNETS_CAL_HYPSTAR_222316_RAD_SWIR_v2.4.nc new file mode 100644 index 00000000..ff08f613 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222316/HYPERNETS_CAL_HYPSTAR_222316_RAD_SWIR_v2.4.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222316/HYPERNETS_CAL_HYPSTAR_222316_RAD_v2.0.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222316/HYPERNETS_CAL_HYPSTAR_222316_RAD_v2.1.nc similarity index 96% rename from hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222316/HYPERNETS_CAL_HYPSTAR_222316_RAD_v2.0.nc rename to hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222316/HYPERNETS_CAL_HYPSTAR_222316_RAD_v2.1.nc index 027e801d..45ab879e 100644 Binary files a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222316/HYPERNETS_CAL_HYPSTAR_222316_RAD_v2.0.nc and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222316/HYPERNETS_CAL_HYPSTAR_222316_RAD_v2.1.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222316/HYPERNETS_CAL_HYPSTAR_222316_RAD_v2.2.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222316/HYPERNETS_CAL_HYPSTAR_222316_RAD_v2.2.nc new file mode 100644 index 00000000..23d38ccb Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222316/HYPERNETS_CAL_HYPSTAR_222316_RAD_v2.2.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222316/HYPERNETS_CAL_HYPSTAR_222316_RAD_v2.3.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222316/HYPERNETS_CAL_HYPSTAR_222316_RAD_v2.3.nc new file mode 100644 index 00000000..8109cb78 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222316/HYPERNETS_CAL_HYPSTAR_222316_RAD_v2.3.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222316/HYPERNETS_CAL_HYPSTAR_222316_RAD_v2.4.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222316/HYPERNETS_CAL_HYPSTAR_222316_RAD_v2.4.nc new file mode 100644 index 00000000..f9c0ea5e Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222316/HYPERNETS_CAL_HYPSTAR_222316_RAD_v2.4.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222317/HYPERNETS_CAL_HYPSTAR_222317_IRR_SWIR_v2.0.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222317/HYPERNETS_CAL_HYPSTAR_222317_IRR_SWIR_v2.1.nc similarity index 63% rename from hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222317/HYPERNETS_CAL_HYPSTAR_222317_IRR_SWIR_v2.0.nc rename to hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222317/HYPERNETS_CAL_HYPSTAR_222317_IRR_SWIR_v2.1.nc index 8b94f2a1..f19fd60f 100644 Binary files a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222317/HYPERNETS_CAL_HYPSTAR_222317_IRR_SWIR_v2.0.nc and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222317/HYPERNETS_CAL_HYPSTAR_222317_IRR_SWIR_v2.1.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222317/HYPERNETS_CAL_HYPSTAR_222317_IRR_SWIR_v2.2.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222317/HYPERNETS_CAL_HYPSTAR_222317_IRR_SWIR_v2.2.nc new file mode 100644 index 00000000..b6d4096a Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222317/HYPERNETS_CAL_HYPSTAR_222317_IRR_SWIR_v2.2.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222317/HYPERNETS_CAL_HYPSTAR_222317_IRR_SWIR_v2.3.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222317/HYPERNETS_CAL_HYPSTAR_222317_IRR_SWIR_v2.3.nc new file mode 100644 index 00000000..d0e6a1a9 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222317/HYPERNETS_CAL_HYPSTAR_222317_IRR_SWIR_v2.3.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222317/HYPERNETS_CAL_HYPSTAR_222317_IRR_SWIR_v2.4.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222317/HYPERNETS_CAL_HYPSTAR_222317_IRR_SWIR_v2.4.nc new file mode 100644 index 00000000..fb7831ec Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222317/HYPERNETS_CAL_HYPSTAR_222317_IRR_SWIR_v2.4.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222317/HYPERNETS_CAL_HYPSTAR_222317_IRR_v2.0.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222317/HYPERNETS_CAL_HYPSTAR_222317_IRR_v2.1.nc similarity index 97% rename from hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222317/HYPERNETS_CAL_HYPSTAR_222317_IRR_v2.0.nc rename to hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222317/HYPERNETS_CAL_HYPSTAR_222317_IRR_v2.1.nc index 56345e2b..866ba928 100644 Binary files a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222317/HYPERNETS_CAL_HYPSTAR_222317_IRR_v2.0.nc and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222317/HYPERNETS_CAL_HYPSTAR_222317_IRR_v2.1.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222317/HYPERNETS_CAL_HYPSTAR_222317_IRR_v2.2.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222317/HYPERNETS_CAL_HYPSTAR_222317_IRR_v2.2.nc new file mode 100644 index 00000000..1d362be2 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222317/HYPERNETS_CAL_HYPSTAR_222317_IRR_v2.2.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222317/HYPERNETS_CAL_HYPSTAR_222317_IRR_v2.3.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222317/HYPERNETS_CAL_HYPSTAR_222317_IRR_v2.3.nc new file mode 100644 index 00000000..2b0fd5d3 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222317/HYPERNETS_CAL_HYPSTAR_222317_IRR_v2.3.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222317/HYPERNETS_CAL_HYPSTAR_222317_IRR_v2.4.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222317/HYPERNETS_CAL_HYPSTAR_222317_IRR_v2.4.nc new file mode 100644 index 00000000..0d1ac045 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222317/HYPERNETS_CAL_HYPSTAR_222317_IRR_v2.4.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222317/HYPERNETS_CAL_HYPSTAR_222317_RAD_SWIR_v2.0.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222317/HYPERNETS_CAL_HYPSTAR_222317_RAD_SWIR_v2.1.nc similarity index 64% rename from hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222317/HYPERNETS_CAL_HYPSTAR_222317_RAD_SWIR_v2.0.nc rename to hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222317/HYPERNETS_CAL_HYPSTAR_222317_RAD_SWIR_v2.1.nc index 04111658..7138133b 100644 Binary files a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222317/HYPERNETS_CAL_HYPSTAR_222317_RAD_SWIR_v2.0.nc and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222317/HYPERNETS_CAL_HYPSTAR_222317_RAD_SWIR_v2.1.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222317/HYPERNETS_CAL_HYPSTAR_222317_RAD_SWIR_v2.2.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222317/HYPERNETS_CAL_HYPSTAR_222317_RAD_SWIR_v2.2.nc new file mode 100644 index 00000000..5309deed Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222317/HYPERNETS_CAL_HYPSTAR_222317_RAD_SWIR_v2.2.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222317/HYPERNETS_CAL_HYPSTAR_222317_RAD_SWIR_v2.3.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222317/HYPERNETS_CAL_HYPSTAR_222317_RAD_SWIR_v2.3.nc new file mode 100644 index 00000000..6461e518 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222317/HYPERNETS_CAL_HYPSTAR_222317_RAD_SWIR_v2.3.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222317/HYPERNETS_CAL_HYPSTAR_222317_RAD_SWIR_v2.4.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222317/HYPERNETS_CAL_HYPSTAR_222317_RAD_SWIR_v2.4.nc new file mode 100644 index 00000000..8ff385f4 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222317/HYPERNETS_CAL_HYPSTAR_222317_RAD_SWIR_v2.4.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222317/HYPERNETS_CAL_HYPSTAR_222317_RAD_v2.0.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222317/HYPERNETS_CAL_HYPSTAR_222317_RAD_v2.1.nc similarity index 97% rename from hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222317/HYPERNETS_CAL_HYPSTAR_222317_RAD_v2.0.nc rename to hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222317/HYPERNETS_CAL_HYPSTAR_222317_RAD_v2.1.nc index 56bf8c60..389d3665 100644 Binary files a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222317/HYPERNETS_CAL_HYPSTAR_222317_RAD_v2.0.nc and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222317/HYPERNETS_CAL_HYPSTAR_222317_RAD_v2.1.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222317/HYPERNETS_CAL_HYPSTAR_222317_RAD_v2.2.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222317/HYPERNETS_CAL_HYPSTAR_222317_RAD_v2.2.nc new file mode 100644 index 00000000..b5883ab7 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222317/HYPERNETS_CAL_HYPSTAR_222317_RAD_v2.2.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222317/HYPERNETS_CAL_HYPSTAR_222317_RAD_v2.3.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222317/HYPERNETS_CAL_HYPSTAR_222317_RAD_v2.3.nc new file mode 100644 index 00000000..746a00e4 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222317/HYPERNETS_CAL_HYPSTAR_222317_RAD_v2.3.nc differ diff --git a/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222317/HYPERNETS_CAL_HYPSTAR_222317_RAD_v2.4.nc b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222317/HYPERNETS_CAL_HYPSTAR_222317_RAD_v2.4.nc new file mode 100644 index 00000000..7ad922f3 Binary files /dev/null and b/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_222317/HYPERNETS_CAL_HYPSTAR_222317_RAD_v2.4.nc differ diff --git a/hypernets_processor/cli/scheduler_cli.py b/hypernets_processor/cli/scheduler_cli.py index 5f0b7b59..5c74d716 100755 --- a/hypernets_processor/cli/scheduler_cli.py +++ b/hypernets_processor/cli/scheduler_cli.py @@ -39,6 +39,13 @@ def configure_parser(): description=description, formatter_class=argparse.ArgumentDefaultsHelpFormatter ) + # parser.add_argument( + # "-o", + # "--output-directory", + # action="store", + # help="Directory to write output data to", + # ) + return parser diff --git a/hypernets_processor/combine_SWIR/combine_SWIR.py b/hypernets_processor/combine_SWIR/combine_SWIR.py index da5d9c17..ab8219a0 100755 --- a/hypernets_processor/combine_SWIR/combine_SWIR.py +++ b/hypernets_processor/combine_SWIR/combine_SWIR.py @@ -95,14 +95,14 @@ def combine( store_unc_percent=True, ) - dataset_l1b_comb[ - "std_" + measurandstring - ].values = combine_function.meas_function( - dataset_l1b["wavelength"].values, - dataset_l1b["std_" + measurandstring].values, - dataset_l1b_swir["wavelength"].values, - dataset_l1b_swir["std_" + measurandstring].values, - 1000, + dataset_l1b_comb["std_" + measurandstring].values = ( + combine_function.meas_function( + dataset_l1b["wavelength"].values, + dataset_l1b["std_" + measurandstring].values, + dataset_l1b_swir["wavelength"].values, + dataset_l1b_swir["std_" + measurandstring].values, + 1000, + ) ) dataset_l1b_comb["n_valid_scans"].values = dataset_l1b["n_valid_scans"].values dataset_l1b_comb["n_valid_scans_SWIR"].values = dataset_l1b_swir[ diff --git a/hypernets_processor/context.py b/hypernets_processor/context.py index 7a66ecec..1c24b031 100755 --- a/hypernets_processor/context.py +++ b/hypernets_processor/context.py @@ -1,6 +1,7 @@ """ Context class """ + from hypernets_processor.version import __version__ from hypernets_processor.utils.config import get_config_value from hypernets_processor.data_io.format.databases import DB_DICT_DEFS diff --git a/hypernets_processor/data_io/data_templates.py b/hypernets_processor/data_io/data_templates.py index 7893b768..d9425564 100755 --- a/hypernets_processor/data_io/data_templates.py +++ b/hypernets_processor/data_io/data_templates.py @@ -37,6 +37,7 @@ def calibration_dataset( cal_dim_sizes_dict = { "wavelength": len(wavs), + "wavelength'": len(wavs), "nonlinearcoef": 13, "wavcoef": len(wavcoef), "calibrationdates": len(caldates), diff --git a/hypernets_processor/data_io/format/anomalies.py b/hypernets_processor/data_io/format/anomalies.py index 719fa042..d71da013 100755 --- a/hypernets_processor/data_io/format/anomalies.py +++ b/hypernets_processor/data_io/format/anomalies.py @@ -13,7 +13,7 @@ "a": { "description": "angle achieved by pan >3 and/or tilt >1 deg away from target angle during sequence", "error": None, - "error_msg": "angle achieved by pan >3 and/or tilt >1 deg away from target angle during sequence", + "error_msg": "angle achieved by pan >3 and/or tilt >1 deg away from target angle during sequence", }, "m": { "description": "Metadata file missing", @@ -33,7 +33,7 @@ "o": { "description": "more than 50% of data has random error above 100% (probably corrupted data)", "error": None, - "error_msg": "more than 50% of data has random error above 100% (probably corrupted data)" + "error_msg": "more than 50% of data has random error above 100% (probably corrupted data)", }, "u": { "description": "some of the uncertainties have negative values", @@ -58,7 +58,7 @@ "ned": { "description": "Not enough Ed scans for series", "error": None, - "error_msg": "Not enough Ed scans for series", + "error_msg": "Not enough Ed scans for series", }, "nu": { "description": "Non constant illumination", @@ -76,7 +76,7 @@ "error_msg": "No clear sky irradiance in sequence (i.e. overcast conditions)", }, "in": { - "description": "Invalid sequence (due to not enough valid radiance or irradiance series)", + "description": "Invalid sequence (due to no valid radiance or irradiance series)", "error": ValueError, "error_msg": "Invalid sequence (due to not enough valid radiance or irradiance series)", }, @@ -97,4 +97,54 @@ "error": ValueError, "error_msg": "Invalid sequence,files mentioned in metadatafile are missing in DATA directory", }, + "per": { + "description": "sequence not within deployment periods", + "error": ValueError, + "error_msg": "sequence not within deployment periods", + }, + "val": { + "description": "sequence within deployment periods that is not valid for satellite validation", + "error": ValueError, + "error_msg": "sequence within deployment periods that is not valid for satellite validation", + }, + "tod": { + "description": "sequence not within selected time of day", + "error": ValueError, + "error_msg": "sequence not within selected time of day", + }, + "hsn": { + "description": "HYPSTAR_SN does not match deployment periods", + "error": ValueError, + "error_msg": "HYPSTAR_SN does not match deployment periods", + }, + "scl": { + "description": "site-specific clear sky check has failed", + "error": ValueError, + "error_msg": "site-specific clear sky check has failed", + }, + "npr": { + "description": "no pre-existing files available, while config is set up to reprocess from pre-existing files", + "error": ValueError, + "error_msg": "no pre-existing files available, while config is set up to reprocess from pre-existing files", + }, + "man": { + "description": "this sequences was manually discarded as unreliable in the config file", + "error": ValueError, + "error_msg": "this sequences was manually discarded as unreliable in the config file", + }, + "wns": { + "description": "wrong number of series (l2a and l1b rad do not match in number of series). No L1d file is produced.", + "error": None, + "error_msg": "wrong number of series (l2a and l1b rad do not match in number of series). No L1d file is produced.", + }, + "nos": { + "description": "No series was present in the sequences after applying all site-specific QC", + "error": ValueError, + "error_msg": "No series was present in the sequences after applying all site-specific QC", + }, + "hos": { + "description": "More than half of series were removed by site-specific QC checks (excluding bad viewing angles and bad dates)", + "error": ValueError, + "error_msg": "More than half series were removed by site-specific QC checks (excluding bad viewing angles and bad dates)", + }, } diff --git a/hypernets_processor/data_io/format/metadata.py b/hypernets_processor/data_io/format/metadata.py index 351fbfc0..c16537fd 100755 --- a/hypernets_processor/data_io/format/metadata.py +++ b/hypernets_processor/data_io/format/metadata.py @@ -65,15 +65,6 @@ "illuminance": "NaN", "system_relative_humidity": "NaN", } -COMPONENTS_METADATA = { # "system_id": system_id, - "instrument_id": "TBD", # key metadata - # "pointing_system_id": "TBD", # key metadata - # "calibration_device_id": "TBD", # key metadata - # "irr_vis_head_id": "TBD", # key metadata - # "rad_vis_head_id": "TBD", # key metadata - # "rain_sensor_id": "TBD", # key metadata - # "rgb_camera_id": "TBD", # key metadata -} CAL_METADATA = { "type": "dataset", @@ -222,16 +213,14 @@ "relative_azimuth_angle_max": "TBD", } -SYSTEM_METADATA = { # "system_id": system_id, - "system_model": "TBD", - "system_manufacturer": "Laboratoire d'Océanographie de Villefranche UMR 7093 - CNRS / Sorbonne Univ", - "system_date_manufacture": "TBD", - "system_version": "TBD", - "system_firmaware_version": "TBD", - "system_documentation_references": "TBD", - "system_deployment_date": "TBD", - "system_deployment_height": "TBD", - "system_comment": "system below bird nest, bad luck", +COMPONENTS_METADATA = { # "system_id": system_id, + "instrument_id": "TBD", # key metadata + # "pointing_system_id": "TBD", # key metadata + # "calibration_device_id": "TBD", # key metadata + # "irr_vis_head_id": "TBD", # key metadata + # "rad_vis_head_id": "TBD", # key metadata + # "rain_sensor_id": "TBD", # key metadata + # "rgb_camera_id": "TBD", # key metadata } INSTRUMENT_METADATA = { @@ -317,6 +306,29 @@ "rgb_camera_date_manufacture": "TBD", } +SYSTEM_METADATA = { # "system_id": system_id, + "system_model": "TBD", + "system_manufacturer": "Laboratoire d'Océanographie de Villefranche UMR 7093 - CNRS / Sorbonne Univ", + "system_date_manufacture": "TBD", + "system_version": "TBD", + "system_firmware_version": "TBD", + "system_documentation_references": "TBD", + "system_deployment_date": "TBD", + "system_deployment_height": "TBD", + "system_comment": "system below bird nest, bad luck", +} + + +# this needs to be further dicsussed after we have achieved conclusion on what to do in terms of QC for deployment periods +DEPLOYMENT_METADATA = { + "start_date": "NaN", + "intstument_id": "NaN", + "hypernets_tools_version": "NaN", + "site_latitude": "NaN", + "site_longitude": "NaN", + # components? how do we gather this info and make sure its up to date? +} + SITE_METADATA = { "site_description": "TBD", "site_latitude": "NaN", @@ -448,6 +460,16 @@ **WATER_NETWORK_METADATA, **W_L2A_REF_METADATA, }, - "L_L2B": {**STANDARD_METADATA, **COMPONENTS_METADATA, **LAND_NETWORK_METADATA, **L_L2A_REF_METADATA}, - "W_L2B": {**STANDARD_METADATA, **COMPONENTS_METADATA, **WATER_NETWORK_METADATA, **W_L2A_REF_METADATA}, + "L_L2B": { + **STANDARD_METADATA, + **COMPONENTS_METADATA, + **LAND_NETWORK_METADATA, + **L_L2A_REF_METADATA, + }, + "W_L2B": { + **STANDARD_METADATA, + **COMPONENTS_METADATA, + **WATER_NETWORK_METADATA, + **W_L2A_REF_METADATA, + }, } diff --git a/hypernets_processor/data_io/format/variables.py b/hypernets_processor/data_io/format/variables.py index 5527fa0c..da74c9a6 100755 --- a/hypernets_processor/data_io/format/variables.py +++ b/hypernets_processor/data_io/format/variables.py @@ -1,6 +1,7 @@ """ Variable definitions for Hypernets land a water network data products """ + from typing import Any from copy import deepcopy @@ -605,7 +606,7 @@ "encoding": {"dtype": np.uint16, "scale_factor": 0.01, "offset": 0.0}, }, "err_corr_systematic_indep_gains": { - "dim": [CD_DIM, WL_DIM, WL_DIM], + "dim": [CD_DIM, WL_DIM, WL_DIM + "'"], "dtype": np.float32, "attributes": { "standard_name": "independent correlation matrix of systematic error on gains", @@ -615,7 +616,7 @@ "encoding": {"dtype": np.int16, "scale_factor": 0.0001, "offset": 0.0}, }, "err_corr_systematic_corr_rad_irr_gains": { - "dim": [CD_DIM, WL_DIM, WL_DIM], + "dim": [CD_DIM, WL_DIM, WL_DIM + "'"], "dtype": np.float32, "attributes": { "standard_name": "correlation matrix of systematic error on gains (correlated radiance and irradiance)", @@ -1532,11 +1533,6 @@ "standard_name": "std radiance", "long_name": "standard deviation for downwelling radiance", "units": "mW m^-2 nm^-1 sr^-1", - "unc_comps": [ - "u_rel_random_downwelling_radiance", - "u_rel_systematic_indep_downwelling_radiance", - "u_rel_systematic_corr_rad_irr_downwelling_radiance", - ], }, }, "u_rel_random_upwelling_radiance": { @@ -1718,11 +1714,6 @@ "standard_name": "std irradiance", "long_name": "standard deviation for downwelling irradiance", "units": "mW m^-2 nm^-1", - "unc_comps": [ - "u_rel_random_irradiance", - "u_rel_systematic_indep_irradiance", - "u_rel_systematic_corr_rad_irr_irradiance", - ], }, }, "rhof": { diff --git a/hypernets_processor/data_io/hypernets_db_builder.py b/hypernets_processor/data_io/hypernets_db_builder.py index 21552440..bc7bb0f6 100755 --- a/hypernets_processor/data_io/hypernets_db_builder.py +++ b/hypernets_processor/data_io/hypernets_db_builder.py @@ -150,7 +150,9 @@ def archive_product(self, ds, path): rel_product_dir=self.writer.return_rel_directory(), product_level=ds.attrs["product_level"], datetime_SEQ=self.context.get_config_value("time"), - datetime_start=dt.fromtimestamp(np.nanmin(ds["acquisition_time"].values)), + datetime_start=dt.fromtimestamp( + np.nanmin(ds["acquisition_time"].values) + ), datetime_end=dt.fromtimestamp(np.nanmax(ds["acquisition_time"].values)), sequence_name=self.context.get_config_value("sequence_name"), sequence_path=self.context.get_config_value("sequence_path"), diff --git a/hypernets_processor/data_io/hypernets_ds_builder.py b/hypernets_processor/data_io/hypernets_ds_builder.py index 85604dfa..927050c9 100755 --- a/hypernets_processor/data_io/hypernets_ds_builder.py +++ b/hypernets_processor/data_io/hypernets_ds_builder.py @@ -56,7 +56,6 @@ def create_ds_template( angles: object = False, ds=None, ) -> object: - """ Returns empty Hypernets dataset @@ -150,26 +149,50 @@ def create_ds_template( if ds_format in ["L_L1A_RAD", "W_L1A_RAD", "L_L1B_RAD"]: metadata["instrument_calibration_file_rad"] = ( - "HYPERNETS_CAL_" + hypstar.upper() + "_RAD_v" + str(version) + ".nc" + "HYPERNETS_CAL_" + + hypstar.upper() + + "_RAD_v" + + str(self.context.get_config_value("calibration_file_version")) + + ".nc" ) if ds_format in ["L_L1A_IRR", "W_L1A_IRR", "L_L1B_IRR"]: metadata["instrument_calibration_file_irr"] = ( - "HYPERNETS_CAL_" + hypstar.upper() + "_IRR_v" + str(version) + ".nc" + "HYPERNETS_CAL_" + + hypstar.upper() + + "_IRR_v" + + str(self.context.get_config_value("calibration_file_version")) + + ".nc" ) if ds_format in ["W_L1B", "L_L1C", "L_L2A"]: metadata["instrument_calibration_file_irr"] = ( - "HYPERNETS_CAL_" + hypstar.upper() + "_IRR_v" + str(version) + ".nc" + "HYPERNETS_CAL_" + + hypstar.upper() + + "_IRR_v" + + str(self.context.get_config_value("calibration_file_version")) + + ".nc" ) metadata["instrument_calibration_file_rad"] = ( - "HYPERNETS_CAL_" + hypstar.upper() + "_RAD_v" + str(version) + ".nc" + "HYPERNETS_CAL_" + + hypstar.upper() + + "_RAD_v" + + str(self.context.get_config_value("calibration_file_version")) + + ".nc" ) if ds_format in ["W_L1C", "W_L2A"]: # W_L1C metadata["instrument_calibration_file_rad"] = ( - "HYPERNETS_CAL_" + hypstar.upper() + "_RAD_v" + str(version) + ".nc" + "HYPERNETS_CAL_" + + hypstar.upper() + + "_RAD_v" + + str(self.context.get_config_value("calibration_file_version")) + + ".nc" ) metadata["instrument_calibration_file_irr"] = ( - "HYPERNETS_CAL_" + hypstar.upper() + "_IRR_v" + str(version) + ".nc" + "HYPERNETS_CAL_" + + hypstar.upper() + + "_IRR_v" + + str(self.context.get_config_value("calibration_file_version")) + + ".nc" ) metadata["rhof_option"] = self.context.get_config_value("rhof_option") diff --git a/hypernets_processor/data_io/hypernets_reader.py b/hypernets_processor/data_io/hypernets_reader.py index 87400013..b669f8fa 100755 --- a/hypernets_processor/data_io/hypernets_reader.py +++ b/hypernets_processor/data_io/hypernets_reader.py @@ -1,6 +1,7 @@ """ Module with functions and classes to read Hypernets data """ + from datetime import datetime, timezone import os, shutil import re # for re.split @@ -171,7 +172,7 @@ def read_wavelength(self, pixcount, cal_data): return wvl def read_angles( - self, ds, scan_number, specattr, offset_pan, offset_tilt, angle2use, land=False + self, ds, scan_number, specattr, offset_pan, offset_tilt, angle2use, land=False ): paa_ask, vza_ask = map(float, specattr["pt_ask"].split(";")) @@ -198,18 +199,18 @@ def read_angles( ) if ( - ((offset_pan is None) or (offset_tilt is None)) - & (angle2use == "pt_ask") - & (not land) + ((offset_pan is None) or (offset_tilt is None)) + & (angle2use == "pt_ask") + & (not land) ): paa = normalizedeg( float(paa_ask) + ds["solar_azimuth_angle"][scan_number], 0, 360 ) vza = vza_ask elif ( - ((offset_pan is None) or (offset_tilt is None)) - & (angle2use == "pt_ask") - & (land) + ((offset_pan is None) or (offset_tilt is None)) + & (angle2use == "pt_ask") + & (land) ): paa = normalizedeg(float(paa_ask), 0, 360) vza = vza_ask @@ -493,21 +494,21 @@ def read_angles( # return ds def read_series( - self, - seq_dir, - series, - lat, - lon, - metadata, - flag, - fileformat, - cal_data, - instrument_id, - site_id, - azimuth_switch, - offset_tilt, - offset_pan, - angle2use, + self, + seq_dir, + series, + lat, + lon, + metadata, + flag, + fileformat, + cal_data, + instrument_id, + site_id, + azimuth_switch, + offset_tilt, + offset_pan, + angle2use, ): FOLDER_NAME = os.path.join(seq_dir, "RADIOMETER/") model_name = self.model @@ -518,15 +519,15 @@ def read_series( self.context.logger.debug("processing " + spectra) model = dict(zip(model_name, spectra.split("_")[:-1])) specBlock = ( - model["series_rep"] - + "_" - + model["series_id"] - + "_" - + model["vaa"] - + "_" - + model["azimuth_ref"] - + "_" - + model["vza"] + model["series_rep"] + + "_" + + model["series_id"] + + "_" + + model["vaa"] + + "_" + + model["azimuth_ref"] + + "_" + + model["vza"] ) # spectra attributes from metadata file specattr = dict(metadata[specBlock]) @@ -576,7 +577,7 @@ def read_series( "A file (%s) listed in the metadata.txt is missing." % (spectra) ) self.context.anomaly_handler.add_anomaly("mf") - #break + # break if len(vnir.shape) == 1: vnir = vnir[None, :] @@ -599,15 +600,15 @@ def read_series( for spectra in series: model = dict(zip(model_name, spectra.split("_")[:-1])) specBlock = ( - model["series_rep"] - + "_" - + model["series_id"] - + "_" - + model["vaa"] - + "_" - + model["azimuth_ref"] - + "_" - + model["vza"] + model["series_rep"] + + "_" + + model["series_id"] + + "_" + + model["vaa"] + + "_" + + model["azimuth_ref"] + + "_" + + model["vza"] ) # spectra attributes from metadata file specattr = dict(metadata[specBlock]) @@ -647,20 +648,19 @@ def read_series( ds["series_id"][scan_number] = series_id # estimate time based on timestamp - ds["acquisition_time"][ - scan_number - ] = datetime.datetime.timestamp(acquisitionTime) + ds["acquisition_time"][scan_number] = ( + datetime.datetime.timestamp(acquisitionTime) + ) if lat is not None: ds.attrs["site_latitude"] = lat ds.attrs["site_longitude"] = lon - ds["solar_zenith_angle"][ - scan_number - ] = 90 - get_altitude( - float(lat), float(lon), acquisitionTime + ds["solar_zenith_angle"][scan_number] = ( + 90 + - get_altitude( + float(lat), float(lon), acquisitionTime + ) ) - ds["solar_azimuth_angle"][ - scan_number - ] = get_azimuth( + ds["solar_azimuth_angle"][scan_number] = get_azimuth( float(lat), float(lon), acquisitionTime ) elif scan_number == 0: @@ -671,9 +671,7 @@ def read_series( ds["integration_time"][ scan_number ] = spectrum.header.exposure_time - ds["temperature"][ - scan_number - ] = spectrum.header.temperature + ds["temperature"][scan_number] = spectrum.header.temperature ds = self.read_angles( ds, @@ -695,24 +693,24 @@ def read_series( # Acceleration for each axis can be calculated per Eq. (4). a = 19.6 - b = 2 ** 15 + b = 2**15 ds["acceleration_x_mean"][scan_number] = ( - spectrum.header.accel_stats.mean_x * a / b + spectrum.header.accel_stats.mean_x * a / b ) ds["acceleration_x_std"][scan_number] = ( - spectrum.header.accel_stats.std_x * a / b + spectrum.header.accel_stats.std_x * a / b ) ds["acceleration_y_mean"][scan_number] = ( - spectrum.header.accel_stats.mean_y * a / b + spectrum.header.accel_stats.mean_y * a / b ) ds["acceleration_y_std"][scan_number] = ( - spectrum.header.accel_stats.std_y * a / b + spectrum.header.accel_stats.std_y * a / b ) ds["acceleration_z_mean"][scan_number] = ( - spectrum.header.accel_stats.mean_z * a / b + spectrum.header.accel_stats.mean_z * a / b ) ds["acceleration_z_std"][scan_number] = ( - spectrum.header.accel_stats.std_z * a / b + spectrum.header.accel_stats.std_z * a / b ) ds["digital_number"][:, scan_number] = scan scan_number += 1 @@ -727,21 +725,21 @@ def read_series( return ds def read_series_L( - self, - seq_dir, - series, - lat, - lon, - metadata, - flag, - fileformat, - cal_data, - cal_data_swir, - instrument_id, - site_id, - offset_tilt, - offset_pan, - angle2use, + self, + seq_dir, + series, + lat, + lon, + metadata, + flag, + fileformat, + cal_data, + cal_data_swir, + instrument_id, + site_id, + offset_tilt, + offset_pan, + angle2use, ): FOLDER_NAME = os.path.join(seq_dir, "RADIOMETER/") model_name = self.model @@ -753,15 +751,15 @@ def read_series_L( self.context.logger.debug("processing " + spectra) model = dict(zip(model_name, spectra.split("_")[:-1])) specBlock = ( - model["series_rep"] - + "_" - + model["series_id"] - + "_" - + model["vaa"] - + "_" - + model["azimuth_ref"] - + "_" - + model["vza"] + model["series_rep"] + + "_" + + model["series_id"] + + "_" + + model["vaa"] + + "_" + + model["azimuth_ref"] + + "_" + + model["vza"] ) # spectra attributes from metadata file specattr = dict(metadata[specBlock]) @@ -816,11 +814,10 @@ def read_series_L( "A file (%s) listed in the metadata.txt is missing." % (spectra) ) - if len(vnir)>0: + if len(vnir) > 0: if len(vnir.shape) == 1: vnir = vnir[None, :] - scanDim = vnir.shape[0] wvl = self.read_wavelength(vnir.shape[1], cal_data) ds = self.templ.l0a_template_dataset(wvl, scanDim, fileformat) @@ -832,7 +829,7 @@ def read_series_L( ds["bandwidth"].values = 3 * np.ones_like(wvl) else: - ds=None + ds = None if len(swir) > 0: if len(swir.shape) == 1: @@ -863,15 +860,15 @@ def read_series_L( for spectra in series: model = dict(zip(model_name, spectra.split("_")[:-1])) specBlock = ( - model["series_rep"] - + "_" - + model["series_id"] - + "_" - + model["vaa"] - + "_" - + model["azimuth_ref"] - + "_" - + model["vza"] + model["series_rep"] + + "_" + + model["series_id"] + + "_" + + model["vaa"] + + "_" + + model["azimuth_ref"] + + "_" + + model["vza"] ) # spectra attributes from metadata file specattr = dict(metadata[specBlock]) @@ -913,21 +910,22 @@ def read_series_L( ds["series_id"][scan_number] = series_id # estimate time based on timestamp - ds["acquisition_time"][ - scan_number - ] = datetime.datetime.timestamp(acquisitionTime) + ds["acquisition_time"][scan_number] = ( + datetime.datetime.timestamp(acquisitionTime) + ) if lat is not None: ds.attrs["site_latitude"] = lat ds.attrs["site_longitude"] = lon - ds["solar_zenith_angle"][ - scan_number - ] = 90 - get_altitude( - float(lat), float(lon), acquisitionTime + ds["solar_zenith_angle"][scan_number] = ( + 90 + - get_altitude( + float(lat), float(lon), acquisitionTime + ) ) - ds["solar_azimuth_angle"][ - scan_number - ] = get_azimuth( - float(lat), float(lon), acquisitionTime + ds["solar_azimuth_angle"][scan_number] = ( + get_azimuth( + float(lat), float(lon), acquisitionTime + ) ) elif scan_number == 0: self.context.logger.warning( @@ -961,24 +959,24 @@ def read_series_L( # Acceleration for each axis can be calculated per Eq. (4). a = 19.6 - b = 2 ** 15 + b = 2**15 ds["acceleration_x_mean"][scan_number] = ( - spectrum.header.accel_stats.mean_x * a / b + spectrum.header.accel_stats.mean_x * a / b ) ds["acceleration_x_std"][scan_number] = ( - spectrum.header.accel_stats.std_x * a / b + spectrum.header.accel_stats.std_x * a / b ) ds["acceleration_y_mean"][scan_number] = ( - spectrum.header.accel_stats.mean_y * a / b + spectrum.header.accel_stats.mean_y * a / b ) ds["acceleration_y_std"][scan_number] = ( - spectrum.header.accel_stats.std_y * a / b + spectrum.header.accel_stats.std_y * a / b ) ds["acceleration_z_mean"][scan_number] = ( - spectrum.header.accel_stats.mean_z * a / b + spectrum.header.accel_stats.mean_z * a / b ) ds["acceleration_z_std"][scan_number] = ( - spectrum.header.accel_stats.std_z * a / b + spectrum.header.accel_stats.std_z * a / b ) ds["digital_number"][:, scan_number] = scan scan_number += 1 @@ -995,9 +993,9 @@ def read_series_L( ds_swir["series_id"][scan_number_swir] = series_id # estimate time based on timestamp - ds_swir["acquisition_time"][ - scan_number_swir - ] = datetime.datetime.timestamp(acquisitionTime) + ds_swir["acquisition_time"][scan_number_swir] = ( + datetime.datetime.timestamp(acquisitionTime) + ) # #print(datetime.fromtimestamp(acquisitionTime)) # # didn't use acquisition time from instrument @@ -1020,15 +1018,16 @@ def read_series_L( if lat is not None: ds_swir.attrs["site_latitude"] = lat ds_swir.attrs["site_longitude"] = lon - ds_swir["solar_zenith_angle"][ - scan_number_swir - ] = 90 - get_altitude( - float(lat), float(lon), acquisitionTime + ds_swir["solar_zenith_angle"][scan_number_swir] = ( + 90 + - get_altitude( + float(lat), float(lon), acquisitionTime + ) ) - ds_swir["solar_azimuth_angle"][ - scan_number_swir - ] = get_azimuth( - float(lat), float(lon), acquisitionTime + ds_swir["solar_azimuth_angle"][scan_number_swir] = ( + get_azimuth( + float(lat), float(lon), acquisitionTime + ) ) elif scan_number_swir == 0: @@ -1068,24 +1067,24 @@ def read_series_L( # Acceleration for each axis can be calculated per Eq. (4). a = 19.6 - b = 2 ** 15 + b = 2**15 ds_swir["acceleration_x_mean"][scan_number_swir] = ( - spectrum.header.accel_stats.mean_x * a / b + spectrum.header.accel_stats.mean_x * a / b ) ds_swir["acceleration_x_std"][scan_number_swir] = ( - spectrum.header.accel_stats.std_x * a / b + spectrum.header.accel_stats.std_x * a / b ) ds_swir["acceleration_y_mean"][scan_number_swir] = ( - spectrum.header.accel_stats.mean_y * a / b + spectrum.header.accel_stats.mean_y * a / b ) ds_swir["acceleration_y_std"][scan_number_swir] = ( - spectrum.header.accel_stats.std_y * a / b + spectrum.header.accel_stats.std_y * a / b ) ds_swir["acceleration_z_mean"][scan_number_swir] = ( - spectrum.header.accel_stats.mean_z * a / b + spectrum.header.accel_stats.mean_z * a / b ) ds_swir["acceleration_z_std"][scan_number_swir] = ( - spectrum.header.accel_stats.std_z * a / b + spectrum.header.accel_stats.std_z * a / b ) ds_swir["digital_number"][:, scan_number_swir] = scan scan_number_swir += 1 @@ -1269,7 +1268,7 @@ def read_metadata(self, seq_dir): # 3. Read series # --------------------------- # check for radiance and irradiance series within the metadata - series_all = metadata.sections()[1: len(metadata)] + series_all = metadata.sections()[1 : len(metadata)] seriesName = [] seriesPict = [] for i in series_all: @@ -1347,13 +1346,13 @@ def read_aux(self, seq_dir): return data["temp"], data["RH"], data["pressure"], data["lux"] def read_sequence( - self, - seq_dir, - calibration_data_rad, - calibration_data_irr, - calibration_data_swir_rad=None, - calibration_data_swir_irr=None, - single_series=None, + self, + seq_dir, + calibration_data_rad, + calibration_data_irr, + calibration_data_swir_rad=None, + calibration_data_swir_irr=None, + single_series=None, ): # define data to return none at end of method if does not exist diff --git a/hypernets_processor/data_io/hypernets_writer.py b/hypernets_processor/data_io/hypernets_writer.py index fb96cd50..b6b6557b 100755 --- a/hypernets_processor/data_io/hypernets_writer.py +++ b/hypernets_processor/data_io/hypernets_writer.py @@ -66,6 +66,23 @@ def write( else: raise IOError("The file already exists: " + path) + if self.context.get_config_value("mcsteps") < 2: + vars_with_unc_comps = [ + var + for var in ds.keys() + if "unc_comps" in ds[var].attrs and len(ds[var].attrs["unc_comps"]) > 0 + ] + for var in vars_with_unc_comps: + unc_comps = ds[var].attrs["unc_comps"] + ds[var].attrs["unc_comps"] = [] + try: + ds = ds.drop(unc_comps) + except: + continue + + err_corr_vars = [var for var in ds.keys() if "err_corr" in var] + ds = ds.drop(err_corr_vars) + if remove_vars_strings is not None: for remove_var_string in remove_vars_strings.split(","): for var_name in ds.data_vars: @@ -250,9 +267,21 @@ def _write_netcdf(ds, path, compression_level=None, encodefloat32=True): comp = dict(zlib=True, complevel=compression_level) encoding = dict() + encoding_keys = ( + "_FillValue", + "dtype", + "scale_factor", + "add_offset", + "grid_mapping", + ) for var_name in ds.data_vars: var_encoding = dict(comp) - var_encoding.update(ds[var_name].encoding) + ds_var_encoding = { + encoding_key: ds[var_name].encoding[encoding_key] + for encoding_key in encoding_keys + if encoding_key in ds[var_name].encoding + } + var_encoding.update(ds_var_encoding) if ds[var_name].values.dtype == np.float64 and encodefloat32: var_encoding["dtype"] = np.float32 if "dtype" in var_encoding.keys(): diff --git a/hypernets_processor/data_io/product_name_util.py b/hypernets_processor/data_io/product_name_util.py index 39d53723..724b9a8f 100755 --- a/hypernets_processor/data_io/product_name_util.py +++ b/hypernets_processor/data_io/product_name_util.py @@ -39,6 +39,10 @@ "W_L2A": "L2A_REF", "L_L2B": "L2B_REF", "W_L2B": "L2B_REF", + "L_L1D_RAD": "L1D_RAD", + "L_L1D_IRR": "L1D_IRR", + "W_L1D_RAD": "L1D_RAD", + "W_L1D_IRR": "L1D_IRR", "IMG": "IMG", } diff --git a/hypernets_processor/data_io/spectrum.py b/hypernets_processor/data_io/spectrum.py index 4e96d0c8..3522adc8 100755 --- a/hypernets_processor/data_io/spectrum.py +++ b/hypernets_processor/data_io/spectrum.py @@ -2,7 +2,6 @@ Spectrum class, written by Kaspars Laizans at Tartu University """ - import struct from enum import Enum @@ -112,9 +111,9 @@ def save(self, path): "Sensor temperature: {} 'C\n" "Pixel count: {}\n" "Tilt:\n" - "\tx:{}\u00B1{}\n" - "\t y:{}\u00B1{}\n" - "\t z:{}\u00B1{}\n".format( + "\tx:{}\u00b1{}\n" + "\t y:{}\u00b1{}\n" + "\t z:{}\u00b1{}\n".format( self.header.total_length, self.header.timestamp, hex(self.crc32[0]), @@ -146,9 +145,9 @@ def return_header(self): "Sensor temperature: {} 'C \t" "Pixel count: {}\n" "Tilt:\n" - "\tx:{}\u00B1{}\n" - "\t y:{}\u00B1{}\n" - "\t z:{}\u00B1{}\n".format( + "\tx:{}\u00b1{}\n" + "\t y:{}\u00b1{}\n" + "\t z:{}\u00b1{}\n".format( self.header.spectrum_type.optics.name, self.header.spectrum_type.radiometer.name, self.header.exposure_time, diff --git a/hypernets_processor/data_utils/quality_checks.py b/hypernets_processor/data_utils/quality_checks.py index 3514345c..fd2dd403 100755 --- a/hypernets_processor/data_utils/quality_checks.py +++ b/hypernets_processor/data_utils/quality_checks.py @@ -143,11 +143,11 @@ def perform_quality_check_L0A(self, datasetl0, series_ids): datasetl0["quality_flag"][np.where(mask_threshold == 1)] = DatasetUtil.set_flag( datasetl0["quality_flag"][np.where(mask_threshold == 1)], "L0_threshold" ) # for i in range(len(mask))] - datasetl0["quality_flag"][ - np.where(mask_discontinuity == 1) - ] = DatasetUtil.set_flag( - datasetl0["quality_flag"][np.where(mask_discontinuity == 1)], - "L0_discontinuity", + datasetl0["quality_flag"][np.where(mask_discontinuity == 1)] = ( + DatasetUtil.set_flag( + datasetl0["quality_flag"][np.where(mask_discontinuity == 1)], + "L0_discontinuity", + ) ) # for i in range(len(mask))] return datasetl0, mask @@ -192,7 +192,7 @@ def perform_quality_check_black(self, datasetl0, series_ids): def perform_quality_check_rand_unc(self, dataset, measurandstring): if np.count_nonzero(dataset["u_rel_random_" + measurandstring].values < 0) > 0: - self.context.anomaly_handler.add_anomaly("u") #, dataset) + self.context.anomaly_handler.add_anomaly("u") # , dataset) if ( np.count_nonzero(dataset["u_rel_random_" + measurandstring].values > 100) @@ -201,7 +201,7 @@ def perform_quality_check_rand_unc(self, dataset, measurandstring): dataset["quality_flag"][:] = DatasetUtil.set_flag( dataset["quality_flag"][:], "half_of_uncertainties_too_big" ) - self.context.anomaly_handler.add_anomaly("o") #, dataset) + self.context.anomaly_handler.add_anomaly("o") # , dataset) def perform_quality_check_comb( self, dataset_l1b, dataset_l1b_swir, measurandstring @@ -763,24 +763,24 @@ def qc_scan(self, dataset, measurandstring, dataset_l1b): if v > self.context.get_config_value("diff_threshold"): if measurandstring == "irradiance": flags[id] = 1 - dataset_l1b["quality_flag"][ - dataset_l1b["scan"] == i - ] = DatasetUtil.set_flag( - dataset_l1b["quality_flag"][ - np.where(dataset_l1b["scan"] == i) - ], - "temp_variability_irr", + dataset_l1b["quality_flag"][dataset_l1b["scan"] == i] = ( + DatasetUtil.set_flag( + dataset_l1b["quality_flag"][ + np.where(dataset_l1b["scan"] == i) + ], + "temp_variability_irr", + ) ) else: flags[id] = 1 - dataset_l1b["quality_flag"][ - dataset_l1b["scan"] == i - ] = DatasetUtil.set_flag( - dataset_l1b["quality_flag"][ - np.where(dataset_l1b["scan"] == i) - ], - "temp_variability_rad", + dataset_l1b["quality_flag"][dataset_l1b["scan"] == i] = ( + DatasetUtil.set_flag( + dataset_l1b["quality_flag"][ + np.where(dataset_l1b["scan"] == i) + ], + "temp_variability_rad", + ) ) seq = dataset.attrs["sequence_id"] diff --git a/hypernets_processor/data_utils/site_specific_quality_checks.py b/hypernets_processor/data_utils/site_specific_quality_checks.py new file mode 100644 index 00000000..8f4af46c --- /dev/null +++ b/hypernets_processor/data_utils/site_specific_quality_checks.py @@ -0,0 +1,1127 @@ +""" +Class to perform site-specific quality checks (i.e. produce L2B data)""" + +from hypernets_processor.version import __version__ +from hypernets_processor.data_io.data_templates import DataTemplates +from hypernets_processor.plotting.plotting import Plotting +from hypernets_processor.data_io.hypernets_writer import HypernetsWriter +from hypernets_processor.utils.utils import convert_datetime +from hypernets_processor.data_io.product_name_util import ProductNameUtil + +import numpy as np +import warnings +import os +import xarray as xr +import datetime +import math +from scipy.optimize import curve_fit +from scipy.interpolate import LinearNDInterpolator + +import scipy +import ast + +import punpy +from obsarray.templater.dataset_util import DatasetUtil +import matheo.band_integration as bi + +"""___Authorship___""" +__author__ = "Pieter De Vis" +__created__ = "18/12/2024" +__version__ = __version__ +__maintainer__ = "Pieter De Vis" +__email__ = "pieter.de.vis@npl.co.uk" +__status__ = "Development" + +dir_path = os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__)))) +postprocessing_path = os.path.join(dir_path, "data", "postprocessing") +irradiance_path = os.path.join(dir_path, "data", "postprocessing", "irradiance_data") +reflectance_bounds_path = os.path.join( + dir_path, "data", "postprocessing", "qc_reflectance_bounds" +) + + +class SiteSpecificQualityChecks: + def __init__(self, context): + self.context = context + self.plot = Plotting(context) + self.writer = HypernetsWriter(context) + self.pu = ProductNameUtil(context=self.context) + self.prop = punpy.MCPropagation( + self.context.get_config_value("mcsteps"), + dtype="float32", # , MCdimlast=True + ) + + def apply_site_specific_QC(self, dataset_l2a, dataset_l1b_rad, dataset_l1b_irr): + """ + + :param dataset_l2a: + :param dataset_l1b_rad: + :param dataset_l1b_irr: + :return: + """ + + # first, we extract all the relevant config values + deploy_periods = ast.literal_eval( + self.context.get_config_value("deployment_periods") + ) + bad_sequences_period = ast.literal_eval( + self.context.get_config_value("bad_sequences_period") + ) + limit_tod_period = ast.literal_eval( + self.context.get_config_value("limit_tod_period") + ) + max_sza_period = ast.literal_eval( + self.context.get_config_value("max_sza_period") + ) + bad_viewing_angles_period = ast.literal_eval( + self.context.get_config_value("bad_viewing_angles_period") + ) + bad_solar_angles_period = ast.literal_eval( + self.context.get_config_value("bad_solar_angles_period") + ) + bad_relative_angles_period = ast.literal_eval( + self.context.get_config_value("bad_viewing_angles_period") + ) + bad_wavelengths_period = ast.literal_eval( + self.context.get_config_value("bad_wavelengths_period") + ) + postprocessing_qc_file_period = ast.literal_eval( + self.context.get_config_value("postprocessing_qc_file_period") + ) + + misalignment_vza = ast.literal_eval( + self.context.get_config_value("misalignment_vza") + ) + misalignment_vaa = ast.literal_eval( + self.context.get_config_value("misalignment_vaa") + ) + + misalignment_vza_unc = ast.literal_eval( + self.context.get_config_value("misalignment_vza_unc") + ) + misalignment_vaa_unc = ast.literal_eval( + self.context.get_config_value("misalignment_vaa_unc") + ) + + misalignment_corr = ast.literal_eval( + self.context.get_config_value("misalignment_corr") + ) + misalignment_corr_unc = ast.literal_eval( + self.context.get_config_value("misalignment_corr_unc") + ) + + # we loop through each of the different deployment periods and identify the applicable one for this sequence + seq_within_period = False + i_dep_save = None + for i_dep in range(len(deploy_periods)): + # first, we check if this data is within the deployment ranges: + if convert_datetime( + dataset_l2a.acquisition_time.values.min() + ) > convert_datetime( + deploy_periods[i_dep]["start_date"] + ) and convert_datetime( + dataset_l2a.acquisition_time.values.max() + ) < convert_datetime( + deploy_periods[i_dep]["stop_date"] + ): + seq_within_period = True + i_dep_save = i_dep + + # anomaly is raised if data has not been found within deployment periods + if not seq_within_period: + self.context.anomaly_handler.add_anomaly("per") + + # anomaly is raised if data is within deployment period that is not valid for satellite validation + if not deploy_periods[i_dep_save]["valid"]: + self.context.anomaly_handler.add_anomaly("val") + + # anomaly is raised if data is outside time of day limits + if limit_tod_period[i_dep_save] is not None and ( + ( + convert_datetime(dataset_l2a.acquisition_time.values.max()).time() + < datetime.time.fromisoformat(limit_tod_period[i_dep_save][0]) + ) + or ( + convert_datetime(dataset_l2a.acquisition_time.values.min()).time() + > datetime.time.fromisoformat(limit_tod_period[i_dep_save][1]) + ) + ): + self.context.anomaly_handler.add_anomaly("tod") + + # anomaly is raised if system id does not match hypstar SN in deployment period + if not dataset_l2a.attrs["system_id"] == "HYPSTAR_" + str( + deploy_periods[i_dep_save]["HYPSTAR_SN"] + ): + self.context.anomaly_handler.add_anomaly("hsn") + + # bad sequences that were manually specified are removed + if dataset_l2a.attrs["sequence_id"] in bad_sequences_period[i_dep_save]: + self.context.anomaly_handler.add_anomaly("man") + + if len(dataset_l2a.series)!=len(dataset_l1b_rad.series): + self.context.anomaly_handler.add_anomaly("wns") + + # next, remove angles for which we know the data is not reliable + id_series_valid = np.where( + dataset_l2a.solar_zenith_angle.values < max_sza_period[i_dep_save] + )[0] + dataset_l2b = dataset_l2a.isel(series=id_series_valid) + dataset_l1d_rad = dataset_l1b_rad.isel(series=id_series_valid) + + ang_tol = self.context.get_config_value("angle_tolerance") + for angle_tup in bad_viewing_angles_period[i_dep_save]: + bad_vza, bad_vaa = angle_tup + if bad_vza == "all" or bad_vza == "*": + cond_vza = np.zeros_like( + dataset_l2b.viewing_zenith_angle.values, dtype=bool + ) + elif isinstance(bad_vza, str) and bad_vza[0] == "<": + cond_vza = dataset_l2b.viewing_zenith_angle.values >= float(bad_vza[1:]) + elif isinstance(bad_vza, str) and bad_vza[0] == ">": + cond_vza = dataset_l2b.viewing_zenith_angle.values <= float(bad_vza[1:]) + else: + cond_vza = ( + np.abs(dataset_l2b.viewing_zenith_angle.values - bad_vza) > ang_tol + ) + + if bad_vaa == "all" or bad_vaa == "*": + cond_vaa = np.zeros_like( + dataset_l2b.viewing_azimuth_angle.values, dtype=bool + ) + elif isinstance(bad_vaa, str) and bad_vaa[0] == "<": + cond_vaa = dataset_l2b.viewing_azimuth_angle.values >= float( + bad_vaa[1:] + ) + elif isinstance(bad_vaa, str) and bad_vaa[0] == ">": + cond_vaa = dataset_l2b.viewing_azimuth_angle.values <= float( + bad_vaa[1:] + ) + else: + cond_vaa = ( + np.abs(dataset_l2b.viewing_azimuth_angle.values - bad_vaa) > ang_tol + ) + + # we only keep data where one of the two conditions is met (the vza or the vaa is good) + id_series_valid = np.where(cond_vza | cond_vaa)[0] + dataset_l2b = dataset_l2b.isel(series=id_series_valid) + dataset_l1d_rad = dataset_l1d_rad.isel(series=id_series_valid) + + for angle_tup in bad_solar_angles_period[i_dep_save]: + bad_sza, bad_saa = angle_tup + if bad_sza == "all" or bad_sza == "*": + cond_sza = np.zeros_like( + dataset_l2b.solar_zenith_angle.values, dtype=bool + ) + elif isinstance(bad_sza, str) and bad_sza[0] == "<": + cond_sza = dataset_l2b.solar_zenith_angle.values >= float(bad_sza[1:]) + elif isinstance(bad_sza, str) and bad_sza[0] == ">": + cond_sza = dataset_l2b.solar_zenith_angle.values <= float(bad_sza[1:]) + else: + cond_sza = ( + np.abs(dataset_l2b.solar_zenith_angle.values - bad_sza) > ang_tol + ) + + if bad_saa == "all" or bad_saa == "*": + cond_saa = np.zeros_like( + dataset_l2b.solar_azimuth_angle.values, dtype=bool + ) + elif isinstance(bad_saa, str) and bad_saa[0] == "<": + cond_saa = dataset_l2b.solar_azimuth_angle.values >= float(bad_vaa[1:]) + elif isinstance(bad_saa, str) and bad_saa[0] == ">": + cond_saa = dataset_l2b.solar_azimuth_angle.values <= float(bad_vaa[1:]) + else: + cond_saa = ( + np.abs(dataset_l2b.solar_azimuth_angle.values - bad_saa) > ang_tol + ) + + # we only keep data where one of the two conditions is met (the sza or the saa is good) + id_series_valid = np.where(cond_sza | cond_saa)[0] + dataset_l2b = dataset_l2b.isel(series=id_series_valid) + dataset_l1d_rad = dataset_l1d_rad.isel(series=id_series_valid) + + raa_ang_tol = self.context.get_config_value("raa_angle_tolerance") + for angle_tup in bad_relative_angles_period[i_dep_save]: + bad_vza, bad_raa = angle_tup + raa = ( + dataset_l2b.viewing_azimuth_angle.values + - dataset_l2b.solar_azimuth_angle.values + ) % 360 + + if bad_vza == "all" or bad_vza == "*": + cond_vza = np.zeros_like( + dataset_l2b.viewing_zenith_angle.values, dtype=bool + ) + elif bad_vza == "sza": + cond_vza = dataset_l2b.viewing_zenith_angle.values > ( + dataset_l2b.solar_zenith_angle.values + ang_tol + ) + elif isinstance(bad_vza, str) and bad_vza[0] == "<": + cond_vza = dataset_l2b.viewing_zenith_angle.values >= float(bad_vza[1:]) + elif isinstance(bad_vza, str) and bad_vza[0] == ">": + cond_vza = dataset_l2b.viewing_zenith_angle.values <= float(bad_vza[1:]) + else: + cond_vza = ( + np.abs(dataset_l2b.viewing_zenith_angle.values - bad_vza) > ang_tol + ) + + if bad_raa == "all" or bad_raa == "*": + cond_raa = np.zeros_like(raa, dtype=bool) + elif isinstance(bad_raa, str) and bad_raa[0] == "<": + cond_raa = raa >= float(bad_raa[1:]) + elif isinstance(bad_raa, str) and bad_raa[0] == ">": + cond_raa = raa <= float(bad_raa[1:]) + else: + cond_raa = np.abs(raa - bad_raa) % 360 > raa_ang_tol + + # we only keep data where one of the two conditions is met (the vza or the raa is good) + id_series_valid = np.where(cond_vza | cond_raa)[0] + dataset_l2b = dataset_l2b.isel(series=id_series_valid) + dataset_l1d_rad = dataset_l1d_rad.isel(series=id_series_valid) + + if len(dataset_l2b.series.values) == 0: + self.context.anomaly_handler.add_anomaly("nos") + + #store number of angles after removing the bad angle ranges + len_angles_removed = len(dataset_l2b.series.values) + + # Next, remove data for which any of the bad flags was set in previous QC + bad_flags = [ + "pt_ref_invalid", + "half_of_scans_masked", + "not_enough_dark_scans", + "not_enough_rad_scans", + "not_enough_irr_scans", + "no_clear_sky_irradiance", + "variable_irradiance", + "half_of_uncertainties_too_big", + "discontinuity_VNIR_SWIR", + "single_irradiance_used", + ] + flagged = DatasetUtil.get_flags_mask_or( + dataset_l2b["quality_flag"], bad_flags + ) # bools for each series if any bad flag is set + id_series_valid = np.where(~flagged)[ + 0 + ] # select indexes for which no bad flags are set + dataset_l2b = dataset_l2b.isel(series=id_series_valid) + dataset_l2b.attrs["product_name"] = self.pu.create_product_name("L_L2B") + dataset_l2b.attrs["product_level"] = "L_L2B" + + dataset_l1d_rad = dataset_l1d_rad.isel(series=id_series_valid) + dataset_l1d_rad.attrs["product_name"] = self.pu.create_product_name("L_L1D_RAD") + dataset_l1d_rad.attrs["product_level"] = "L_L1D_RAD" + + flagged_l1b_irr = DatasetUtil.get_flags_mask_or( + dataset_l1b_irr["quality_flag"], bad_flags + ) # bools for each series if any bad flag is set + id_series_valid_l1b_irr = np.where(~flagged_l1b_irr)[ + 0 + ] # select indexes for which no bad flags are set + dataset_l1d_irr = dataset_l1b_irr.isel(series=id_series_valid_l1b_irr) + dataset_l1d_irr.attrs["product_name"] = self.pu.create_product_name("L_L1D_IRR") + dataset_l1d_irr.attrs["product_level"] = "L_L1D_IRR" + + + + # Then, bad wavelength ranges are omited + for bad_wav in bad_wavelengths_period[i_dep_save]: + id_wav_valid = np.where( + (dataset_l2b.wavelength.values < bad_wav[0]) + | (dataset_l2b.wavelength.values > bad_wav[1]) + )[0] + dataset_l2b = dataset_l2b.isel(wavelength=id_wav_valid) + dataset_l1d_rad = dataset_l1d_rad.isel(wavelength=id_wav_valid) + + # Next, the site-specific clear sky check is applied + irr_model_irrwav = xr.open_dataset( + os.path.join( + irradiance_path, + "%s_clear_sky_medianaod_irrwav.nc" + % (self.context.get_config_value("site_id")), + ) + ) + + irr_model_irrwav_noaer = xr.open_dataset( + os.path.join( + irradiance_path, + "%s_clear_sky_aod0.0_irrwav.nc" + % (self.context.get_config_value("site_id")), + ) + ) + + i_wav_550_data = np.argmin(np.abs(dataset_l1b_irr.wavelength.values - 550)) + i_wav_550_model = np.argmin(np.abs(irr_model_irrwav.wavelength.values - 550)) + i_wav_550_model_noaer = np.argmin( + np.abs(irr_model_irrwav_noaer.wavelength.values - 550) + ) + + for i_series in range(len(dataset_l1b_irr.series.values)): + sza = dataset_l1b_irr.solar_zenith_angle.values[i_series] + saa = dataset_l1b_irr.solar_azimuth_angle.values[i_series] + irradiance, dir_dif_ratio = self.interpolate_irradiance_sza( + sza, irr_model_irrwav + ) + irradiance_noaer, _ = self.interpolate_irradiance_sza( + sza, irr_model_irrwav_noaer + ) + + # before performing the clear sky check, we perform correction for misalignment + if misalignment_vza[i_dep_save] is not None: + dir_dif_intfunc = scipy.interpolate.interp1d( + irr_model_irrwav.wavelength.values, + dir_dif_ratio, + fill_value="extrapolate", + ) + dir_dif_ratio = dir_dif_intfunc(dataset_l1b_irr.wavelength.values) + ratio = self.misalignment_ratio_calculator( + misalignment_vza[i_dep_save], + misalignment_vaa[i_dep_save], + 0, + sza, + saa, + misalignment_corr[i_dep_save], + dir_dif_ratio, + ) + ratio_unc = self.prop.propagate_systematic( + self.misalignment_ratio_calculator, + [ + misalignment_vza[i_dep_save], + misalignment_vaa[i_dep_save], + 0, + sza, + saa, + misalignment_corr[i_dep_save], + dir_dif_ratio, + ], + [ + misalignment_vza_unc[i_dep_save], + misalignment_vaa_unc[i_dep_save], + None, + None, + None, + misalignment_corr_unc[i_dep_save], + None, + ], + ) + dataset_l1b_irr.irradiance.values[:, i_series] *= ratio + dataset_l1b_irr.u_rel_systematic_indep_irradiance.values[:, i_series] = ( + dataset_l1b_irr.u_rel_systematic_indep_irradiance.values[:, i_series] + ** 2 + + (ratio_unc / ratio / 100) ** 2 + ) ** 0.5 + + # clear sky check is performed + if ( + dataset_l1b_irr.irradiance.values[i_wav_550_data, i_series] + < 0.9 * irradiance[i_wav_550_model] + ) or ( + dataset_l1b_irr.irradiance.values[i_wav_550_data, i_series] + > 1.05 * irradiance_noaer[i_wav_550_model_noaer] + ): + self.context.anomaly_handler.add_anomaly("scl") + + # perform correction for reflectance (to account for change in irradiance) + if misalignment_vza[i_dep_save] is not None: + for i_series in range(len(dataset_l2b.series.values)): + sza = dataset_l2b.solar_zenith_angle.values[i_series] + saa = dataset_l2b.solar_azimuth_angle.values[i_series] + irradiance, dir_dif_ratio = self.interpolate_irradiance_sza( + sza, irr_model_irrwav + ) + dir_dif_intfunc = scipy.interpolate.interp1d( + irr_model_irrwav.wavelength.values, + dir_dif_ratio, + fill_value="extrapolate", + ) + dir_dif_ratio = dir_dif_intfunc(dataset_l2b.wavelength.values) + + # Perform correction for misalignment + ratio = self.misalignment_ratio_calculator( + misalignment_vza[i_dep_save], + misalignment_vaa[i_dep_save], + 0, + sza, + saa, + misalignment_corr[i_dep_save], + dir_dif_ratio, + ) + ratio_unc = self.prop.propagate_systematic( + self.misalignment_ratio_calculator, + [ + misalignment_vza[i_dep_save], + misalignment_vaa[i_dep_save], + 0, + sza, + saa, + misalignment_corr[i_dep_save], + dir_dif_ratio, + ], + [ + misalignment_vza_unc[i_dep_save], + misalignment_vaa_unc[i_dep_save], + None, + None, + None, + misalignment_corr_unc[i_dep_save], + None, + ], + ) + dataset_l2b.reflectance.values[:, i_series] /= ratio + dataset_l2b.u_rel_systematic_reflectance.values[:, i_series] = ( + dataset_l2b.u_rel_systematic_reflectance.values[:, i_series] ** 2 + + (ratio_unc / ratio / 100) ** 2 + ) ** 0.5 + + # next, we check if the reflectances are within the bounds + if postprocessing_qc_file_period[i_dep_save] is None: + ds_bounds = None + else: + ds_bounds = xr.open_dataset( + os.path.join( + reflectance_bounds_path, postprocessing_qc_file_period[i_dep_save] + ) + ) + raas = [ + np.array(string.split("_")).astype(float).mean() + for string in ds_bounds.raa.values + ] + raas[-1] = 360 + ds_bounds = ds_bounds.assign_coords(raa=raas) + i_550 = np.argmin(np.abs(dataset_l2b.wavelength.values - 550)) + + sza = dataset_l2b.solar_zenith_angle.values + saa = dataset_l2b.solar_azimuth_angle.values + vza = dataset_l2b.viewing_zenith_angle.values + vaa = dataset_l2b.viewing_azimuth_angle.values + + bounds_down, bounds_up = self.calculate_bounds(ds_bounds, sza, saa, vza, vaa) + id_series_valid = np.where( + (dataset_l2b.reflectance.values[i_550, :] < bounds_up) + & (dataset_l2b.reflectance.values[i_550, :] > bounds_down) + )[0] + dataset_l2b = dataset_l2b.isel(series=id_series_valid) + dataset_l1d_rad = dataset_l1d_rad.isel(series=id_series_valid) + + if len(dataset_l2b.series.values) == 0: + self.context.anomaly_handler.add_anomaly("nos") + + if len(dataset_l2b.series.values) < 1 / 2 * len_angles_removed: + self.context.anomaly_handler.add_anomaly("hos") + + # finally, make plots and write file + if self.context.get_config_value("write_l2b"): + self.writer.write( + dataset_l2b, + overwrite=True, + remove_vars_strings=self.context.get_config_value( + "remove_vars_strings_L2" + ), + ) + + if self.context.get_config_value("write_l1d"): + self.writer.write( + dataset_l1d_rad, + overwrite=True, + remove_vars_strings=self.context.get_config_value( + "remove_vars_strings" + ), + ) + self.writer.write( + dataset_l1d_irr, + overwrite=True, + remove_vars_strings=self.context.get_config_value( + "remove_vars_strings" + ), + ) + + if self.context.get_config_value("plot_l2b"): + self.plot.plot_series_in_sequence("reflectance", dataset_l2b) + self.plot.plot_series_in_sequence_vaa("reflectance", dataset_l2b, 98) + self.plot.plot_series_in_sequence_vza("reflectance", dataset_l2b, 30) + if self.context.get_config_value("plot_polar_wav") is not None: + self.plot.plot_polar_reflectance( + dataset_l2b, self.context.get_config_value("plot_polar_wav") + ) + if self.context.get_config_value("plot_polar_ndvi"): + self.plot.plot_polar_reflectance(dataset_l2b, "ndvi") + if self.context.get_config_value("plot_uncertainty"): + self.plot.plot_relative_uncertainty( + "reflectance", dataset_l2b, refl=True + ) + + if self.context.get_config_value("plot_correlation"): + self.plot.plot_correlation("reflectance", dataset_l2b, refl=True) + + if self.context.get_config_value("plot_l1d"): + self.plot.plot_series_in_sequence("radiance", dataset_l1d_rad) + self.plot.plot_series_in_sequence("irradiance", dataset_l1d_irr) + self.plot.plot_series_in_sequence_vaa("radiance", dataset_l1d_rad, 98) + self.plot.plot_series_in_sequence_vza("radiance", dataset_l1d_rad, 30) + + if self.context.get_config_value("plot_uncertainty"): + self.plot.plot_relative_uncertainty("radiance", dataset_l1d_rad) + self.plot.plot_relative_uncertainty("irradiance", dataset_l1d_irr) + + if self.context.get_config_value("plot_correlation"): + self.plot.plot_correlation("radiance", dataset_l1d_rad) + self.plot.plot_correlation("irradiance", dataset_l1d_irr) + + return dataset_l2b, dataset_l1d_rad, dataset_l1d_irr + + def interpolate_irradiance_sza(self, sza, ds_irr): + ds_irr_temp = ds_irr.copy() + ds_irr_temp["solar_irradiance_BOA"].values = ( + ds_irr_temp["solar_irradiance_BOA"].values + / np.cos(ds_irr_temp["sza"].values / 180 * np.pi)[:, None] + ) + ds_irr_temp["direct_to_diffuse_irradiance_ratio"].values = ( + ds_irr_temp["direct_to_diffuse_irradiance_ratio"].values + / np.cos(ds_irr_temp["sza"].values / 180 * np.pi)[:, None] + ) + ds_irr_temp = ds_irr_temp.interp(sza=sza, method="linear") + ds_irr_temp["solar_irradiance_BOA"].values = ds_irr_temp[ + "solar_irradiance_BOA" + ].values * np.cos(ds_irr_temp["sza"].values / 180 * np.pi) + ds_irr_temp["direct_to_diffuse_irradiance_ratio"].values = ds_irr_temp[ + "direct_to_diffuse_irradiance_ratio" + ].values * np.cos(ds_irr_temp["sza"].values / 180 * np.pi) + return ( + ds_irr_temp["solar_irradiance_BOA"].values, + ds_irr_temp["direct_to_diffuse_irradiance_ratio"].values, + ) + + def perform_quality_check_angles( + self, datasetl0, scan_number, vza_abs, vza_ref, paa_abs, paa_ref + ): + self.context.logger.debug( + "vza_ref:{}, vza_abs:{}, paa_ref:{}, paa_abs:{}".format( + vza_ref, vza_abs, paa_ref, paa_abs + ) + ) + + if (vza_ref == -1 and paa_ref == -1) or (vza_ref <= -999 and paa_ref <= -999): + self.context.logger.debug( + "vza_ref and paa_ref are both invalid, using pt_abs instead" + ) + paa_ref, vza_ref = paa_abs, vza_abs + datasetl0["quality_flag"].values[scan_number] = DatasetUtil.set_flag( + datasetl0["quality_flag"][scan_number], "pt_ref_invalid" + ) + + angacc_vza = abs(vza_abs - vza_ref) + angacc_vaa = abs(paa_abs - paa_ref) + + if angacc_vza > 180: + angacc_vza = 360 - angacc_vza + + if angacc_vaa > 180: + angacc_vaa = 360 - angacc_vaa + + self.context.logger.debug( + "Angle accuracy vza {:.4f} ={:.4f}-{:.4f}".format( + angacc_vza, vza_abs, vza_ref + ) + ) + self.context.logger.debug( + "Angle accuracy vaa {:.4f} ={:.4f}-{:.4f}".format( + angacc_vaa, paa_abs, paa_ref + ) + ) + + if angacc_vza > self.context.get_config_value("bad_pointing_threshold_zenith"): + datasetl0["quality_flag"].values[scan_number] = DatasetUtil.set_flag( + datasetl0["quality_flag"][scan_number], "bad_pointing" + ) + self.context.logger.warning( + "Error in Accuracy of tilt is above %s° (vza_abs=%s; vza_ref=%s). Check your system and/or data before processing." + % ( + self.context.get_config_value("bad_pointing_threshold_zenith"), + vza_abs, + vza_ref, + ) + ) + self.context.anomaly_handler.add_anomaly("a") + + if angacc_vaa > self.context.get_config_value("bad_pointing_threshold_azimuth"): + datasetl0["quality_flag"].values[scan_number] = DatasetUtil.set_flag( + datasetl0["quality_flag"][scan_number], "bad_pointing" + ) + self.context.logger.warning( + "Error in Accuracy of pan is above %s° (vaa_abs=%s; vaa_ref=%s). Check your system and/or data before processing." + % ( + self.context.get_config_value("bad_pointing_threshold_azimuth"), + paa_abs, + paa_ref, + ) + ) + self.context.anomaly_handler.add_anomaly("a") + + return datasetl0 + + def perform_quality_check_L0A(self, datasetl0, series_ids): + mask = [] + mask_threshold = [] + mask_outliers = [] + mask_discontinuity = [] + raise_mask_anomaly = False + for i in range(len(series_ids)): + ids = np.where(datasetl0["series_id"] == series_ids[i])[0] + data_subset = datasetl0["digital_number"].values[:, ids] + intsig = np.nanmean(data_subset, axis=0) + mask_all_i = np.zeros_like(intsig) # mask the columns that have NaN + mask_threshold_i = self.threshold_checks(data_subset) + mask_outliers_i = self.outlier_checks(data_subset) + mask_discontinuity_i = self.discontinuity_checks(data_subset) + mask_all_i[np.where(mask_threshold_i == 1)] = 1 + mask_all_i[np.where(mask_outliers_i == 1)] = 1 + mask_all_i[np.where(mask_discontinuity_i == 1)] = 1 + + if all(mask_all_i == 1): + self.context.logger.warning( + "None of the scans for series passed the quality control criteria" + ) + raise_mask_anomaly = True + + mask = np.append(mask, mask_all_i) + mask_threshold = np.append(mask_threshold, mask_threshold_i) + mask_outliers = np.append(mask_outliers, mask_outliers_i) + mask_discontinuity = np.append(mask_discontinuity, mask_discontinuity_i) + datasetl0["quality_flag"][np.where(mask_outliers == 1)] = DatasetUtil.set_flag( + datasetl0["quality_flag"][np.where(mask_outliers == 1)], "outliers" + ) # for i in range(len(mask))] + datasetl0["quality_flag"][np.where(mask_threshold == 1)] = DatasetUtil.set_flag( + datasetl0["quality_flag"][np.where(mask_threshold == 1)], "L0_threshold" + ) # for i in range(len(mask))] + datasetl0["quality_flag"][np.where(mask_discontinuity == 1)] = ( + DatasetUtil.set_flag( + datasetl0["quality_flag"][np.where(mask_discontinuity == 1)], + "L0_discontinuity", + ) + ) # for i in range(len(mask))] + + return datasetl0, mask + + def perform_quality_check_black(self, datasetl0, series_ids): + mask = [] + mask_outliers = [] + mask_threshold = [] + mask_discontinuity = [] + raise_mask_anomaly = False + for i in range(len(series_ids)): + ids = np.where(datasetl0["series_id"] == series_ids[i])[0] + + data_subset = datasetl0["digital_number"].values[:, ids] + intsig = np.nanmean(data_subset, axis=0) + mask_all_i = np.zeros_like(intsig) # mask the columns that have NaN + mask_threshold_i = self.threshold_checks(data_subset) + mask_outliers_i = self.outlier_checks(data_subset) + mask_discontinuity_i = self.discontinuity_checks(data_subset) + mask_all_i[np.where(mask_threshold_i == 1)] = 1 + mask_all_i[np.where(mask_outliers_i == 1)] = 1 + # mask_all_i[np.where(mask_discontinuity_i==1)] = 1 + + if all(mask_all_i == 1): + self.context.logger.warning( + "None of the dark scans for series passed the quality control criteria" + ) + raise_mask_anomaly = True + + mask = np.append(mask, mask_all_i) + mask_threshold = np.append(mask_threshold, mask_threshold_i) + mask_outliers = np.append(mask_outliers, mask_outliers_i) + datasetl0["quality_flag"][np.where(mask == 1)] = DatasetUtil.set_flag( + datasetl0["quality_flag"][np.where(mask == 1)], "dark_masked" + ) + # datasetl0["quality_flag"][np.where(mask_threshold == 1)] = DatasetUtil.set_flag( + # datasetl0["quality_flag"][np.where(mask_threshold == 1)], "L0_threshold" + # ) # for i in range(len(mask))] + + return datasetl0, mask + + def perform_quality_check_rand_unc(self, dataset, measurandstring): + + if np.count_nonzero(dataset["u_rel_random_" + measurandstring].values < 0) > 0: + self.context.anomaly_handler.add_anomaly("u") # , dataset) + + if ( + np.count_nonzero(dataset["u_rel_random_" + measurandstring].values > 100) + > 0.5 * dataset["u_rel_random_" + measurandstring].values.size + ): + dataset["quality_flag"][:] = DatasetUtil.set_flag( + dataset["quality_flag"][:], "half_of_uncertainties_too_big" + ) + self.context.anomaly_handler.add_anomaly("o") # , dataset) + + def perform_quality_check_comb( + self, dataset_l1b, dataset_l1b_swir, measurandstring + ): + wav_range = 20 + + if len(dataset_l1b["series_id"]) != len(dataset_l1b_swir["series_id"]): + + id_series_swir = np.concatenate( + [ + np.where( + dataset_l1b_swir["series_id"].values + == dataset_l1b["series_id"].values[i] + )[0] + for i in range(len(dataset_l1b["series_id"])) + ] + ) + + id_series = np.concatenate( + [ + np.where( + dataset_l1b["series_id"].values + == dataset_l1b_swir["series_id"].values[i] + )[0] + for i in range(len(dataset_l1b_swir["series_id"])) + ] + ) + + dataset_l1b_swir = dataset_l1b_swir.isel(series=id_series_swir) + dataset_l1b = dataset_l1b.isel(series=id_series) + + for i in range(len(dataset_l1b["series_id"])): + if dataset_l1b["series_id"][i] != dataset_l1b_swir["series_id"][i]: + raise ValueError("Series ID of VNIR and SWIR should be the same!") + + idwav = np.where( + (dataset_l1b["wavelength"].values < 1000) + & (dataset_l1b["wavelength"].values > (1000 - wav_range)) + )[0] + refl_VNIR_edge = np.mean(dataset_l1b[measurandstring][idwav, i]) + idwav_swir = np.where( + (dataset_l1b_swir["wavelength"].values > 1000) + & (dataset_l1b_swir["wavelength"].values < (1000 + wav_range)) + )[0] + refl_SRIW_edge = np.mean(dataset_l1b_swir[measurandstring][idwav_swir, i]) + if ( + 2 + * np.abs(refl_VNIR_edge - refl_SRIW_edge) + / (refl_VNIR_edge + refl_SRIW_edge) + > self.context.get_config_value("vnir_swir_discontinuity_percent") / 100 + ): + dataset_l1b["quality_flag"][i] = DatasetUtil.set_flag( + dataset_l1b["quality_flag"][i], "discontinuity_VNIR_SWIR" + ) + dataset_l1b_swir["quality_flag"][i] = DatasetUtil.set_flag( + dataset_l1b_swir["quality_flag"][i], "discontinuity_VNIR_SWIR" + ) + return dataset_l1b, dataset_l1b_swir + + def outlier_checks(self, data_subset, k_unc=3): + intsig = np.nanmean(data_subset, axis=0) + mask = np.zeros_like(intsig) # mask the columns that have NaN + if len(intsig) > 1: + noisestd, noiseavg = self.sigma_clip(intsig) + mask[np.where(np.abs(intsig - noiseavg) >= k_unc * noisestd)] = 1 + mask[np.where(np.abs(intsig - noiseavg) >= 0.25 * intsig)] = 1 + return mask + + def threshold_checks(self, data_subset): + mask = np.zeros_like(data_subset[0]) # mask the columns that have NaN + for i in range(len(data_subset[0])): + if ( + self.context.get_config_value("l0_threshold") + and np.any( + data_subset[:, i] > self.context.get_config_value("l0_threshold") + ) + ) or np.any(data_subset[:, i] < 0): + mask[i] = 1 + return mask + + def discontinuity_checks(self, data_subset): + diff = abs(data_subset[1:, :].astype(int) - data_subset[:-1, :].astype(int)) + mask = np.zeros_like(diff[0]) # mask the columns that have NaN + for i in range(len(diff[0])): + if self.context.get_config_value("l0_discontinuity") and np.any( + diff[:, i] > self.context.get_config_value("l0_discontinuity") + ): + mask[i] = 1 + return mask + + def sigma_clip(self, values, tolerance=0.01, median=True, sigma_thresh=3.0): + # Remove NaNs from input values + values = np.array(values) + values = values[np.where(np.isnan(values) == False)] + values_original = np.copy(values) + + # Continue loop until result converges + diff = 10e10 + while diff > tolerance: + # Assess current input iteration + if median == False: + average = np.mean(values) + elif median == True: + average = np.median(values) + sigma_old = np.std(values) + + # Mask those pixels that lie more than 3 stdev away from mean + check = np.zeros([len(values)]) + check[np.where(values > (average + (sigma_thresh * sigma_old)))] = 1 + # check[ np.where( values<(average-(sigma_thresh*sigma_old)) ) ] = 1 + values = values[np.where(check < 1)] + + # Re-measure sigma and test for convergence + sigma_new = np.std(values) + diff = abs(sigma_old - sigma_new) / sigma_old + + # Perform final mask + check = np.zeros([len(values)]) + check[np.where(values > (average + (sigma_thresh * sigma_old)))] = 1 + check[np.where(values < (average - (sigma_thresh * sigma_old)))] = 1 + values = values[np.where(check < 1)] + + # Return results + return sigma_new, average + + # Water based quality checks includes: + # 1. qc_similarity + # 2. qc_birds + # 3. qc_scan: difference at 550 nm (wavelength ref) < 25% (diff_threshold) with neighbours -> if triggered 'temporal variability in Lu, Ed or Ld" + def qc_similarity(self, L1c): + + wave = L1c["wavelength"] + wr = L1c.attrs["similarity_waveref"] + wp = L1c.attrs["similarity_wavethres"] + + epsilon = L1c["epsilon"] + ## get pixel index for wavelength + irefr, wrefr = self.rhymershared.closest_idx(wave, wr) + + failSimil = [] + scans = L1c["scan"] + for i in range(len(scans)): + data = L1c["reflectance_nosc"].sel(scan=i).values + if abs(epsilon[i]) > wp * data[irefr]: + failSimil.append(1) + else: + failSimil.append(0) + return failSimil + + def fitcurve(self, wv, ld, ed): + def func(x, a, b): + return a + b * (x / 100) ** (-4) + + if ld.ndim > 1: + y = np.mean(ld / ed, axis=1) + else: + y = ld / ed + popt, pcov = curve_fit(func, wv, y) + residuals = y - func(wv, *popt) + ss_res = np.sum(residuals**2) + return (popt, pcov, ss_res) + + def qc_bird(self, l1c): + ld = np.nanmean(l1c.downwelling_radiance.values, axis=1) + ed = np.mean(l1c.irradiance.values, axis=1) + wv = l1c.wavelength.values + popt, pcov, ss_res = self.fitcurve(wv, ld, ed) + # print("this is ss_res:{}".format(ss_res)) + # popt[0]+popt[1]*(x/100)**(-4) + # plt.plot(wv, popt[0] + popt[1] * (wv / 100) ** (-4), label="Fitted Curve") + + # sum of squares regression + # sum of the differences between the predicted value by the model and the mean of the dependent variable + l1c.attrs["ss_res"] = str(ss_res) + return l1c + + def qc_scan(self, dataset, measurandstring, dataset_l1b): + ## no inclination + ## difference at 550 nm < 25% with neighbours + ## + ## QV July 2018 + ## Last modifications: 2019-07-10 (QV) renamed from PANTR, integrated in rhymer + # Modified 10/09/2020 by CG for the PANTHYR + verbosity = self.context.get_config_value("verbosity") + series_id = np.unique(dataset["series_id"]) + wave = dataset["wavelength"].values + flags = np.zeros(shape=len(dataset["scan"])) + id = 0 + for s in series_id: + + scans = dataset["scan"][dataset["series_id"] == s] + + ## + n = len(scans) + ## get pixel index for wavelength + iref, wref = self.rhymershared.closest_idx( + wave, self.context.get_config_value("diff_wave") + ) + + cos_sza = [] + for i in dataset["solar_zenith_angle"].sel(scan=scans).values: + cos_sza.append(math.cos(math.radians(i))) + + ## go through the current set of scans + for i in range(n): + ## test inclination + ## not done + + if measurandstring == "irradiance": + data = dataset["irradiance"].sel(scan=scans).T.values + ## test variability at 550 nm + if i == 0: + v = abs( + 1 + - ( + (data[i][iref] / cos_sza[i]) + / (data[i + 1][iref] / cos_sza[i + 1]) + ) + ) + elif i < n - 1: + v = max( + abs( + 1 + - ( + (data[i][iref] / cos_sza[i]) + / (data[i + 1][iref] / cos_sza[i + 1]) + ) + ), + abs( + 1 + - ( + (data[i][iref] / cos_sza[i]) + / (data[i - 1][iref] / cos_sza[i - 1]) + ) + ), + ) + else: + v = abs( + 1 + - ( + (data[i][iref] / cos_sza[i]) + / (data[i - 1][iref] / cos_sza[i - 1]) + ) + ) + else: + data = dataset["radiance"].sel(scan=scans).T.values + ## test variability at 550 nm + if i == 0: + v = abs(1 - (data[i][iref] / data[i + 1][iref])) + elif i < n - 1: + v = max( + abs(1 - (data[i][iref] / data[i + 1][iref])), + abs(1 - (data[i][iref] / data[i - 1][iref])), + ) + else: + v = abs(1 - (data[i][iref] / data[i - 1][iref])) + + ## continue if value exceeds the cv threshold + if v > self.context.get_config_value("diff_threshold"): + if measurandstring == "irradiance": + flags[id] = 1 + dataset_l1b["quality_flag"][dataset_l1b["scan"] == i] = ( + DatasetUtil.set_flag( + dataset_l1b["quality_flag"][ + np.where(dataset_l1b["scan"] == i) + ], + "temp_variability_irr", + ) + ) + + else: + flags[id] = 1 + dataset_l1b["quality_flag"][dataset_l1b["scan"] == i] = ( + DatasetUtil.set_flag( + dataset_l1b["quality_flag"][ + np.where(dataset_l1b["scan"] == i) + ], + "temp_variability_rad", + ) + ) + + seq = dataset.attrs["sequence_id"] + ts = datetime.datetime.utcfromtimestamp( + dataset["acquisition_time"][i] + ) + + if verbosity > 2: + self.context.logger.info( + "Temporal jump: in {}: Aquisition time {}:, {}".format( + seq, + ts, + ", ".join( + [ + "{}:{}:{}".format( + k, + dataset[k][scans[i]].values, + dataset[k][scans[i]].values, + dataset[k][scans[i]].values, + ) + for k in [ + "scan", + "series_id", + "viewing_zenith_angle", + "quality_flag", + ] + ] + ), + ) + ) + id += 1 + + return dataset_l1b, flags + + def misalignment_ratio_calculator( + self, vza, vaa, offset, sza, saa, corr, direct_to_diffuse=1000 + ): + if isinstance(vza, np.ndarray): + vaa[vza < 0] += -180 + vza[vza < 0] = -vza[vza < 0] + + elif vza < 0: + vza = -vza + vaa += -180 + + sza = np.radians(sza) + saa = np.radians(saa) + vza = np.radians(vza) + vaa = np.radians(vaa) + new_sza = np.arccos( + np.cos(sza) * np.cos(vza) + np.sin(sza) * np.sin(vza) * np.cos((saa - vaa)) + ) + new_direct_to_diffuse = direct_to_diffuse * np.cos(new_sza) / np.cos(sza) + return ((new_direct_to_diffuse + 1) / (direct_to_diffuse + 1) + offset) / corr + + def calculate_bounds(self, refl_bounds_ds, sza, saa, vza, vaa): + if refl_bounds_ds is not None: + raa = (vaa - saa) % 360 + sza_bounds = refl_bounds_ds.sza.values.flatten() + vza_bounds = refl_bounds_ds.vza.values.flatten() + raa_bounds = np.repeat( + refl_bounds_ds.raa.values, refl_bounds_ds.sza.values.shape[1] + ) + lower_bounds = refl_bounds_ds.lower_bound.values.flatten() + upper_bounds = refl_bounds_ds.upper_bound.values.flatten() + id_360 = np.where(raa_bounds == 360)[0] + sza_bounds = np.concatenate([sza_bounds[id_360], sza_bounds]) + vza_bounds = np.concatenate([vza_bounds[id_360], vza_bounds]) + raa_bounds = np.concatenate([np.zeros_like(raa_bounds[id_360]), raa_bounds]) + lower_bounds = np.concatenate([lower_bounds[id_360], lower_bounds]) + upper_bounds = np.concatenate([upper_bounds[id_360], upper_bounds]) + id_valid = np.where(np.isfinite(lower_bounds))[0] + # id_valid = np.where((np.isfinite(lower_bounds)) & (sza_bounds < 30) & (vza_bounds < 15))[0] + + lower_interp = LinearNDInterpolator( + list( + zip( + sza_bounds[id_valid], vza_bounds[id_valid], raa_bounds[id_valid] + ) + ), + lower_bounds[id_valid], + ) + upper_interp = LinearNDInterpolator( + list( + zip( + sza_bounds[id_valid], vza_bounds[id_valid], raa_bounds[id_valid] + ) + ), + upper_bounds[id_valid], + ) + # print(sza[20],vza[20],raa[20], lower_interp(sza[20],vza[20],raa[20]), lower_bounds[id_valid]) + + return lower_interp(sza, vza, raa), upper_interp(sza, vza, raa) + else: + return 0.0, 1.1 diff --git a/hypernets_processor/etc/processor_land_defaults.config b/hypernets_processor/etc/processor_land_defaults.config index 1ae9686a..b9be8c22 100755 --- a/hypernets_processor/etc/processor_land_defaults.config +++ b/hypernets_processor/etc/processor_land_defaults.config @@ -1,5 +1,5 @@ [Site_specific] -site_id= +site_id = GHNA offset_pan= offset_tilt= use_config_latlon= @@ -8,15 +8,39 @@ lon= azimuth_switch= angle2use= pt_ask +[Site_specific_QC] +deployment_periods = [{"start_date": "2022-05-17" , "stop_date": "2023-10-17" ,"HYPSTAR_SN": 220261, "valid": True, "comments": "v1 instrument"},{"start_date": "2023-10-20" , "stop_date": "2024-05-23" ,"HYPSTAR_SN": 222316, "valid": True, "comments": "v3 instrument"},{"start_date": "2024-05-24" , "stop_date": "2025-03-27" ,"HYPSTAR_SN": 222316, "valid": True, "comments": "instrument cleaned and realigned but not replaced/calibrated"}, {"start_date": "2025-03-27" , "stop_date": "2025-09-27" ,"HYPSTAR_SN": 222316, "valid": True, "comments": "Grassy period following rain event"}, {"start_date": "2025-10-15" , "stop_date": "present" ,"HYPSTAR_SN": 220251, "valid": False, "comments": "instrument replaced, not enough data yet for valid L2B QC"}] +bad_dates_period = [None, None, None, None, None] +limit_tod_period = [["08:30","19:00"], ["08:30","19:00"], ["08:30","19:00"], ["08:30","19:00"], ["08:30","19:00"]] +bad_sequences_period = [[], [], [], [], []] # list (periods) of list with strings with bad sequence IDs +max_sza_period = [60,60,60, 60, 60] +bad_viewing_angles_period = [[(0, "all"),(5, "all")], [(0, "all"),(5, "all")], [(0, "all"),(5, "all")], [(0, "all"),(5, "all")], [(0, "all"),(5, "all")]] # list (periods) of list of tuples where each tuple is a (vza,vaa) pair to mask ("all" indicates all angles of that type should be masked) +bad_solar_angles_period = [[], [], [], [], []] # list (periods) of list of tuples where each tuple is a (vza,vaa) pair to mask ("all" indicates all angles of that type should be masked) +bad_relative_angles_period = [[("sza", 0),("sza", 360)], [("sza", 0),("sza", 360)], [("sza", 0),("sza", 360)], [("sza", 0),("sza", 360)], [("sza", 0),("sza", 360)]] # list (periods) of list of tuples where each tuple is a (vza,vaa) pair to mask ("sza" indicates all angles smaller than sza will be selected) +angle_tolerance=1 +raa_angle_tolerance = 10 +bad_wavelengths_period = [[], [], [], [], []] # list (periods) of list(multiple wavelength ranges) of list(bad_start_Wav,bad_end_wav) +postprocessing_qc_file_period = ["GHNA_2022May_2023Oct_bounds.nc","GHNA_2023Oct_2024May_bounds.nc","GHNA_2024May_2025Aug_bounds.nc", None, None] # list of strings to filenames from separate post-processing analysis +misalignment_vza=[1.01, 1.96, 1.28, 1.28, None] +misalignment_vza_unc=[0.07,0.06, 0.04, 0.04, None] +misalignment_vaa=[97,-72, -36, -36, None] +misalignment_vaa_unc=[6, 3, 2, 2, None] +misalignment_corr=[0.99, 1.01, 1.01, 1.01, None] +misalignment_corr_unc=[0.01, 0.02, 0.01, 0.01, None] +apply_site_specific_cloud_check = True + + [Processor] archive_directory= network= l -version= 2.0 +version= 2.2 mcsteps= 100 -max_level= L2A +max_level= L2B +reprocess_from = uncertainty_l1a= False -bad_wavelenth_ranges= 757.5-767.5, 1350-1390 +bad_wavelength_ranges= 757.5-767.5, 1350-1390 verbose= +delay_hours= [Databases] metadata_db_url= @@ -34,7 +58,7 @@ responsible_party= National Physical Laboratory (NPL), UK model= series_rep,series_id,vaa,azimuth_ref,vza,mode,action,it,scan_total,series_time [Quality] -l0_threshold=64000 +l0_threshold= l0_discontinuity=10000 bad_pointing_threshold_zenith=3 bad_pointing_threshold_azimuth=3 @@ -47,8 +71,10 @@ clear_sky_check=True vnir_swir_discontinuity_percent=25 [Calibration] -hypstar_cal_number= -measurement_function_calibrate= StandardMeasurementFunction +hypstar_cal_number = +measurement_function_calibrate = StandardMeasurementFunction +calibration_interpolation_method = previous +calibration_file_version = 2.4 [CombineSWIR] combine_lim_wav= 1000 @@ -71,21 +97,25 @@ write_l1a= True write_l1b= True write_l1c= True write_l2a= True +write_l2b= True +write_l1d= True [Plotting] plotting_format= png plot_fontsize= 14 -plot_legendfontsize= 10 +plot_legendfontsize= 14 plot_l0= False plot_l1a= False plot_l1a_diff= False plot_l1b= True plot_l1c= False plot_l2a= True +plot_l2b= True +plot_l1d= True plot_uncertainty= True plot_correlation= False plot_clear_sky_check= True plot_polar_wav= 900 plot_polar_min= plot_polar_max= - +plot_polar_ndvi= True diff --git a/hypernets_processor/etc/processor_water_defaults.config b/hypernets_processor/etc/processor_water_defaults.config index 53390b4a..6b98295b 100755 --- a/hypernets_processor/etc/processor_water_defaults.config +++ b/hypernets_processor/etc/processor_water_defaults.config @@ -14,8 +14,9 @@ network= w mcsteps= 0 max_level= L2A uncertainty_l1a=True -bad_wavelenth_ranges= 757.5-767.5, 1350-1390 +bad_wavelength_ranges= 757.5-767.5, 1350-1390 verbose= +delay_hours= [Databases] to_archive= True @@ -50,6 +51,8 @@ clear_sky_check=True [Calibration] hypstar_cal_number=120241 measurement_function_calibrate= StandardMeasurementFunction +calibration_interpolation_method = previous +calibration_file_version = 2.1 [Interpolate] measurement_function_interpolate_time= InterpolationTimeLinearCoscorrected diff --git a/hypernets_processor/interpolation/interpolate.py b/hypernets_processor/interpolation/interpolate.py index f3806f1a..1ecd974d 100755 --- a/hypernets_processor/interpolation/interpolate.py +++ b/hypernets_processor/interpolation/interpolate.py @@ -71,9 +71,9 @@ def interpolate_l1b_w( .sel(scan=upscan) .values ) - dataset_l1c_int[ - "err_corr_systematic_indep_upwelling_radiance" - ].values = dataset_l1a_uprad["err_corr_systematic_indep_radiance"].values + dataset_l1c_int["err_corr_systematic_indep_upwelling_radiance"].values = ( + dataset_l1a_uprad["err_corr_systematic_indep_radiance"].values + ) dataset_l1c_int[ "err_corr_systematic_corr_rad_irr_upwelling_radiance" ].values = dataset_l1a_uprad[ @@ -135,7 +135,7 @@ def interpolate_l1c( def interpolate_irradiance(self, dataset_l1c, dataset_l1b_irr, razangle=None): - dataset_l1c=self.qual.check_overcast(dataset_l1c,dataset_l1b_irr) + dataset_l1c = self.qual.check_overcast(dataset_l1c, dataset_l1b_irr) flags = [ "vza_irradiance", "not_enough_dark_scans", @@ -145,13 +145,18 @@ def interpolate_irradiance(self, dataset_l1c, dataset_l1b_irr, razangle=None): flagged = DatasetUtil.get_flags_mask_or(dataset_l1b_irr["quality_flag"], flags) mask_notflagged = np.where(flagged == False)[0] if len(mask_notflagged) == 0: - raise ValueError("one of the quality checks in previous steps has failed, and not correctly picked up that there is no valid irradiance") + raise ValueError( + "one of the quality checks in previous steps has failed, and not correctly picked up that there is no valid irradiance" + ) measurement_function_interpolate_wav = self.context.get_config_value( "measurement_function_interpolate_wav" ) prop = punpy.MCPropagation( - self.context.get_config_value("mcsteps"), dtype="float32", parallel_cores=1, verbose=False + self.context.get_config_value("mcsteps"), + dtype="float32", + parallel_cores=1, + verbose=False, ) if self.context.get_config_value("network") == "w": interpolation_function_wav = self._measurement_function_factory( @@ -181,21 +186,12 @@ def interpolate_irradiance(self, dataset_l1c, dataset_l1b_irr, razangle=None): store_unc_percent=True, ) else: - measurandstring = 'irradiance' - measurand = interpolation_function_wav.run( + measurandstring = "irradiance" + measurand = interpolation_function_wav.run_meas_function( dataset_l1c.rename({"wavelength": "radiance_wavelength"}), - dataset_l1b_irr) - dataset_l1c_temp[measurandstring].values = measurand - dataset_l1c_temp[measurandstring].attrs["unc_comps"] = [] - dataset_l1c_temp = dataset_l1c_temp.drop( - [ - "u_rel_random_" + measurandstring, - "u_rel_systematic_indep_" + measurandstring, - "u_rel_systematic_corr_rad_irr_" + measurandstring, - "err_corr_systematic_indep_" + measurandstring, - "err_corr_systematic_corr_rad_irr_" + measurandstring, - ] + dataset_l1b_irr, ) + dataset_l1c_temp[measurandstring].values = measurand measurement_function_interpolate_time = self.context.get_config_value( "measurement_function_interpolate_time" @@ -213,14 +209,15 @@ def interpolate_irradiance(self, dataset_l1c, dataset_l1b_irr, razangle=None): # Interpolate in time to radiance times acqui_rad = dataset_l1c["acquisition_time"].values output_sza = dataset_l1c["solar_zenith_angle"].values - input_sza = dataset_l1b_irr["solar_zenith_angle"].values + + acqui_irr = dataset_l1b_irr["acquisition_time"].values[mask_notflagged] + input_sza = dataset_l1b_irr["solar_zenith_angle"].values[mask_notflagged] if self.context.get_config_value("network") == "w": dataset_l1c_temp = dataset_l1c_temp.isel(scan=mask_notflagged) - acqui_irr = dataset_l1b_irr["acquisition_time"].values[mask_notflagged] + else: dataset_l1c_temp = dataset_l1c_temp.isel(series=mask_notflagged) - acqui_irr = dataset_l1b_irr["acquisition_time"].values[mask_notflagged] if self.context.get_config_value("mcsteps") > 0: dataset_l1c = interpolation_function_time.propagate_ds_specific( @@ -236,35 +233,17 @@ def interpolate_irradiance(self, dataset_l1c, dataset_l1b_irr, razangle=None): store_unc_percent=True, ) else: - measurandstring = 'irradiance' - measurand = interpolation_function_time.run(dataset_l1c_temp, { + measurandstring = "irradiance" + measurand = interpolation_function_time.run_meas_function( + dataset_l1c_temp, + { "input_time": acqui_irr, "output_time": acqui_rad, "input_sza": input_sza, "output_sza": output_sza, - }) - dataset_l1c[measurandstring].values = measurand - dataset_l1c[measurandstring].attrs["unc_comps"] = [] - dataset_l1c = dataset_l1c.drop( - [ - "u_rel_random_" + measurandstring, - "u_rel_systematic_indep_" + measurandstring, - "u_rel_systematic_corr_rad_irr_" + measurandstring, - "err_corr_systematic_indep_" + measurandstring, - "err_corr_systematic_corr_rad_irr_" + measurandstring, - ] - ) - dataset_l1c["radiance"].attrs["unc_comps"] = [] - dataset_l1c = dataset_l1c.drop( - [ - "u_rel_random_" + "radiance", - "u_rel_systematic_indep_" + "radiance", - "u_rel_systematic_corr_rad_irr_" + "radiance", - "err_corr_systematic_indep_" + "radiance", - "err_corr_systematic_corr_rad_irr_" + "radiance", - ] + }, ) - + dataset_l1c[measurandstring].values = measurand if len(acqui_irr) == 1: dataset_l1c["quality_flag"] = DatasetUtil.set_flag( @@ -307,24 +286,17 @@ def interpolate_skyradiance(self, dataset_l1c, dataset_l1b_skyrad): store_unc_percent=True, ) else: - measurandstring='downwelling_radiance' - measurand = interpolation_function_time.run(dataset_l1b_skyrad, { + measurandstring = "downwelling_radiance" + measurand = interpolation_function_time.run_meas_function( + dataset_l1b_skyrad, + { "input_time": acqui_skyrad, "output_time": acqui_rad, "input_sza": input_sza, "output_sza": output_sza, - }) - dataset_l1c[measurandstring].values = measurand - dataset_l1c = dataset_l1c.drop( - [ - "u_rel_random_" + measurandstring, - "u_rel_systematic_indep_" + measurandstring, - "u_rel_systematic_corr_rad_irr_" + measurandstring, - "err_corr_systematic_indep_" + measurandstring, - "err_corr_systematic_corr_rad_irr_" + measurandstring, - ] + }, ) - + dataset_l1c[measurandstring].values = measurand if len(acqui_skyrad) == 1: dataset_l1c["quality_flag"] = DatasetUtil.set_flag( diff --git a/hypernets_processor/interpolation/measurement_functions/interpolate_time_linear_coscorrected.py b/hypernets_processor/interpolation/measurement_functions/interpolate_time_linear_coscorrected.py index 155c8da1..6b19afbc 100644 --- a/hypernets_processor/interpolation/measurement_functions/interpolate_time_linear_coscorrected.py +++ b/hypernets_processor/interpolation/measurement_functions/interpolate_time_linear_coscorrected.py @@ -69,7 +69,9 @@ def get_argument_names(self): return ["output_time", "input_time", "irradiance", "output_sza", "input_sza"] -class WaterNetworkInterpolationSkyRadianceLinearCoscorrected(InterpolationTimeLinearCoscorrected): +class WaterNetworkInterpolationSkyRadianceLinearCoscorrected( + InterpolationTimeLinearCoscorrected +): # def meas_function(self, output_time, input_time, radiance, output_sza, input_sza): # """ # This function implements the measurement function. diff --git a/hypernets_processor/main/sequence_processor_main.py b/hypernets_processor/main/sequence_processor_main.py index 847af9e5..f5f43d8b 100755 --- a/hypernets_processor/main/sequence_processor_main.py +++ b/hypernets_processor/main/sequence_processor_main.py @@ -2,6 +2,8 @@ Module with main to run sequence file processing chain """ +import datetime + from hypernets_processor.version import __version__ from hypernets_processor.utils.config import read_config_file from hypernets_processor.utils.logging import configure_logging @@ -15,6 +17,7 @@ import numpy as np from multiprocessing import Pool import gc +import glob """___Authorship___""" __author__ = "Sam Hunt" @@ -41,21 +44,35 @@ def get_target_sequences(context, to_archive): # raw_data_directory may either be a sequence path or directory of sequence paths raw_paths = [] + incomplete_downloads = [] if parse_sequence_path(context.get_config_value("raw_data_directory")) is not None: raw_paths.append(context.get_config_value("raw_data_directory")) else: - for path in os.listdir(context.get_config_value("raw_data_directory")): - if parse_sequence_path(path) is not None: - sequence_path=os.path.join(context.get_config_value("raw_data_directory"), path) - if os.path.exists(os.path.join(sequence_path,"metadata.txt")): + # first add SEQ paths in raw_data_directory + paths = os.listdir(context.get_config_value("raw_data_directory")) + # then add paths in raw_data_directory/YYYY/mm/DD/SEQ* + paths.extend( + glob.glob( + os.path.join( + context.get_config_value("raw_data_directory"), + "20*", + "*", + "*", + "SEQ*", + ) + ) + ) + + for path in paths: + if parse_sequence_path(path, context) is not None: + sequence_path = os.path.join( + context.get_config_value("raw_data_directory"), path + ) + if os.path.exists(os.path.join(sequence_path, "metadata.txt")): raw_paths.append(sequence_path) else: - context.set_config_value("time", parse_sequence_path(path)["datetime"]) - context.set_config_value("sequence_name", path) - context.set_config_value("sequence_path", sequence_path) - context.logger.error("metadata.txt not found in directory %s, will try processing again later"%(sequence_path)) - #context.anomaly_handler.anomaly_db.add_anomaly("m") + incomplete_downloads.append(sequence_path) # If adding to archive, remove previously processed paths from list by referencing # archive db @@ -68,26 +85,86 @@ def get_target_sequences(context, to_archive): for product in context.archive_db["products"].find( site_id=context.get_config_value("site_id") ) - if "L2A" in product["product_level"] + if context.get_config_value("max_level") in product["product_level"] ] + if context.get_config_value("verbose"): + print("Already processed products: " + str(processed_products)) + failed_products = [ anomaly["sequence_name"] for anomaly in context.anomaly_db["anomalies"].find( site_id=context.get_config_value("site_id") ) + if ((anomaly["anomaly_id"] in context.anomaly_handler.get_crashing_anomaly_ids()) and anomaly["anomaly_id"]!="m" + and not context.get_config_value("reprocess_anomalies")) ] + + if context.get_config_value("verbose"): + print("Previously failed products: " + str(failed_products)) + complete_products = processed_products + failed_products - directory = os.path.dirname(raw_paths[0]) + if context.get_config_value("reprocess_from"): + raw_products = [ + product["sequence_name"] + for product in context.archive_db["products"].find( + site_id=context.get_config_value("site_id") + ) + if context.get_config_value("reprocess_from") + in product["product_level"] + ] + else: + raw_products = [os.path.basename(raw_path) for raw_path in raw_paths] - raw_products = [os.path.basename(raw_path) for raw_path in raw_paths] raw_products = list(set(raw_products) - set(complete_products)) - raw_paths = [ - os.path.join(directory, raw_product) for raw_product in raw_products - ] - return raw_paths + paths_to_process = [] + for i in range(len(raw_paths)): + raw_product = os.path.basename(raw_paths[i]) + raw_product_datetime = parse_sequence_path(raw_product)["datetime"] + if raw_product in raw_products: + if context.get_config_value("delay_hours") is not None and np.abs( + raw_product_datetime - datetime.datetime.now() + ) < datetime.timedelta(hours=context.get_config_value("delay_hours")): + print( + "%s is not processed yet due to not having reached required delay (%s hours)" + % (raw_product, context.get_config_value("delay_hours")) + ) + else: + paths_to_process.append(raw_paths[i]) + + # next, check if incompete downloads have already been added to anomaly db, and if not add them + if not context.get_config_value("max_level") == "L2B": + incomplete_products = [ + anomaly["sequence_name"] + for anomaly in context.anomaly_db["anomalies"].find( + site_id=context.get_config_value("site_id"),anomaly_id="m" + ) + ] + + for incomplete_download_path in incomplete_downloads: + seq_id = os.path.basename(incomplete_download_path) + if not seq_id in incomplete_products: + context.set_config_value( + "time", + parse_sequence_path(incomplete_download_path)["datetime"], + ) + context.set_config_value("sequence_name", seq_id) + context.set_config_value("sequence_path", incomplete_download_path) + context.logger.error( + "metadata.txt not found in directory %s, will try processing again later" + % (incomplete_download_path) + ) + context.anomaly_handler.anomaly_db.add_anomaly("m") + + + if context.get_config_value("verbose"): + print("Sequences to process: " + str([os.path.basename(p) for p in paths_to_process])) + + return paths_to_process + else: + return raw_paths def run_sequence(inputs): @@ -101,30 +178,34 @@ def run_sequence(inputs): # profiler.disable() # stats = pstats.Stats(profiler).sort_stats('tottime') # stats.print_stats(100) - if context.anomaly_handler.anomalies_added is not []: + if context.anomaly_handler.anomalies_SEQ is not []: context.logger.info( - "Processing Anomalies: " + str(context.anomaly_handler.anomalies_added) + "Processing Anomalies: " + str(context.anomaly_handler.anomalies_SEQ) ) context.logger.info(target_sequence + " Complete") del sp gc.collect() - return 1 + return context.get_config_value("max_level") except Exception as e: - + print("Exception occurred during processing of sequence: " + target_sequence) context.anomaly_handler.add_x_anomaly() - if context.anomaly_handler.anomalies_added is not []: + + if context.anomaly_handler.anomalies_SEQ is not []: context.logger.info( - "Processing Anomalies: " + str(context.anomaly_handler.anomalies_added) + "Processing Anomalies: " + str(context.anomaly_handler.anomalies_SEQ) ) logger.error(target_sequence + "Failed: " + repr(e)) logger.info(traceback.format_exc()) del sp gc.collect() - return 0 - + if np.any([anomaly in ['per','val','tod','hsn','scl','npr','man','wns','nos','hos'] for anomaly in context.anomaly_handler.anomalies_SEQ]): + context.anomaly_handler.anomalies_SEQ.clear() + return "L2A" + else: + return "L1" def main(processor_config, job_config, to_archive, parallel=None): """ @@ -158,20 +239,26 @@ def main(processor_config, job_config, to_archive, parallel=None): # Run processor target_sequences_total = len(target_sequences) + print("Total sequences to process: " + str(target_sequences_total)) + if target_sequences_total == 0: msg = "No sequences to process" else: - success = np.zeros_like(target_sequences, dtype=int) + success = np.zeros_like(target_sequences, dtype=str) for i, target_sequence in enumerate(target_sequences): success[i] = run_sequence((target_sequence, context, logger)) msg = ( - str(np.sum(success)) + str(np.sum(success[np.where(success == "L2B")])) + + "/" + + str(target_sequences_total) + + " sequences successfully processed to L2B and an additional" + + str(np.sum(success[np.where(success == "L2A")])) + "/" + str(target_sequences_total) - + " sequences successfully processed" + + " sequences successfully processed to L2A (but failed to reach L2B)." ) return msg diff --git a/hypernets_processor/plotting/plotting.py b/hypernets_processor/plotting/plotting.py index 3c3eb0c3..d1c68d3a 100755 --- a/hypernets_processor/plotting/plotting.py +++ b/hypernets_processor/plotting/plotting.py @@ -124,9 +124,14 @@ def plot_series_in_sequence(self, measurandstring, dataset, ylim=None): if len(angle_labels) > 10: linestyles = [ - (0, (3, np.abs(dataset["viewing_zenith_angle"].values[i]) / 10.0)) - if (np.abs(dataset["viewing_azimuth_angle"].values[i]) < 180) - else (0, (1, np.abs(dataset["viewing_zenith_angle"].values[i]) / 10.0)) + ( + (0, (3, np.abs(dataset["viewing_zenith_angle"].values[i]) / 10.0)) + if (np.abs(dataset["viewing_azimuth_angle"].values[i]) < 180) + else ( + 0, + (1, np.abs(dataset["viewing_zenith_angle"].values[i]) / 10.0), + ) + ) for i in range(len(dataset["viewing_zenith_angle"].values)) ] @@ -627,13 +632,24 @@ def plot_quality_irradiance(self, dataset, irrscaled, irrref, refsza): ) def plot_polar_reflectance(self, dataset, wavelength): - plotpath = os.path.join( - self.path, - "plot_polar_reflectance_" - + dataset.attrs["product_name"] - + "." - + self.plot_format, - ) + if wavelength == "ndvi": + plotpath = os.path.join( + self.path, + "plot_polar_ndvi_" + + dataset.attrs["product_name"] + + "." + + self.plot_format, + ) + label="NDVI" + else: + plotpath = os.path.join( + self.path, + "plot_polar_reflectance_" + + dataset.attrs["product_name"] + + "." + + self.plot_format, + ) + label="reflectance at %s nm" % wavelength saa = np.mean(dataset.solar_azimuth_angle.values % 360) sza = np.mean(dataset.solar_zenith_angle.values) @@ -646,7 +662,12 @@ def plot_polar_reflectance(self, dataset, wavelength): vza_grid = np.array([0, 5, 10, 20, 30, 40, 50, 60]) raa_grid = vaa_grid - saa - id_wav = np.argmin(np.abs(wavelength - dataset.wavelength.values)) + if wavelength == "ndvi": + id_865 = np.argmin(np.abs(865 - dataset.wavelength.values)) + id_665 = np.argmin(np.abs(665 - dataset.wavelength.values)) + + else: + id_wav = np.argmin(np.abs(wavelength - dataset.wavelength.values)) vaa_mesh, vza_mesh = np.meshgrid(np.radians(vaa_grid), vza_grid) @@ -666,7 +687,12 @@ def plot_polar_reflectance(self, dataset, wavelength): ) )[0] if len(id_series) == 1: - refl_2d[i, ii] = np.abs(refl[id_wav, id_series]) + if wavelength == "ndvi": + refl_2d[i, ii] = ( + refl[id_865, id_series] - refl[id_665, id_series] + ) / (refl[id_865, id_series] + refl[id_665, id_series]) + else: + refl_2d[i, ii] = np.abs(refl[id_wav, id_series]) elif len(id_series) > 1: self.context.logger.info( "There are multiple series that match the same vaa (%s) and vza (%s) " @@ -682,8 +708,20 @@ def plot_polar_reflectance(self, dataset, wavelength): ), ) ) - refl_2d[i, ii] = np.mean(np.abs(refl[id_wav, id_series])) + if wavelength == "ndvi": + refl_2d[i, ii] = np.mean( + (refl[id_865, id_series] - refl[id_665, id_series]) + / (refl[id_865, id_series] + refl[id_665, id_series]) + ) + else: + refl_2d[i, ii] = np.mean(np.abs(refl[id_wav, id_series])) refl_2d[refl_2d == 0] = np.nan + if wavelength == "ndvi": + vmin = None + vmax = None + else: + vmin = self.context.get_config_value("plot_polar_min") + vmax = self.context.get_config_value("plot_polar_max") fig = plt.figure() ax = plt.subplot(1, 1, 1, projection="polar") @@ -695,14 +733,14 @@ def plot_polar_reflectance(self, dataset, wavelength): refl_2d.T, shading="auto", cmap=plt.get_cmap("jet"), - vmin=self.context.get_config_value("plot_polar_min"), - vmax=self.context.get_config_value("plot_polar_max"), + vmin=vmin, + vmax=vmax, ) ax.plot(np.radians(saa), sza, color="k", ls="none", marker="o") cbar = fig.colorbar(im) - cbar.set_label("reflectance at %s nm" % wavelength, rotation=270, labelpad=15) + cbar.set_label(label, rotation=270, labelpad=15) fig.savefig(plotpath) plt.close(fig) diff --git a/hypernets_processor/post_processing/LOBE_analysis.py b/hypernets_processor/post_processing/LOBE_analysis.py new file mode 100644 index 00000000..dbd69455 --- /dev/null +++ b/hypernets_processor/post_processing/LOBE_analysis.py @@ -0,0 +1,25 @@ +import numpy as np +import pandas as pd +import matplotlib.pyplot as plt + + + +data = pd.read_csv('T:/ECO/EOServer/data/insitu/hypernets/post_processing_qc/LOBE_2025Apr_2025May_QC_good.csv') +print(data.head) + +refl_keys = [x for x in data.columns if 'refl' in x] +spectra = np.ones((len(data), len(refl_keys))) +wavs = [''.join(filter(str.isdigit, key)) for key in refl_keys] + +blue_swir_index = (data[' refl_1020nm'] - data[' refl_415nm'])/(data[' refl_1020nm'] + data[' refl_415nm']) + +for i in range(len(data)): + for j, key in enumerate(refl_keys): + spectra[i, j] = data[key][i] + +print(data[' vaa'][1746]) +plt.plot(wavs, spectra[1746,:], label = 'Mast') +plt.plot(wavs, spectra[1747,:], label = 'Soil') + +plt.legend() +plt.show() \ No newline at end of file diff --git a/hypernets_processor/post_processing/ML_testing.py b/hypernets_processor/post_processing/ML_testing.py new file mode 100644 index 00000000..1e7e4f38 --- /dev/null +++ b/hypernets_processor/post_processing/ML_testing.py @@ -0,0 +1,103 @@ +from random import Random + +import xarray as xr +import pandas as pd +import numpy as np +from sklearn.ensemble import RandomForestClassifier + +from sklearn.model_selection import train_test_split, cross_val_score +from sklearn.metrics import ( + confusion_matrix, + accuracy_score, + classification_report, + recall_score, + make_scorer, +) +from sklearn.tree import DecisionTreeClassifier, plot_tree, export_text +from sklearn.feature_selection import SelectFromModel + +import seaborn as sns +import matplotlib.pyplot as plt + + +windows_results_path = r"T:/ECO/EOServer/data/insitu/hypernets/post_processing_qc/joe" + +linux_data_path = r"/mnt/t/data/insitu/hypernets/post_processing_qc" +windows_data_path = r"T:/ECO/EOServer/data/insitu/hypernets/post_processing_qc" + +outliers = pd.read_csv( + r"T:\ECO\EOServer\data\insitu\hypernets\post_processing_qc\joe\GHNA_2024_outliers.csv" +) +good_data = pd.read_csv( + r"T:\ECO\EOServer\data\insitu\hypernets\post_processing_qc\joe\GHNA_2024_good.csv" +) + +outliers["is_outlier"] = "outlier" +good_data["is_outlier"] = "good" + +data = pd.concat(([outliers, good_data])) +IDs = data["# id"] + +data.drop(columns=data.columns[:6], axis=1, inplace=True) +data.drop(columns=["date", "time", " refl_550nm"], axis=1, inplace=True) + +# X = data.drop('is_outlier', axis = 1) +X = data[[" sza", " vza", " saa", " vaa", "raa"]] +y = data["is_outlier"] + +X_train, X_test, y_train, y_test = train_test_split( + X, y, train_size=0.8, test_size=0.2, random_state=1 +) +""" +def select_features(X_train, y_train, X_test, fs_model): + fs = SelectFromModel(fs_model, max_features = 7) + fs.fit(X_train, y_train) + fs_model.fit(X_train, y_train) + X_train_fs = fs.transform(X_train) + X_test_fs = fs.transform(X_test) + importances = fs_model.feature_importances_ + return X_train_fs, X_test_fs, fs, importances + +fs_model = RandomForestClassifier(n_estimators = 100, max_depth = 10, random_state = 1) +X_train_fs, X_test_fs, fs, importances = select_features(X_train, y_train, X_test, fs_model) +""" + +tree = DecisionTreeClassifier(max_depth=5, random_state=1, class_weight="balanced") +tree.fit(X_train, y_train) +y_pred = tree.predict(X_test) + +##printing +# print paths +tree_rules = export_text(tree, feature_names=X.keys()) +print(tree_rules) +print(classification_report(y_test, y_pred)) + +recall = make_scorer(recall_score, pos_label="outlier") +print(recall_score(y_test, y_pred, pos_label="outlier")) +print(cross_val_score(tree, X, y, scoring=recall)) +##plotting +# use if using feature selection model for selected keys +# fs_features = [i for (i, v) in zip(X.keys(), np.array(fs.get_support())) if v] +""" +#plot tree +plt.figure(figsize = (100,10)) +plot_tree(tree, filled = True, fontsize = 10, class_names = ['good', 'outlier'], feature_names = X.keys()) +plt.show() + +#plot confusion matrix +ax = plt.subplot() +conf = confusion_matrix(y_test, y_pred, labels = ['good', 'outlier']) +sns.heatmap(conf, annot = True, fmt = 'g', ax = ax) +ax.set_xlabel('Predicted Labels') +ax.set_ylabel('True Labels') +ax.set_title('Confusion Matrix') +ax.xaxis.set_ticklabels(['Good', 'Outlier']) +ax.yaxis.set_ticklabels(['Good', 'Outlier']) +plt.show() + +#plot feature importance +plt.bar(X.keys(), tree.feature_importances_) +plt.xticks(rotation = 30, ha = 'right') +plt.tight_layout() +plt.show() +""" diff --git a/hypernets_processor/post_processing/QC_pipeline.py b/hypernets_processor/post_processing/QC_pipeline.py new file mode 100644 index 00000000..cff5c0a1 --- /dev/null +++ b/hypernets_processor/post_processing/QC_pipeline.py @@ -0,0 +1,2301 @@ +import numpy as np +import pandas as pd +import xarray as xr +from scipy.optimize import curve_fit +import matplotlib.pyplot as plt +import os +from scipy.stats import binned_statistic_2d +from scipy.ndimage import gaussian_filter +from scipy.interpolate import CloughTocher2DInterpolator, LinearNDInterpolator +import math +import datetime +import matplotlib +import glob +import comet_maths as cm + +wav_df = xr.open_dataset( + r"T:\ECO\EOServer\data\insitu\hypernets\\archive\GHNA\\2024\\03\\14\SEQ20240314T070025\HYPERNETS_L_GHNA_L1B_IRR_20240314T0700_20240416T1138_v2.0.nc" +) +wav = wav_df.wavelength.values + +root = "T:/ECO/EOServer/" + + +def find_nearest_to_wav(array, wv, value): + wv = np.asarray(wv) + idx = (np.abs(wv - value)).argmin() + mean = np.mean( + [array[idx - 2], array[idx - 1], array[idx], array[idx + 1], array[idx + 2]] + ) + return mean + + +def interpolate_irradiance_sza(sza, ds_irr, variable): + ds_irr_temp = ds_irr.copy() + ds_irr_temp[variable].values = ( + ds_irr_temp[variable].values + / np.cos(np.float64(ds_irr_temp["sza"].values) / 180 * np.pi)[:, None] + ) + ds_irr_temp = ds_irr_temp.interp(sza=sza, wavelength=wav, method="linear") + ds_irr_temp[variable].values = ds_irr_temp[variable].values * np.cos( + np.float64(ds_irr_temp["sza"].values) / 180 * np.pi + ) + return ds_irr_temp[variable].values + + +def modelled_data_read_and_interp(sza, mod_data, variable): + sza = float(sza) + interp_mod_data = interpolate_irradiance_sza(sza, mod_data, variable) + return interp_mod_data + + +def percentile95(array): + return np.nanpercentile(array, 95) + + +def percentile5(array): + return np.nanpercentile(array, 5) + + +def percentile2_5(array): + return np.nanpercentile(array, 2.5) + + +def percentile97_5(array): + return np.nanpercentile(array, 97.5) + + +def percentile1(array): + return np.nanpercentile(array, 1) + + +def percentile99(array): + return np.nanpercentile(array, 99) + + +def percentile_3sigma_min(array): + return np.nanpercentile(array, 0.3) + + +def percentile_3sigma_max(array): + return np.nanpercentile(array, 99.7) + + +def sza_vza_bin_and_calc( + data, cutting_data, stat, bound, input_data=None, input_wav="550" +): + sza = data[" sza"].values + vza = data[" vza"].values + refl = data[f" refl_{input_wav}nm"].values + cutting_refl = cutting_data[f" refl_{input_wav}nm"].values + cutting_sza = cutting_data[" sza"].values + cutting_vza = cutting_data[" vza"].values + + bins = [ + 0, + 2.5, + 7.5, + 12.5, + 17.5, + 22.5, + 27.5, + 32.5, + 37.5, + 42.5, + 47.5, + 52.5, + 57.5, + 62.5, + 67.5, + 72.5, + ] + bin_centers = [(bins[i] + bins[i + 1]) / 2 for i in range(len(bins) - 1)] + + sza_bin_idx = [] + vza_bin_idx = [] + + for j in range(len(cutting_sza)): + for i in range(len(bins)): + if bins[i] <= cutting_sza[j] < bins[i + 1]: + sza_bin_idx.append(i) + if bins[i] <= cutting_vza[j] < bins[i + 1]: + vza_bin_idx.append(i) + + if stat == "median": + binned_data, x, y, b = binned_statistic_2d( + sza, vza, refl, "median", bins, expand_binnumbers=True + ) + elif stat == "95th": + binned_data, x, y, b = binned_statistic_2d( + sza, vza, refl, percentile95, bins, expand_binnumbers=True + ) + elif stat == "5th": + binned_data, x, y, b = binned_statistic_2d( + sza, vza, refl, percentile5, bins, expand_binnumbers=True + ) + elif stat == "2.5th": + binned_data, x, y, b = binned_statistic_2d( + sza, vza, refl, percentile2_5, bins, expand_binnumbers=True + ) + elif stat == "97.5th": + binned_data, x, y, b = binned_statistic_2d( + sza, vza, refl, percentile97_5, bins, expand_binnumbers=True + ) + elif stat == "1st": + binned_data, x, y, b = binned_statistic_2d( + sza, vza, refl, percentile1, bins, expand_binnumbers=True + ) + elif stat == "99th": + binned_data, x, y, b = binned_statistic_2d( + sza, vza, refl, percentile99, bins, expand_binnumbers=True + ) + elif stat == "3sigma_min": + binned_data, x, y, b = binned_statistic_2d( + sza, vza, refl, percentile_3sigma_min, bins, expand_binnumbers=True + ) + elif stat == "3sigma_max": + binned_data, x, y, b = binned_statistic_2d( + sza, vza, refl, percentile_3sigma_max, bins, expand_binnumbers=True + ) + elif stat == "count": + binned_data, x, y, b = binned_statistic_2d( + sza, vza, refl, "count", bins, expand_binnumbers=True + ) + elif stat == "std": + binned_data, x, y, b = binned_statistic_2d( + sza, vza, refl, "std", bins, expand_binnumbers=True + ) + elif stat == "median_plus_3std": + binned_std, x, y, b = binned_statistic_2d( + sza, vza, refl, "std", bins, expand_binnumbers=True + ) + binned_median, x, y, b = binned_statistic_2d( + sza, vza, refl, "median", bins, expand_binnumbers=True + ) + binned_data = binned_median + 3 * binned_std + elif stat == "median_minus_3std": + binned_std, x, y, b = binned_statistic_2d( + sza, vza, refl, "std", bins, expand_binnumbers=True + ) + binned_median, x, y, b = binned_statistic_2d( + sza, vza, refl, "median", bins, expand_binnumbers=True + ) + binned_data = binned_median - 3 * binned_std + else: + print("stat not supported") + binned_data, x, y, b = None, None, None, None + + if input_data is not None: + binned_data = input_data + + refl_output = [] + sza_output = [] + vza_output = [] + + if bound == "lower": + bound_passes = [ + False if v < binned_data[sza_bin_idx[i], vza_bin_idx[i]] else True + for i, v in enumerate(cutting_refl) + ] + elif bound == "upper": + bound_passes = [ + False if v > binned_data[sza_bin_idx[i], vza_bin_idx[i]] else True + for i, v in enumerate(cutting_refl) + ] + else: + print("bound must be upper or lower") + bound_passes = None + + for i in range(len(bin_centers)): + for j in range(len(bin_centers)): + if not np.isnan(binned_data[i][j]): + refl_output.append(binned_data[i][j]) + sza_output.append(bin_centers[i]) + vza_output.append(bin_centers[j]) + + return refl_output, sza_output, vza_output, bound_passes, binned_data + + +def raa_binning(data, raa_limits): + + if raa_limits[1] > raa_limits[0]: + new_data = data[(data["raa"] > raa_limits[0]) & (data["raa"] < raa_limits[1])] + new_data = new_data.reset_index(drop=True) + return new_data + + if raa_limits[1] < raa_limits[0]: + new_data = data[(data["raa"] > raa_limits[0]) | (data["raa"] < raa_limits[1])] + new_data = new_data.reset_index(drop=True) + return new_data + + +def quad_plane(xy, a, b, c, d, e, f): + x, y = xy + return a + b * x + c * y + d * x**2 + e * y**2 + f * x * y + + +def linear_plane(xy, a, b, c): + x, y = xy + return a + b * x + c * y + + +def cubic_plane(xy, a, b, c, d, e, f, g, h, i, j): + x, y = xy + return ( + a + + b * x + + c * y + + d * x**2 + + e * y**2 + + f * x * y + + g * x**3 + + h * y**3 + + i * x * y**2 + + j * y * x**2 + ) + + +def plane_fit_and_plot( + data, data_setting, stds, plot=True, raa_bin=None, iteration=0, input_wav="550" +): + x = data[" sza"].values + y = data[" vza"].values + z = data[f" refl_{input_wav}nm"].values + + popt, pcov = curve_fit(quad_plane, (x, y), z) + below = popt - stds * np.sqrt(np.diag(pcov)) + above = popt + stds * np.sqrt(np.diag(pcov)) + + z_below = ( + quad_plane((data_setting[" sza"].values, data_setting[" vza"].values), *below) + # - data_setting[" refl_550nm"].values * 0.05 + ) + z_above = ( + quad_plane((data_setting[" sza"].values, data_setting[" vza"].values), *above) + # + data_setting[" refl_550nm"].values * 0.05 + ) + + outliers = data_setting[ + (data_setting[f" refl_{input_wav}nm"] > z_above) + | (data_setting[f" refl_{input_wav}nm"] < z_below) + ] + good_data = data_setting[ + (data_setting[f" refl_{input_wav}nm"] < z_above) + & (data_setting[f" refl_{input_wav}nm"] > z_below) + ] + good_data.reset_index(drop=True, inplace=True) + outliers.reset_index(drop=True, inplace=True) + + if plot: + x_range = np.linspace(0, 61, 1000) + y_range = np.linspace(0, 61, 1000) + X, Y = np.meshgrid(x_range, y_range) + """ + fig = plt.figure() + ax = fig.add_subplot(111, projection="3d") + + ax.scatter( + data_setting[" sza"].values, + data_setting[" vza"].values, + data_setting[" refl_550nm"].values, + ) + + + + + Z = quad_plane((X, Y), *popt) + Z_below = quad_plane((X, Y), *below) + Z_above = quad_plane((X, Y), *above) + + ax.plot_surface(X, Y, Z, alpha=0.5) + ax.plot_surface(X, Y, Z_above, alpha=0.5) + ax.plot_surface(X, Y, Z_below, alpha=0.5) + + ax.set_xlabel("SZA") + ax.set_ylabel("VZA") + ax.set_zlabel("Reflectance") + # fig.savefig(results_path + title) + plt.show() + """ + + fig, axs = plt.subplots(1, 3, figsize=(6, 4), sharey=True) + for i in range(3): + axs[i].plot( + Y, quad_plane(((i + 1) * 20, Y), *popt), color="black", alpha=0.1 + ) + axs[i].plot(Y, quad_plane(((i + 1) * 20, Y), *below), color="black") + axs[i].plot(Y, quad_plane(((i + 1) * 20, Y), *above), color="black") + axs[i].set_title(f"SZA = {(i+1)*20}") + axs[i].scatter( + data_setting[ + data_setting[" sza"].between((i + 1) * 20 - 1, (i + 1) * 20 + 1) + ][" vza"].values, + data_setting[ + data_setting[" sza"].between((i + 1) * 20 - 1, (i + 1) * 20 + 1) + ][f" refl_{input_wav}nm"].values, + color="black", + s=5, + ) + axs[i].scatter( + good_data[ + good_data[" sza"].between((i + 1) * 20 - 1, (i + 1) * 20 + 1) + ][" vza"].values, + good_data[ + good_data[" sza"].between((i + 1) * 20 - 1, (i + 1) * 20 + 1) + ][f" refl_{input_wav}nm"].values, + color="green", + s=5, + ) + axs[i].scatter( + outliers[outliers[" sza"].between((i + 1) * 20 - 1, (i + 1) * 20 + 1)][ + " vza" + ].values, + outliers[outliers[" sza"].between((i + 1) * 20 - 1, (i + 1) * 20 + 1)][ + f" refl_{input_wav}nm" + ].values, + color="red", + s=5, + ) + axs[i].set_xlabel("VZA") + axs[0].set_ylabel("Reflectance") + fig.suptitle(raa_bin) + fig.savefig( + r"T:/ECO/EOServer/joe/hypernets_plots/plane_models/" + + f"sza_slices_plane_{raa_bin}_{iteration}.png" + ) + plt.close() + + return outliers, good_data + + +def new_plane_fit(input_sza, input_vza, input_refl, plane_type="quad", count=None): + x = input_sza + y = input_vza + z = input_refl + + if plane_type == "quad": + popt, pcov = curve_fit(quad_plane, (x, y), z, nan_policy="omit", sigma=count) + elif plane_type == "linear": + popt, pcov = curve_fit(linear_plane, (x, y), z, nan_policy="omit", sigma=count) + elif plane_type == "cubic": + popt, pcov = curve_fit(cubic_plane, (x, y), z, nan_policy="omit", sigma=count) + + return popt, pcov + + +def compile_irradiance_list(irr_folder, site): + files = [f for f in os.listdir(irr_folder) if f"{site}_clear_sky" in f] + + return files + + +def ratio_calculator(vza, vaa, sza, saa, direct_to_diffuse): + sza = np.radians(sza) + saa = np.radians(saa) + vza = np.radians(vza) + vaa = np.radians(vaa) + # new_sza = np.cos(sza + vza*np.cos((saa - vaa + 360) % 360)) + new_sza = np.arccos( + np.cos(sza) * np.cos(vza) + np.sin(sza) * np.sin(vza) * np.cos((saa - vaa)) + ) + new_direct_to_diffuse = direct_to_diffuse * np.cos(new_sza) / np.cos(sza) + return (direct_to_diffuse + 1) / (new_direct_to_diffuse + 1) + + +class PostProcessingDataset: + + def __init__( + self, + rad_path, + irr_path, + clear_sky_path, + site, + period_name=None, + clear_sky_aod=None, + input_wav="550", + ): + + # read in radiance + data = pd.read_csv(rad_path) + # define raa + data.loc[:, "raa"] = (data.loc[:, " vaa"] - data.loc[:, " saa"] + 360) % 360 + # extract dates and times + dates = [] + times = [] + datetimes = [] + for i in range(len(data)): + date = data["# id"][i][3:11] + time = data["# id"][i][12:16] + dates.append(date) + times.append(time) + datetimes.append( + datetime.datetime.strptime(date + ":" + time, "%Y%m%d:%H%M") + ) + dt_data = data.assign(date=dates, time=times, datetimes=datetimes) + labelled_data = dt_data.assign( + post_processing_flags=[[] for _ in range(len(data))] + ) + # set radiance + self.radiance = dt_data + self.labelled_radiance = labelled_data + # read in irradiance + self.irradiance = pd.read_csv(irr_path) + # define other features + self.site = site + if period_name is None: + self.period_name = site + else: + self.period_name = period_name + self.cc_radiance = None + self.radiance_good = None + self.radiance_outliers = None + self.cloud_passes = None + self.cloud_check_tol = None + self.clear_sky_path = clear_sky_path + self.bound_iteration = 0 + self.plane_iteration = 0 + self.input_wav = input_wav + if clear_sky_aod is None: + self.clear_sky_list = compile_irradiance_list(clear_sky_path, site) + self.clear_sky_aod = None + self.clear_sky_model = None + else: + self.clear_sky_list = None + self.clear_sky_aod = clear_sky_aod + if self.clear_sky_aod == "median": + self.clear_sky_model = xr.open_dataset( + "T:\ECO\EOServer\data\insitu\hypernets\post_processing_qc\irradiance\{}_clear_sky_medianaod.nc".format( + site, clear_sky_aod + ) + ) + else: + self.clear_sky_model = xr.open_dataset( + "T:\ECO\EOServer\data\insitu\hypernets\post_processing_qc\irradiance\{}_clear_sky_aod{}.nc".format( + site, clear_sky_aod + ) + ) + + def choose_clear_sky_model(self, aod): + if self.clear_sky_list is not None: + model = [f for f in self.clear_sky_list if aod in f] + self.clear_sky_aod = aod + self.clear_sky_model = xr.open_dataset(self.clear_sky_path + str(model[0])) + + def size(self, var): + if var == "radiance": + print(len(self.radiance)) + elif var == "good": + print(len(self.radiance_good)) + elif var == "outliers": + print(len(self.radiance_outliers)) + else: + print("var must be radiance, good, or outliers") + + def save(self, var, filepath): + if var == "radiance": + data = self.radiance + elif var == "good": + data = self.radiance_good + elif var == "outliers": + data = self.radiance_outliers + else: + print("data not saved") + quit() + + data.to_csv(filepath, index=False) + + def calculate_ndvi(self): + self.radiance["ndvi"] = ( + self.radiance[" refl_842nm"] - self.radiance[" refl_665nm"] + ) / (self.radiance[" refl_842nm"] + self.radiance[" refl_665nm"]) + + def calculate_savi(self): + self.radiance["savi"] = ( + (self.radiance[" refl_842nm"] - self.radiance[" refl_665nm"]) + / (self.radiance[" refl_842nm"] + self.radiance[" refl_665nm"] + 0.5) + ) * 1.5 + + def calculate_s2tci(self): + self.radiance["s2tci"] = ( + self.radiance[" refl_740nm"] - self.radiance[" refl_705nm"] + ) / (self.radiance[" refl_705nm"] - self.radiance[" refl_665nm"]) + + def split_by_date(self, start_date, end_date): + self.radiance["datetime"] = [ + datetime.datetime.strptime(x, "%Y%m%d") for x in self.radiance.date + ] + self.radiance = self.radiance[ + ( + (self.radiance.datetime < pd.Timestamp(end_date)) + & (self.radiance.datetime >= pd.Timestamp(start_date)) + ) + ] + + def count_by_bin(self): + bins = [ + 0, + 2.5, + 7.5, + 12.5, + 17.5, + 22.5, + 27.5, + 32.5, + 37.5, + 42.5, + 47.5, + 52.5, + 57.5, + 62.5, + 67.5, + 72.5, + ] + bin_centers = [(bins[i] + bins[i + 1]) / 2 for i in range(len(bins) - 1)] + + good_count, sza, vza, b, n = sza_vza_bin_and_calc( + self.radiance_good, + self.radiance_good, + "count", + "na", + input_wav=self.input_wav, + ) + out_count, out_sza, out_vza, out_b, out_n = sza_vza_bin_and_calc( + self.radiance_outliers, + self.radiance_outliers, + "count", + "na", + input_wav=self.input_wav, + ) + percentage_removed = ( + np.true_divide( + np.array(out_count), (np.array(out_count) + np.array(good_count)) + ) + * 100 + ) + + fig, ax = plt.subplots(1, 1, figsize=(5, 6)) + img = ax.imshow( + percentage_removed.reshape(15, 15), origin="lower", extent=[0, 75, 0, 75] + ) + ax.set_xticks(bin_centers) + ax.set_yticks(bin_centers) + ax.set_ylabel("SZA") + ax.set_xlabel("VZA") + fig.colorbar(img) + # plt.show() + + def basic_outlier_definition(self): + mean = self.radiance[f" refl_{self.input_wav}nm"].mean() + stde = self.radiance[f" refl_{self.input_wav}nm"].std() + max_range = mean + (stde * 3) + min_range = mean - (stde * 3) + outliers = self.radiance[ + (self.radiance[f" refl_{self.input_wav}nm"] > max_range) + | (self.radiance[f" refl_{self.input_wav}nm"] < min_range) + ] + good_data = self.radiance[ + (self.radiance[f" refl_{self.input_wav}nm"] < max_range) + & (self.radiance[f" refl_{self.input_wav}nm"] > min_range) + ] + good_data.reset_index(drop=True, inplace=True) + outliers.reset_index(drop=True, inplace=True) + + self.radiance_outliers = outliers + self.radiance_good = good_data + + def maintenance_check(self, bad_dates): + ### note bad_dates need to be in speech marks### + bad_inds = [] + # get dates of measurements + for i in range(len(self.radiance)): + date = self.radiance["date"][i] + + # remove maintenance dates + if date in bad_dates: + bad_index = i + bad_inds.append(bad_index) + else: + continue + + cleaned_data = self.radiance.drop(bad_inds) + cleaned_data.reset_index(drop=True, inplace=True) + self.radiance = cleaned_data + + for x in self.labelled_radiance["post_processing_flags"][ + self.labelled_radiance["date"].isin(bad_dates) + ]: + x.append(1) + + def sza_check(self, sza_limit): + bad_inds = [] + for i in range(len(self.radiance)): + sza = self.radiance[" sza"][i] + if sza > sza_limit: + bad_index = i + bad_inds.append(bad_index) + else: + continue + + cleaned_data = self.radiance.drop(bad_inds) + cleaned_data.reset_index(drop=True, inplace=True) + self.radiance = cleaned_data + + for x in self.labelled_radiance["post_processing_flags"].loc[ + self.labelled_radiance[" sza"] > sza_limit + ]: + x.append(2) + + def vza_check(self, vza_limit): + bad_inds = [] + for i in range(len(self.radiance)): + vza = self.radiance[" vza"][i] + if vza < vza_limit: + bad_index = i + bad_inds.append(bad_index) + else: + continue + + cleaned_data = self.radiance.drop(bad_inds) + cleaned_data.reset_index(drop=True, inplace=True) + self.radiance = cleaned_data + + for x in self.labelled_radiance["post_processing_flags"].loc[ + self.labelled_radiance[" vza"] < vza_limit + ]: + x.append(999) + + def vza_check_toplimit(self, vza_limit): + bad_inds = [] + for i in range(len(self.radiance)): + vza = self.radiance[" vza"][i] + if vza > vza_limit: + bad_index = i + bad_inds.append(bad_index) + else: + continue + + cleaned_data = self.radiance.drop(bad_inds) + cleaned_data.reset_index(drop=True, inplace=True) + self.radiance = cleaned_data + + for x in self.labelled_radiance["post_processing_flags"].loc[ + self.labelled_radiance[" vza"] > vza_limit + ]: + x.append(999) + + def raa_cut(self, raa_limit): + bad_inds = [] + for i in range(len(self.radiance)): + raa = self.radiance["raa"][i] + if (raa < raa_limit) | (raa > 360 - raa_limit): + bad_index = i + bad_inds.append(bad_index) + else: + continue + + cleaned_data = self.radiance.drop(bad_inds) + cleaned_data.reset_index(drop=True, inplace=True) + self.radiance = cleaned_data + + for x in self.labelled_radiance["post_processing_flags"].loc[ + (self.labelled_radiance["raa"] < raa_limit) + | (self.labelled_radiance["raa"] > 360 - raa_limit) + ]: + x.append(6) + + def clean_irr(self): + IDs = self.irradiance["ID"] + flags = self.irradiance["Flag"] + + bad_inds = [] + bad_flags = [] + bad_IDs = [] + # find flags in irr data + for i in range(len(flags)): + if (flags.values[i] & 773892) != 0: + bad_inds.append(i) + bad_flags.append(flags.values[i]) + bad_IDs.append(IDs[i]) + # check irradiance file exists for every radiance + for i, ID in enumerate(list(self.irradiance["ID"])): + if ID in list(self.radiance["# id"]): + continue + else: + bad_IDs.append(ID) + bad_inds.append(i) + + # clean irradiance + df_clean = self.irradiance.drop(bad_inds) + df_clean.reset_index(drop=True, inplace=True) + self.irradiance = df_clean + # clean radiance + new = self.radiance[~self.radiance["# id"].isin(bad_IDs)] + new.reset_index(drop=True, inplace=True) + self.radiance = new + + for x in self.labelled_radiance["post_processing_flags"].loc[ + self.labelled_radiance["# id"].isin(bad_IDs) + ]: + x.append(3) + + def append_aod(self): + if self.site == "GHNA": + aod_path = r"T:/ECO/EOServer/data/insitu/radcalnet/RadCalNet-All-Sites-Feb2025/GONA/" + dates = [x[3:11] for x in self.irradiance["ID"].values] + years = [x[0:4] for x in dates] + doy = [ + datetime.date(int(x[0:4]), int(x[4:6]), int(x[6:8])).timetuple().tm_yday + for x in dates + ] + aod_files = [ + aod_path + f"GONA01_{years[i]}_{doy[i]}_v00.09.input" + for i in range(len(dates)) + ] + aods = [] + for path in aod_files: + try: + f = open(path, "r") + datafile = f.readlines() + row = [i for i, s in enumerate(datafile) if "AOD:\t" in s][0] + aod = np.mean( + [ + float(i.rstrip()) + for i in datafile[row].split("\t")[1::] + if i.rstrip() != "" + ] + ) + aods.append(aod) + except: + aods.append(np.nan) + + for i, v in enumerate(aods): + if aods[i] > 10: + aods[i] = np.nan + + self.irradiance["aod"] = aods + + def misalignment_correction(self, vza, vaa, mean_corr, multiple_periods = False, period_date = None): + szas = self.radiance[' sza'] + saas = self.radiance[' saa'] + data = np.zeros((self.irradiance.shape[0], len(self.irradiance.columns) - 6)) + wav_list = [ + int("".join(filter(str.isdigit, s))) + for s in self.radiance.keys() + if "refl" in s + ] + + for wv in wav_list: + dir_to_diff = np.zeros(self.radiance.shape[0]) + + for i in range(dir_to_diff.shape[0]): + dir_to_diff[i] = find_nearest_to_wav( + modelled_data_read_and_interp( + szas[i], + self.clear_sky_model, + "direct_to_diffuse_irradiance_ratio", + ), + wav, + wv, + ) + + if not multiple_periods: + self.radiance[f' refl_{wv}nm'] = self.radiance[f' refl_{wv}nm'] * ratio_calculator(vza, vaa, szas, saas, dir_to_diff)/mean_corr + else: + divider_idx = np.min(np.argwhere(self.radiance['date'] == period_date)) + self.radiance[f' refl_{wv}nm'][:divider_idx] = ( + self.radiance[f' refl_{wv}nm'][:divider_idx] * + ratio_calculator(vza[0], vaa[0], szas[:divider_idx], saas[:divider_idx], + dir_to_diff[:divider_idx]))/mean_corr[0] + self.radiance[f' refl_{wv}nm'][divider_idx:] = ( + self.radiance[f' refl_{wv}nm'][divider_idx:] * + ratio_calculator(vza[1], vaa[1], szas[divider_idx:], saas[divider_idx:], + dir_to_diff[divider_idx:]))/mean_corr[1] + + + def cloud_check(self, tolerance, wavelength, rewrite, rewrite_var=None): + szas = self.irradiance["SZA"] + IDs = self.irradiance["ID"] + saas = self.irradiance["SAA"] + + if self.cloud_passes is None: + data = np.zeros( + (self.irradiance.shape[0], len(self.irradiance.columns) - 6) + ) + + for j in range(self.irradiance.shape[0]): + for i in range(len(self.irradiance.columns) - 6): + data[j, i] = self.irradiance["{}".format(i)][j] + + passes = np.zeros(self.irradiance.shape[0]) + values = np.zeros(self.irradiance.shape[0]) + abs_diff = np.zeros(self.irradiance.shape[0]) + model = np.zeros(self.irradiance.shape[0]) + observ = np.zeros(self.irradiance.shape[0]) + + for i in range(data.shape[0]): + aod_1 = find_nearest_to_wav( + modelled_data_read_and_interp( + szas[i], self.clear_sky_model, "solar_irradiance_BOA" + ), + wav, + wavelength, + ) + meas = find_nearest_to_wav(data[i, :], wav, wavelength) + values[i] = aod_1 * tolerance - meas + abs_diff[i] = aod_1 - meas + model[i] = aod_1 + observ[i] = meas + + if values[i] > 0: + passes[i] = 1 # cloudy + else: + passes[i] = 0 # clear + + print(i, passes[i]) + + # save passes + self.cloud_passes = passes + output_data = np.array( + [ + IDs, + szas, + saas, + model, + observ, + abs_diff, + [x[12:16] for x in IDs], + [x[3:11] for x in IDs], + self.irradiance["Flag"], + ] + ).T + # np.savetxt(r'T:\ECO\EOServer\data\insitu\hypernets\post_processing_qc\joe\irradiance_GHNA_v1_diffs.csv',output_data, fmt = '%s',delimiter=',') + + cut_IDs = IDs[self.cloud_passes == 1] + + # rewrite datasets + if rewrite_var == "radiance": + data_refl = self.radiance + elif rewrite_var == "good": + data_refl = self.radiance_good + elif rewrite_var == "outliers": + data_refl = self.radiance_outliers + else: + data_refl = self.radiance + + new = data_refl[~data_refl["# id"].isin(cut_IDs)] + new.reset_index(drop=True, inplace=True) + + if rewrite: + + if rewrite_var == "radiance": + self.radiance = new + elif rewrite_var == "good": + self.radiance_good = new + elif rewrite_var == "outliers": + self.radiance_outliers = new + elif rewrite_var == "labels": + for x in self.labelled_radiance["post_processing_flags"].loc[ + self.labelled_radiance["# id"].isin(cut_IDs) + ]: + x.append(4) + + else: + print("rewrite_var must be radiance, good, outliers, or labels") + print("radiances not filtered by cloud check") + self.cc_radiance = new + + def write_irr_analysis_files( + self, filename, wavelengths, save=True, plot=False, interp=False + ): + szas = self.irradiance["SZA"] + IDs = self.irradiance["ID"] + saas = self.irradiance["SAA"] + + data = np.zeros((self.irradiance.shape[0], len(self.irradiance.columns) - 6)) + for j in range(self.irradiance.shape[0]): + for i in range(len(self.irradiance.columns) - 6): + data[j, i] = self.irradiance["{}".format(i)][j] + + output_data = pd.DataFrame( + { + "IDs": IDs, + "SZA": szas, + "SAA": saas, + "Time": [x[12:16] for x in IDs], + "Date": [x[3:11] for x in IDs], + "Flag": self.irradiance["Flag"], + } + ) + if interp: + output_data["AOD"] = self.irradiance["aod"] + + for wv in wavelengths: + model = np.zeros(output_data.shape[0]) + observ = np.zeros(output_data.shape[0]) + dir_to_diff = np.zeros(output_data.shape[0]) + + for i in range(output_data.shape[0]): + if not interp: + model[i] = find_nearest_to_wav( + modelled_data_read_and_interp( + szas[i], self.clear_sky_model, "solar_irradiance_BOA" + ), + wav, + wv, + ) + else: + aod_0 = find_nearest_to_wav( + modelled_data_read_and_interp( + szas[i], + xr.open_dataset( + "T:\ECO\EOServer\data\insitu\hypernets\post_processing_qc\irradiance\{}_clear_sky_aod{}.nc".format( + self.site, "0.0" + ) + ), + "solar_irradiance_BOA", + ), + wav, + wv, + ) + aod_1 = find_nearest_to_wav( + modelled_data_read_and_interp( + szas[i], + xr.open_dataset( + "T:\ECO\EOServer\data\insitu\hypernets\post_processing_qc\irradiance\{}_clear_sky_aod{}.nc".format( + self.site, "0.1" + ) + ), + "solar_irradiance_BOA", + ), + wav, + wv, + ) + aod_2 = find_nearest_to_wav( + modelled_data_read_and_interp( + szas[i], + xr.open_dataset( + "T:\ECO\EOServer\data\insitu\hypernets\post_processing_qc\irradiance\{}_clear_sky_aod{}.nc".format( + self.site, "0.2" + ) + ), + "solar_irradiance_BOA", + ), + wav, + wv, + ) + aod_3 = find_nearest_to_wav( + modelled_data_read_and_interp( + szas[i], + xr.open_dataset( + "T:\ECO\EOServer\data\insitu\hypernets\post_processing_qc\irradiance\{}_clear_sky_aod{}.nc".format( + self.site, "0.3" + ) + ), + "solar_irradiance_BOA", + ), + wav, + wv, + ) + + mod_arr = [aod_0, aod_1, aod_2, aod_3] + mod_aod = [0, 0.1, 0.2, 0.3] + model[i] = np.interp(output_data["AOD"][i], mod_aod, mod_arr) + + observ[i] = find_nearest_to_wav(data[i, :], wav, wv) + dir_to_diff[i] = find_nearest_to_wav( + modelled_data_read_and_interp( + szas[i], + self.clear_sky_model, + "direct_to_diffuse_irradiance_ratio", + ), + wav, + wv, + ) + + output_data[f"model_{wv}nm"] = model + output_data[f"obs_{wv}nm"] = observ + output_data[f"dir_diff_ratio_{wv}nm"] = dir_to_diff + output_data[f"ratio_{wv}nm"] = model / observ + + if plot: + fig, ax = plt.subplots( + 1, + 1, + ) + ax.scatter(output_data["AOD"], model / observ) + fig.savefig( + r"T:\ECO\EOServer\joe\hypernets_plots\aod_plots\{}_aod_ratio_interpolated_{}.png".format( + self.period_name, wv + ) + ) + + if save: + output_data = output_data[output_data["IDs"].isin(self.radiance["# id"])] + output_data.to_csv( + r"T:\ECO\EOServer\data\insitu\hypernets\post_processing_qc\joe\{}.csv".format( + filename + ) + ) + + def plane_fitter(self, labelled=False, plot=False): + + if self.cc_radiance is None: + self.cc_radiance = self.radiance + + # define datasets + # data = self.cc_radiance + data = self.radiance_good + self.plane_iteration += 1 + + if labelled: + data_unmasked = self.labelled_radiance + else: + data_unmasked = self.radiance + + data_30_90 = raa_binning(data, (30, 90)) + data_90_150 = raa_binning(data, (90, 150)) + data_150_210 = raa_binning(data, (150, 210)) + data_210_270 = raa_binning(data, (210, 270)) + data_270_330 = raa_binning(data, (270, 330)) + data_330_30 = raa_binning(data, (330, 30)) + + data_list = [ + data_30_90, + data_90_150, + data_150_210, + data_210_270, + data_270_330, + data_330_30, + ] + + data_unmasked_30_90 = raa_binning(data_unmasked, (30, 90)) + data_unmasked_90_150 = raa_binning(data_unmasked, (90, 150)) + data_unmasked_150_210 = raa_binning(data_unmasked, (150, 210)) + data_unmasked_210_270 = raa_binning(data_unmasked, (210, 270)) + data_unmasked_270_330 = raa_binning(data_unmasked, (270, 330)) + data_unmasked_330_30 = raa_binning(data_unmasked, (330, 30)) + + data_unmasked_list = [ + data_unmasked_30_90, + data_unmasked_90_150, + data_unmasked_150_210, + data_unmasked_210_270, + data_unmasked_270_330, + data_unmasked_330_30, + ] + raa_limits_list = ["30_90", "90_150", "150_210", "210_270", "270_330", "330_30"] + + outs, good = plane_fit_and_plot( + data_list[0], + data_unmasked_list[0], + 3, + plot, + raa_limits_list[0], + self.plane_iteration, + ) + for i in range(len(data_list) - 1): + outs = pd.concat( + [ + outs, + plane_fit_and_plot( + data_list[i + 1], + data_unmasked_list[i + 1], + 3, + plot, + raa_limits_list[i + 1], + self.plane_iteration, + )[0], + ], + ignore_index=True, + ) + good = pd.concat( + [ + good, + plane_fit_and_plot( + data_list[i + 1], + data_unmasked_list[i + 1], + 3, + plot, + raa_limits_list[i + 1], + self.plane_iteration, + )[1], + ], + ignore_index=True, + ) + + if labelled: + for y in self.labelled_radiance["post_processing_flags"][ + self.labelled_radiance.apply(tuple, axis=1).isin( + [tuple(x) for x in outs.values] + ) + ]: + y.append(5) + else: + self.radiance_good = good + self.radiance_outliers = outs + + # make this general so fitted variables and binned variables can be changed + + def new_plane_fitter( + self, + labelled=False, + first_iteration=True, + plot=False, + method="plane_fit", + save_bounds=False, + nc_path=None, + last_iteration=False, + ): + + if self.cc_radiance is None: + self.cc_radiance = self.radiance + + if not first_iteration: + data = self.radiance_good + else: + data = self.cc_radiance + + if labelled: + data_unmasked = self.labelled_radiance + else: + data_unmasked = self.radiance + + raa_limits_list = ["30_90", "90_150", "150_210", "210_270", "270_330", "330_30"] + + data_30_90 = raa_binning(data, (30, 90)) + data_90_150 = raa_binning(data, (90, 150)) + data_150_210 = raa_binning(data, (150, 210)) + data_210_270 = raa_binning(data, (210, 270)) + data_270_330 = raa_binning(data, (270, 330)) + data_330_30 = raa_binning(data, (330, 30)) + + data_list = [ + data_30_90, + data_90_150, + data_150_210, + data_210_270, + data_270_330, + data_330_30, + ] + + data_unmasked_30_90 = raa_binning(data_unmasked, (30, 90)) + data_unmasked_90_150 = raa_binning(data_unmasked, (90, 150)) + data_unmasked_150_210 = raa_binning(data_unmasked, (150, 210)) + data_unmasked_210_270 = raa_binning(data_unmasked, (210, 270)) + data_unmasked_270_330 = raa_binning(data_unmasked, (270, 330)) + data_unmasked_330_30 = raa_binning(data_unmasked, (330, 30)) + + data_unmasked_list = [ + data_unmasked_30_90, + data_unmasked_90_150, + data_unmasked_150_210, + data_unmasked_210_270, + data_unmasked_270_330, + data_unmasked_330_30, + ] + ds_dict = {} + for i in range(len(data_list)): + data_cutting = data_unmasked_list[i] + + if len(data_list[i]) == 0: + continue + + bins = [ + 0, + 2.5, + 7.5, + 12.5, + 17.5, + 22.5, + 27.5, + 32.5, + 37.5, + 42.5, + 47.5, + 52.5, + 57.5, + 62.5, + 67.5, + 72.5, + ] + bin_cents = [(bins[i] + bins[i + 1]) / 2 for i in range(len(bins) - 1)] + + if method == "plane_fit": + + binned95, bin_centers_sza, bin_centers_vza, upper_passes, da = ( + sza_vza_bin_and_calc( + data_list[i], + data_cutting, + "3sigma_max", + "upper", + input_wav=self.input_wav, + ) + ) + binned5, bin_centers_sza, bin_centers_vza, lower_passes, da = ( + sza_vza_bin_and_calc( + data_list[i], + data_cutting, + "3sigma_min", + "lower", + input_wav=self.input_wav, + ) + ) + + below, below_cov = new_plane_fit( + bin_centers_sza, bin_centers_vza, binned5 + ) + above, above_cov = new_plane_fit( + bin_centers_sza, bin_centers_vza, binned95 + ) + z_lower = quad_plane( + (data_cutting[" sza"].values, data_cutting[" vza"].values), *below + ) + z_upper = quad_plane( + (data_cutting[" sza"].values, data_cutting[" vza"].values), *above + ) + + outliers = data_cutting[ + (data_cutting[f" refl_{self.input_wav}nm"] > z_upper) + | (data_cutting[f" refl_{self.input_wav}nm"] < z_lower) + ] + good_data = data_cutting[ + (data_cutting[f" refl_{self.input_wav}nm"] < z_upper) + & (data_cutting[f" refl_{self.input_wav}nm"] > z_lower) + ] + + elif method == "bounds": + if i == 0: + self.bound_iteration += 1 + iter_num = self.bound_iteration + + + binned95, bin_centers_sza, bin_centers_vza, upper_passes, da = sza_vza_bin_and_calc(data_list[i], + data_cutting, + 'median_plus_3std', + 'upper', + input_wav = self.input_wav) + binned5, bin_centers_sza, bin_centers_vza, lower_passes, da = sza_vza_bin_and_calc(data_list[i], + data_cutting, + 'median_minus_3std', + 'lower', + input_wav = self.input_wav) + + + if len(binned5) < 6: + print('Too few points to fit') + continue + + below, below_cov = new_plane_fit(bin_centers_sza, bin_centers_vza, binned5) + above, above_cov = new_plane_fit(bin_centers_sza, bin_centers_vza, binned95) + + df_below = pd.DataFrame({'refl': binned5, 'vza': bin_centers_vza, 'sza': bin_centers_sza}) + df_above = pd.DataFrame({'refl': binned95, 'vza': bin_centers_vza, 'sza': bin_centers_sza}) + + passes = [True if v is True and lower_passes[i] is True else False for i, v in enumerate(upper_passes)] + good_data = data_cutting[passes] + outliers = data_cutting[[not el for el in passes]] + if save_bounds: + df_both = df_below.rename(columns={"refl": "lower_bound"}) + df_both = df_below.assign(upper_bound=df_above["refl"].values) + ds_dict[f"{raa_limits_list[i]}"] = df_both + + elif method == "plane_std_fit": + if i == 0: + self.plane_iteration += 1 + iter_num = self.plane_iteration + std_flat, bin_centers_sza, bin_centers_vza, lower_passes, stds = ( + sza_vza_bin_and_calc( + data_list[i], + data_cutting, + "std", + "na", + input_wav=self.input_wav, + ) + ) + + med_flat, bin_centers_sza, bin_centers_vza, lower_passes, medians = ( + sza_vza_bin_and_calc( + data_list[i], + data_cutting, + "median", + "na", + input_wav=self.input_wav, + ) + ) + + count_flat, bin_centers_sza, bin_centers_vza, lower_passes, counts = ( + sza_vza_bin_and_calc( + data_list[i], + data_cutting, + "count", + "na", + input_wav=self.input_wav, + ) + ) + + sza_fit = [] + vza_fit = [] + std_fit = [] + med_fit = [] + cnt_fit = [] + + for j, cn in enumerate(count_flat): + if ((bin_centers_vza[j] <= 5 and cn < 3) or + math.isnan(medians.flatten()[j]) or + (bin_centers_vza[j] >= 55 and cn < 50) or + (math.isnan(stds.flatten()[j])) or + cn < 3 + ): + continue + else: + sza_fit.append(bin_centers_sza[j]) + vza_fit.append(bin_centers_vza[j]) + std_fit.append(stds.flatten()[j]) + med_fit.append(medians.flatten()[j]) + cnt_fit.append(1 / counts.flatten()[j]) + + if len(med_fit) < 6: + print("Too few points to fit use bounds only") + continue + + max_sza = np.ceil(max(data_cutting[" sza"].values)) + max_vza = np.ceil(max(data_cutting[" vza"].values)) + max_sza_idx = np.abs(bin_cents - max_sza).argmin() + 1 + max_vza_idx = np.abs(bin_cents - max_vza).argmin() + 1 + + min_sza = np.floor(min(data_cutting[" sza"].values)) + min_vza = np.floor(min(data_cutting[" vza"].values)) + min_sza_idx = np.abs(bin_cents - min_sza).argmin() + min_vza_idx = np.abs(bin_cents - min_vza).argmin() + + if self.site == "GHNA": + std_coeffs, std_cov = new_plane_fit( + sza_fit, + vza_fit, + std_fit, + plane_type="quad", + ) # count = cnt_fit) + sza, vza = np.meshgrid( + bin_cents[min_sza_idx:max_sza_idx], + bin_cents[min_vza_idx:max_vza_idx], + ) + plane_std = quad_plane((sza, vza), *std_coeffs) + med_coeffs, med_cov = new_plane_fit( + sza_fit, + vza_fit, + med_fit, + ) # count = cnt_fit) + plane_med = quad_plane((sza, vza), *med_coeffs) + elif self.site == 'WWUK': + std_coeffs, std_cov = new_plane_fit(sza_fit, vza_fit, std_fit, plane_type='linear')#, count = cnt_fit) + sza, vza = np.meshgrid(bin_cents[min_sza_idx:max_sza_idx], bin_cents[min_vza_idx:max_vza_idx]) + plane_std = linear_plane((sza, vza), *std_coeffs) + med_coeffs, med_cov = new_plane_fit(sza_fit, vza_fit, med_fit, plane_type = 'linear')#, count = cnt_fit) + plane_med = linear_plane((sza, vza), *med_coeffs) + else: + std_coeffs, std_cov = new_plane_fit( + sza_fit, + vza_fit, + std_fit, + plane_type="quad", + ) # count = cnt_fit) + sza, vza = np.meshgrid( + bin_cents[min_sza_idx:max_sza_idx], + bin_cents[min_vza_idx:max_vza_idx], + ) + plane_std = quad_plane((sza, vza), *std_coeffs) + med_coeffs, med_cov = new_plane_fit( + sza_fit, + vza_fit, + med_fit, + ) # count = cnt_fit) + plane_med = quad_plane((sza, vza), *med_coeffs) + + plane_std[plane_std < 0] = 0 + plane_med[plane_med < 0] = 0 + plane_std = np.where(plane_std <= 0.03*plane_med, 0.03*plane_med, plane_std) + plane_std[plane_std <= 0] = gaussian_filter(plane_std, 1)[plane_std <= 0] + + upper_bounds = plane_med + 3 * plane_std + lower_bounds = plane_med - 3 * plane_std + + sza = np.pad(sza, 1, "edge") + sza[:, 0] = 0 + sza[:, -1] = sza[:, -2] + 5 + + vza = np.pad(vza, 1, "edge") + vza[0, :] = 0 + vza[-1, :] = vza[-2, :] + 5 + + upper_bounds = np.pad(upper_bounds, 1, "edge") + lower_bounds = np.pad(lower_bounds, 1, "edge") + + upper_bounds = upper_bounds.flatten()[~np.isnan(upper_bounds.flatten())] + lower_bounds = lower_bounds.flatten()[~np.isnan(lower_bounds.flatten())] + + lower_bounds[lower_bounds < 0] = 0 + upper_bounds = np.where(lower_bounds + 0.1*np.mean(plane_med) >= upper_bounds, + [np.max([x, 0.05]) for x in lower_bounds + 0.1*np.mean(plane_med)], + upper_bounds) + + df_below = pd.DataFrame( + {"refl": lower_bounds, "vza": vza.flatten(), "sza": sza.flatten()} + ) + df_above = pd.DataFrame( + {"refl": upper_bounds, "vza": vza.flatten(), "sza": sza.flatten()} + ) + + lower_interp = LinearNDInterpolator( + (sza.flatten(), vza.flatten()), lower_bounds + ) + lower_passes = [ + ( + True + if v + > lower_interp(data_cutting[" sza"], data_cutting[" vza"])[i] + else False + ) + for i, v in enumerate(data_cutting[f" refl_{self.input_wav}nm"]) + ] + + upper_interp = LinearNDInterpolator( + (sza.flatten(), vza.flatten()), upper_bounds + ) + upper_passes = [ + ( + True + if v + < upper_interp(data_cutting[" sza"], data_cutting[" vza"])[i] + else False + ) + for i, v in enumerate(data_cutting[f" refl_{self.input_wav}nm"]) + ] + + passes = [ + True if v is True and lower_passes[i] is True else False + for i, v in enumerate(upper_passes) + ] + good_data = data_cutting[passes] + outliers = data_cutting[[not el for el in passes]] + if save_bounds: + df_both = df_below.rename(columns={"refl": "lower_bound"}) + df_both = df_below.assign(upper_bound=df_above["refl"].values) + ds_dict[f"{raa_limits_list[i]}"] = df_both + else: + print("invalid_method") + exit() + + good_data.reset_index(drop=True, inplace=True) + outliers.reset_index(drop=True, inplace=True) + + if plot: + x_range = np.arange(0, 65, 5) + fig, axs = plt.subplots(1, 3, figsize=(6, 4), sharey=True) + for j in range(3): + axs[j].set_title(f"SZA = {(j + 1) * 20}") + axs[j].scatter( + outliers[ + outliers[" sza"].between((j + 1) * 20 - 1, (j + 1) * 20 + 1) + ][" vza"].values, + outliers[ + outliers[" sza"].between((j + 1) * 20 - 1, (j + 1) * 20 + 1) + ][f" refl_{self.input_wav}nm"].values, + color="red", + s=5, + ) + axs[j].scatter( + good_data[ + good_data[" sza"].between( + (j + 1) * 20 - 1, (j + 1) * 20 + 1 + ) + ][" vza"].values, + good_data[ + good_data[" sza"].between( + (j + 1) * 20 - 1, (j + 1) * 20 + 1 + ) + ][f" refl_{self.input_wav}nm"].values, + color="green", + s=5, + ) + + axs[j].scatter( + df_below.loc[ + (df_below["sza"] > (j + 1) * 20 - 1) + & (df_below["sza"] < (j + 1) * 20 + 1) + ]["vza"].values, + df_below.loc[ + (df_below["sza"] > (j + 1) * 20 - 1) + & (df_below["sza"] < (j + 1) * 20 + 1) + ]["refl"].values, + color="blue", + s=20, + marker="v", + ) + axs[j].scatter( + df_above.loc[ + (df_above["sza"] > (j + 1) * 20 - 1) + & (df_above["sza"] < (j + 1) * 20 + 1) + ]["vza"].values, + df_above.loc[ + (df_above["sza"] > (j + 1) * 20 - 1) + & (df_above["sza"] < (j + 1) * 20 + 1) + ]["refl"].values, + color="blue", + s=20, + marker="^", + ) + """ + axs[1, j].scatter( + df_below.loc[(df_below['sza'] > (j + 1) * 20 - 1) & (df_below['sza'] < (j + 1) * 20 + 1)][ + 'vza'].values, + df_below.loc[(df_below['sza'] > (j + 1) * 20 - 1) & (df_below['sza'] < (j + 1) * 20 + 1)][ + 'refl'].values, + color='blue', s=20, marker='v') + axs[1, j].scatter( + df_above.loc[(df_above['sza'] > (j + 1) * 20 - 1) & (df_above['sza'] < (j + 1) * 20 + 1)][ + 'vza'].values, + df_above.loc[(df_above['sza'] > (j + 1) * 20 - 1) & (df_above['sza'] < (j + 1) * 20 + 1)][ + 'refl'].values, + color='blue', s=20, marker='^') + + #norm = matplotlib.colors.CenteredNorm() + im = axs[1, j].scatter( + data_cutting[data_cutting[' sza'].between((j + 1) * 20 - 1, (j + 1) * 20 + 1)][' vza'].values, + data_cutting[data_cutting[' sza'].between((j + 1) * 20 - 1, (j + 1) * 20 + 1)][' refl_550nm'].values, + c = data_cutting[data_cutting[' sza'].between((j + 1) * 20 - 1, (j + 1) * 20 + 1)][' vaa'].values, + s=5, cmap = 'hsv') + cbar = fig.colorbar(im, ax=axs[1,2]) + cbar.set_label(label='VAA', fontsize=16) + + """ + axs[j].set_xlabel("VZA") + axs[0].set_ylabel("Reflectance") + fig.suptitle("RAA {}".format(raa_limits_list[i])) + fig.savefig( + r"T:/ECO/EOServer/joe/hypernets_plots/plane_models/" + + "sza_slices_3sig_{}_{}_{}_{}.png".format( + raa_limits_list[i], iter_num, method, self.period_name + ) + ) + plt.close() + # plt.show() + + if ("outs" and "good") not in locals(): + outs = outliers + good = good_data + + else: + outs = pd.concat([outs, outliers], ignore_index=True) + good = pd.concat([good, good_data], ignore_index=True) + + if nc_path is not None: + ds_raa_list = [] + ds_raa_keys_list = [] + for i, key in enumerate(ds_dict.keys()): + ds_raa_list.append(xr.Dataset.from_dataframe(ds_dict[key])) + ds_raa_keys_list.append(key) + ds_raa_vals_list = [ + np.mean([float(x) for x in ds_raa_keys_list[i].split("_")]) + for i in range(len(ds_raa_keys_list)) + ] + if "330_30" in ds_raa_keys_list: + ds_raa_vals_list[ds_raa_keys_list.index("330_30")] = 0 + + comb_ds = xr.concat(ds_raa_list, pd.Index(ds_raa_keys_list, name="raa")) + comb_ds = comb_ds.rename_vars({"refl": "lower_bound"}) + comb_ds.set_coords(("vza", "sza")) + + comb_ds.to_netcdf(nc_path) + + if labelled: + for y in self.labelled_radiance["post_processing_flags"][ + self.labelled_radiance.apply(tuple, axis=1).isin( + [tuple(x) for x in outs.values] + ) + ]: + y.append(5) + else: + self.radiance_good = good + self.radiance_outliers = outs + + if last_iteration: + self.radiance = good + + del outs + del good + + def bound_plotter(self, bounds_filepath,): + + bound_data = xr.open_dataset(bounds_filepath) + data = self.radiance + + data_30_90 = raa_binning(data, (30, 90)) + data_90_150 = raa_binning(data, (90, 150)) + data_150_210 = raa_binning(data, (150, 210)) + data_210_270 = raa_binning(data, (210, 270)) + data_270_330 = raa_binning(data, (270, 330)) + data_330_30 = raa_binning(data, (330, 30)) + + data_list = [data_30_90, data_90_150, data_150_210, data_210_270, data_270_330, data_330_30] + + for i, v in enumerate(bound_data.raa.values): + try: + raa_bin_data = bound_data.sel(raa = v) + data_cutting = data_list[i] + raa_bin_data = raa_bin_data.dropna(dim = 'index') + + sza = raa_bin_data.sza.values + vza = raa_bin_data.vza.values + lower_bounds = raa_bin_data.lower_bound.values + upper_bounds = raa_bin_data.upper_bound.values + + sza_matrix = sza.reshape((len(np.unique(vza)), np.sum(vza == vza[0]))) + vza_matrix = vza.reshape((len(np.unique(vza)), np.sum(vza == vza[0]))) + lower_matrix = lower_bounds.reshape((len(np.unique(vza)), np.sum(vza == vza[0]))) + upper_matrix = upper_bounds.reshape((len(np.unique(vza)), np.sum(vza == vza[0]))) + + sza_matrix = np.pad(sza_matrix, 1, 'edge') + sza_matrix[:,0] = 0 + try: + sza_matrix[:,-1] = np.nanmax(data_cutting[' sza']) + except ValueError: + sza_matrix[:,-1] = np.nanmax(sza_matrix) + + vza_matrix = np.pad(vza_matrix, 1, 'edge') + vza_matrix[0, :] = 0 + try: + vza_matrix[-1, :] = np.nanmax(data_cutting[' vza']) + except ValueError: + vza_matrix[-1, :] = np.nanmax(vza_matrix) + + lower_matrix = np.pad(lower_matrix, 1, 'edge') + + upper_matrix = np.pad(upper_matrix, 1, 'edge') + + sza = sza_matrix.flatten() + vza = vza_matrix.flatten() + lower_bounds = lower_matrix.flatten() + upper_bounds = upper_matrix.flatten() + + lower_interp = LinearNDInterpolator((sza, vza), lower_bounds) + lower_passes = [True if v > lower_interp(data_cutting[' sza'], data_cutting[' vza'])[i] else False for i, v in + enumerate(data_cutting[f' refl_{self.input_wav}nm'])] + + upper_interp = LinearNDInterpolator((sza, vza), upper_bounds) + upper_passes = [True if v < upper_interp(data_cutting[' sza'], data_cutting[' vza'])[i] else False for i, v in + enumerate(data_cutting[f' refl_{self.input_wav}nm'])] + + passes = [True if v is True and lower_passes[i] is True else False for i, v in enumerate(upper_passes)] + good_data = data_cutting[passes] + outliers = data_cutting[[not el for el in passes]] + + good_data.reset_index(drop=True, inplace=True) + outliers.reset_index(drop=True, inplace=True) + + df_below = pd.DataFrame({'refl': lower_bounds, 'vza': vza, 'sza': sza}) + df_above = pd.DataFrame({'refl': upper_bounds, 'vza': vza, 'sza': sza}) + + fig, axs = plt.subplots(1, 3, figsize=(6, 4), sharey=True) + for j in range(3): + axs[j].set_title(f'SZA = {(j + 1) * 20}') + axs[j].scatter( + outliers[outliers[' sza'].between((j + 1) * 20 - 1, (j + 1) * 20 + 1)][' vza'].values, + outliers[outliers[' sza'].between((j + 1) * 20 - 1, (j + 1) * 20 + 1)][f' refl_{self.input_wav}nm'].values, + color='red', s=5) + axs[j].scatter( + good_data[good_data[' sza'].between((j + 1) * 20 - 1, (j + 1) * 20 + 1)][' vza'].values, + good_data[good_data[' sza'].between((j + 1) * 20 - 1, (j + 1) * 20 + 1)][f' refl_{self.input_wav}nm'].values, + color='green', s=5) + + axs[j].scatter( + df_below.loc[(df_below['sza'] > (j + 1)*20 -1) & (df_below['sza'] < (j+1) *20 + 1)]['vza'].values, + df_below.loc[(df_below['sza'] > (j + 1)*20 -1) & (df_below['sza'] < (j+1) *20 + 1)]['refl'].values, + color='blue', s=20, marker = 'v') + axs[j].scatter( + df_above.loc[(df_above['sza'] > (j + 1) * 20 - 1) & (df_above['sza'] < (j + 1) * 20 + 1)]['vza'].values, + df_above.loc[(df_above['sza'] > (j + 1) * 20 - 1) & (df_above['sza'] < (j + 1) * 20 + 1)]['refl'].values, + color='blue', s=20, marker = '^') + axs[j].set_xlabel('VZA') + axs[0].set_ylabel('Reflectance') + fig.suptitle('RAA {}'.format(v)) + fig.savefig(r'T:/ECO/EOServer/joe/hypernets_plots/plane_models/' + + 'sza_slices_3sig_{}_{}.png'.format(v, self.period_name)) + except: + continue + + + def tol_optimiser(self, tol_start, tol_stop, tol_step, wavelength, good_limit, + plot = False): + IDs = self.irradiance['ID'] + tol = np.arange(tol_start, tol_stop, tol_step) + outs = [] + good = [] + + for i in tol: + self.cloud_passes = None + self.cloud_check(i, wavelength, False) + self.cloud_check(i, wavelength, False) + + cut_IDs = IDs[self.cloud_passes == 1] + out_cut = self.radiance_outliers[ + self.radiance_outliers["# id"].isin(cut_IDs) + ] + good_cut = self.radiance_good[self.radiance_good["# id"].isin(cut_IDs)] + outs.append(len(out_cut) * 100 / len(self.radiance_outliers)) + good.append(len(good_cut) * 100 / len(self.radiance_good)) + + optimiser = pd.DataFrame( + np.array((tol, outs, good)).T, columns=["tol", "outs", "good"] + ) + optimised = optimiser[optimiser["good"] < good_limit] + best_tol = optimised["tol"][(optimised["outs"] - optimised["good"]).argmax()] + + self.cloud_check_tol = best_tol + self.cloud_passes = None + + if plot: + fig, ax = plt.subplots(1, 1, figsize=(6, 4), dpi=150) + ax.plot(tol, outs, label="Outliers", color="r", linestyle="solid") + ax.plot(tol, good, label="Good Data", color="r", linestyle="dotted") + ax.set_xlabel("Tolerance") + ax.set_ylabel("Percentage of Data Removed") + ax.set_title("Cloud Check Tolerance Analysis for 550nm") + ax.legend() + plt.show() + + fig, ax = plt.subplots(1, 1, figsize=(6, 4), dpi=150) + ax.plot(tol, np.array(outs) - np.array(good), color="r", linestyle="solid") + ax.set_xlabel("Tolerance") + ax.set_ylabel("Percentage of Data Removed") + ax.set_title("Cloud Check Tolerance Analysis for 550nm") + plt.show() + + def mask(self, data_type): + + if data_type == "radiance": + data = self.radiance + elif data_type == "outliers": + data = self.radiance_outliers + elif data_type == "labelled": + data = self.labelled_radiance + else: + print("data not masked") + quit() + + mask_LOBE = np.where( + ((data[" vza"] < 10.5) & (data[" vza"] > 9.5) & (data[" vaa"] < 114) & (data[" vaa"] > 111)) + )[0] + + mask_LOBE_2 = np.where( + ((data[" vza"] < 10.5) & (data[" vza"] > 9.5) & (data[" vaa"] < 95) & (data[" vaa"] > 100)) + )[0] + + mask_WWUK = np.where( + ((data[" vaa"] < 70) | (data[" vaa"] > 300)) + )[0] + + mask11 = np.where( + ( + (data[" vza"] < 6) + & (data[" saa"] < 90) + & (data[" saa"] > 30) + & (data[" sza"] < 14) + ) + )[0] + + mask12 = np.where( + ( + (data[" vza"] < 6) + & (data[" saa"] < 90) + & (data[" saa"] > 60) + & (data[" sza"] < 20) + & (data[" sza"] > 14) + ) + )[0] + + mask13 = np.where( + ( + (data[" vza"] < 6) + & (data[" saa"] < 90) + & (data[" saa"] > 60) + & (data[" sza"] < 26) + & (data[" sza"] > 22) + ) + )[0] + + mask14 = np.where( + ( + (data[" vza"] < 6) + & (data[" saa"] < 110) + & (data[" saa"] > 70) + & (data[" sza"] > 53) + ) + )[0] + + mask2 = np.where( + ( + (data[" vza"] < 6) + & (data[" saa"] < 320) + & (data[" saa"] > 260) + & (data[" sza"] < 20) + ) + )[0] + + mask3 = np.where(data[" vza"] < 1)[0] + + mask4 = np.where( + ( + (data[" vza"] > 45) + & (data[" saa"] < 264) + & (data[" saa"] > 254) + & (data[" sza"] > 50) + ) + )[0] + + # mask5 = np.where(((data[' vza'] > 45) & (data['raa'] < 230) & (data['raa'] > 210) & (data[' sza'] > 65)))[0] #i think same data as 14 in v3 + + if self.site == "LOBE": + mask1 = np.unique(np.concat((mask_LOBE, mask_LOBE_2))) + if self.site == "WWUK": + mask1 = mask_WWUK + mask = np.unique((mask1)) + + if data_type == "radiance": + self.radiance = data.loc[~data.index.isin(mask)] + self.radiance.reset_index(drop=True, inplace=True) + elif data_type == "outliers": + self.radiance_outliers = data.loc[~data.index.isin(mask)] + elif data_type == "labelled": + for x in self.labelled_radiance["post_processing_flags"][mask]: + x.append(7) + else: + print("data not masked") + + def plot_ndvi_timeseries(self): + # mask = np.where((self.radiance[' vza'] > 8) & + # (self.radiance[' vza'] < 12) & + # (self.radiance[' vaa'] < 115) & + # (self.radiance[' vaa'] > 96))[0] + # self.radiance = self.radiance.drop(mask) + # self.radiance.reset_index(drop = True, inplace = True) + x = self.radiance.datetimes.values + y = self.radiance.ndvi.values + dates = self.radiance['date'] + summer_dates = [dat for dat in dates if dat[4:6] == '06' or dat[4:6] == '07' or dat[4:6] == '08'] + summer_y = [v for i, v in enumerate(y) if dates[i] in summer_dates] + y_mean = np.nanmean(summer_y) + y_std = np.nanstd(summer_y) + ys = pd.Series(y) + rolling_mean = ys.rolling(100).mean() + rolling_std = ys.rolling(100).std() + + crossings_inds = np.where(np.diff((rolling_std > 1.5*y_std).astype(int)) != 0)[0] + crossings = [int(x) for x in list(set(dates[crossings_inds].values))] + crossings.sort() + print(crossings) + + fig, ax = plt.subplots(2, 1, figsize = (20, 14)) + ax[0].scatter(x, y, color='g', s=2, alpha=0.7) + ax[0].plot(x, rolling_mean, color = 'b') + ax[0].fill_between(x, rolling_mean - rolling_std, rolling_mean + rolling_std, color = 'blue', alpha = 0.2) + #ax[0].axhline(y_mean, color = 'black') + #ax[0].axhline(y_mean + y_std, color='red') + #ax[0].axhline(y_mean - y_std, color='red') + #ax[0].axhline(y_mean + 2*y_std, color='orange') + #ax[0].axhline(y_mean - 2*y_std, color='orange') + #ax[0].axhline(y_mean + 3*y_std, color='yellow') + #ax[0].axhline(y_mean - 3*y_std, color='yellow') + + ax[1].plot(x, rolling_std) + ax[1].axhline(y_std, color='red') + ax[1].axhline(2 * y_std, color='orange') + ax[1].axhline(3 * y_std, color='yellow') + + ax[0].set_ylabel("NDVI") + ax[1].set_ylabel("NDVI StD") + ax[1].set_xlabel("Date") + ax[0].set_title(f"{self.period_name} NDVI Timeseries") + plt.gcf().autofmt_xdate() + fig.savefig( + root + + f"/joe/hypernets_plots/QC_pipeline_plots/{self.period_name}_ndvi_timeseries" + ) + + +def pipeline(dataset: PostProcessingDataset, maintenance, sza_limit, raa_limit): + print("Initial", len(dataset.radiance)) + dataset.maintenance_check(maintenance) + print("Maintenance", len(dataset.radiance)) + dataset.sza_check(sza_limit) + print("SZA", len(dataset.radiance)) + dataset.raa_cut(raa_limit) + print("RAA", len(dataset.radiance)) + dataset.clean_irr() + print("IRR Flags", len(dataset.radiance)) + dataset.cloud_check(0.9, 550, False) + dataset.plane_fitter() + dataset.size("good") + dataset.size("outliers") + dataset.cloud_check(0.9, 550, True, rewrite_var="good") + dataset.cloud_check(0.9, 550, True, rewrite_var="outliers") + dataset.size("good") + dataset.size("outliers") + # dataset.save('good', r'T:\ECO\EOServer\data\insitu\hypernets\post_processing_qc\GHNAv1_processed_good.csv') + # dataset.save('outliers', r'T:\ECO\EOServer\data\insitu\hypernets\post_processing_qc\GHNAv1_processed_out.csv') + dataset.mask("good") + dataset.mask("outliers") + dataset.size("good") + dataset.size("outliers") + # dataset.save('good', r'T:\ECO\EOServer\data\insitu\hypernets\post_processing_qc\GHNAv3_processed1_good.csv') + # dataset.save('outliers', r'T:\ECO\EOServer\data\insitu\hypernets\post_processing_qc\GHNAv3_processed1_out.csv') + + +def tolerance_analysis( + dataset: PostProcessingDataset, maintenance, sza_limit, raa_limit +): + # inital checks + dataset.maintenance_check(maintenance) + dataset.sza_check(sza_limit) + dataset.raa_cut(raa_limit) + dataset.clean_irr() + # inital outlier definition + dataset.new_plane_fitter(plot=True, method="bounds") + # sparse tolerance analysis + dataset.tol_optimiser(0.4, 1.1, 0.1, 550, 100, True) + # use best tolerance to define planes + dataset.cloud_check(dataset.cloud_check_tol, 550, False) + dataset.plane_fitter() + # in depth tolerance analysis + dataset.tol_optimiser( + dataset.cloud_check_tol - 0.1, + dataset.cloud_check_tol + 0.1, + 0.01, + 550, + 10, + True, + ) + # use best tolerance to define planes + dataset.cloud_check(dataset.cloud_check_tol, 550, False) + dataset.plane_fitter() + dataset.plane_fitter(True) + dataset.size("good") + dataset.size("outliers") + # cloud check final definition of data + dataset.cloud_check(dataset.cloud_check_tol, 550, True, rewrite_var="good") + dataset.cloud_check(dataset.cloud_check_tol, 550, True, rewrite_var="outliers") + dataset.cloud_check(dataset.cloud_check_tol, 550, True, rewrite_var="labels") + print(dataset.cloud_check_tol) + dataset.size("good") + dataset.size("outliers") + + +def JSIT_pipeline(dataset: PostProcessingDataset, maintenance, sza_limit, raa_limit): + print("Initial", len(dataset.radiance)) + dataset.maintenance_check(maintenance) + print("Maintenance", len(dataset.radiance)) + dataset.sza_check(sza_limit) + print("SZA", len(dataset.radiance)) + dataset.raa_cut(raa_limit) + print("RAA", len(dataset.radiance)) + dataset.clean_irr() + print("IRR Flags", len(dataset.radiance)) + + dataset.cloud_check(0.9, 550, True, "radiance") + dataset.save( + "radiance", + r"T:\ECO\EOServer\data\insitu\hypernets\post_processing_qc\JSIT_initial_cc9.csv", + ) + + +def multi_aod_tolerance_analysis( + dataset: PostProcessingDataset, maintenance, sza_limit, raa_limit +): + # inital checks + dataset.maintenance_check(maintenance) + dataset.sza_check(sza_limit) + dataset.raa_cut(raa_limit) + dataset.clean_irr() + +def new_plane_pipeline(dataset: PostProcessingDataset, maintenance, sza_limit, raa_limit, misalign_vza, misalign_vaa, mean_corr, + multiple_periods = False, period_date = None, fit_method = 'plane_fit', + split_by_date = False, start = None, end = None): + print('Initial', len(dataset.radiance)) + dataset.maintenance_check(maintenance) + print("Maintenance", len(dataset.radiance)) + dataset.sza_check(sza_limit) + print("SZA", len(dataset.radiance)) + dataset.raa_cut(raa_limit) + print("RAA", len(dataset.radiance)) + if dataset.site == "JSIT": + dataset.vza_check(7) + print('VZA', len(dataset.radiance)) + if dataset.site == 'WWUK': + dataset.vza_check(15) + print('VZA', len(dataset.radiance)) + dataset.clean_irr() + print('IRR Flags', len(dataset.radiance)) + dataset.vza_check_toplimit(90) + dataset.calculate_ndvi() + dataset.calculate_savi() + dataset.calculate_s2tci() + + #dataset.save('radiance', r'T:/ECO/EOServer/data/insitu/hypernets/post_processing_qc/WWUK_initalQC.csv') + if misalign_vza != 0 and misalign_vaa != 0: + dataset.misalignment_correction(misalign_vza, misalign_vaa, mean_corr, multiple_periods, period_date) + #dataset.save('radiance', r'T:/ECO/EOServer/data/insitu/hypernets/post_processing_qc/GHNA_v3_misalignment_corrected.csv') + dataset.cloud_check(0.9, 550, True, rewrite_var = 'radiance') + dataset.save('radiance', f'T:/ECO/EOServer/data/insitu/hypernets/post_processing_qc/{dataset.period_name}_CC.csv') + print('Cloud Check', len(dataset.radiance)) + if split_by_date is True: + dataset.split_by_date(start, end) + if dataset.site == 'LOBE' or dataset.site == 'WWUK': + dataset.mask('radiance') + dataset.new_plane_fitter(plot = True, method = fit_method) + print('1st Iteration') + dataset.size('good') + dataset.size('outliers') + dataset.new_plane_fitter(first_iteration = False, plot = True, method = fit_method) + print('2nd Iteration') + dataset.size('good') + dataset.size('outliers') + dataset.new_plane_fitter(first_iteration = False, plot = True, method = fit_method, last_iteration = True) + print('3rd Iteration') + dataset.size('good') + dataset.size('outliers') + dataset.save('good', f'T:/ECO/EOServer/data/insitu/hypernets/post_processing_qc/{dataset.period_name}_bounds_good.csv') + dataset.save('outliers', f'T:/ECO/EOServer/data/insitu/hypernets/post_processing_qc/{dataset.period_name}_bounds_out.csv') + + dataset.new_plane_fitter(first_iteration = False, plot = True, method = 'plane_std_fit') + print('Plane Fit 1st Iteration') + dataset.size('good') + dataset.size('outliers') + dataset.new_plane_fitter(first_iteration = False, plot = True, method = 'plane_std_fit') + print('Plane Fit 2nd Iteration') + dataset.size('good') + dataset.size('outliers') + dataset.new_plane_fitter(first_iteration = False, plot = True, method = 'plane_std_fit', last_iteration = True, + save_bounds=True, + nc_path=f'T:/ECO/EOServer//data/insitu/hypernets/post_processing_qc/bounds/{dataset.period_name}_bounds.nc' + ) + print('Plane Fit 3rd Iteration') + dataset.size('good') + dataset.size('outliers') + #dataset.count_by_bin() + dataset.save('good', f'T:/ECO/EOServer/data/insitu/hypernets/post_processing_qc/{dataset.period_name}_QC_good.csv') + dataset.save('outliers', f'T:/ECO/EOServer/data/insitu/hypernets/post_processing_qc/{dataset.period_name}_QC_out.csv') + + +def alternate_bound_pipeline(dataset: PostProcessingDataset, maintenance, sza_limit, raa_limit, misalign_vza, misalign_vaa, mean_corr, + bound_filepath, + multiple_periods = False, period_date = None, + split_by_date = False, start = None, end = None): + print('Initial', len(dataset.radiance)) + dataset.maintenance_check(maintenance) + print('Maintenance', len(dataset.radiance)) + dataset.sza_check(sza_limit) + print('SZA', len(dataset.radiance)) + dataset.raa_cut(raa_limit) + print('RAA', len(dataset.radiance)) + if dataset.site == 'JSIT': + dataset.vza_check(7) + print('VZA', len(dataset.radiance)) + if dataset.site == 'WWUK': + dataset.vza_check(15) + print('VZA', len(dataset.radiance)) + dataset.clean_irr() + print('IRR Flags', len(dataset.radiance)) + dataset.vza_check_toplimit(90) + dataset.calculate_ndvi() + dataset.calculate_savi() + dataset.calculate_s2tci() + + #dataset.save('radiance', r'T:/ECO/EOServer/data/insitu/hypernets/post_processing_qc/WWUK_initalQC.csv') + if misalign_vza != 0 and misalign_vaa != 0: + dataset.misalignment_correction(misalign_vza, misalign_vaa, mean_corr, multiple_periods, period_date) + #dataset.save('radiance', r'T:/ECO/EOServer/data/insitu/hypernets/post_processing_qc/GHNA_v3_misalignment_corrected.csv') + dataset.cloud_check(0.9, 550, True, rewrite_var = 'radiance') + dataset.save('radiance', f'T:/ECO/EOServer/data/insitu/hypernets/post_processing_qc/{dataset.period_name}_CC.csv') + print('Cloud Check', len(dataset.radiance)) + if split_by_date is True: + dataset.split_by_date(start, end) + if dataset.site == 'LOBE' or dataset.site == 'WWUK': + dataset.mask('radiance') + dataset.bound_plotter(bound_filepath) + +def irradiance_analysis_writer(dataset: PostProcessingDataset, maintenance, sza_limit, raa_limit, filename, wvs): + print('Initial', len(dataset.radiance)) + dataset.maintenance_check(maintenance) + print('Maintenance', len(dataset.radiance)) + dataset.sza_check(sza_limit) + print('SZA', len(dataset.radiance)) + dataset.raa_cut(raa_limit) + print('RAA', len(dataset.radiance)) + dataset.clean_irr() + print('IRR Flags', len(dataset.radiance)) + dataset.write_irr_analysis_files(filename, wvs) + print('File Written') + +def aod_plotter(dataset: PostProcessingDataset, maintenance, sza_limit, raa_limit, filename, wvs): + print('Initial', len(dataset.radiance)) + dataset.maintenance_check(maintenance) + print("Maintenance", len(dataset.radiance)) + dataset.sza_check(sza_limit) + print("SZA", len(dataset.radiance)) + dataset.raa_cut(raa_limit) + print("RAA", len(dataset.radiance)) + dataset.clean_irr() + print('IRR Flags', len(dataset.radiance)) + dataset.append_aod() + dataset.write_irr_analysis_files(filename, wvs, save = True, plot = True, interp = True) + print('Done') + +def ndvi_bound_calculator(dataset: PostProcessingDataset, maintenance, sza_limit, raa_limit): + dataset.vza_check_toplimit(90) + print('Initial', len(dataset.radiance)) + dataset.maintenance_check(maintenance) + print("Maintenance", len(dataset.radiance)) + dataset.sza_check(sza_limit) + print("SZA", len(dataset.radiance)) + dataset.raa_cut(raa_limit) + print('RAA', len(dataset.radiance)) + if dataset.site == 'JSIT': + dataset.vza_check(7) + print('VZA', len(dataset.radiance)) + if dataset.site == 'WWUK': + dataset.vza_check(15) + print('VZA', len(dataset.radiance)) + dataset.clean_irr() + print('IRR Flags', len(dataset.radiance)) + if dataset.site == 'LOBE' or dataset.site == 'WWUK': + dataset.mask('radiance') + print('Mask', len(dataset.radiance)) + dataset.calculate_ndvi() + dataset.plot_ndvi_timeseries() + +def misalignment_correction_writer(dataset: PostProcessingDataset, vza, vaa, mean_corr, + multiple_periods = False, period_date = None): + dataset.misalignment_correction(vza, vaa, mean_corr, multiple_periods, period_date) + dataset.save('radiance', + r'T:/ECO/EOServer/data/insitu/hypernets/post_processing_qc/GHNA_2023-10-26_present_None_None_None_None_misalign_corrected.csv') + +JSIT_dict = {'NovDec24Jan25': ('2024-11-14', '2025-01-31'), + 'FebApr25': ('2025-02-01', '2025-04-30')} + +JSIT_maintenance_dates = [] +''' +for k, v in JSIT_dict.items(): + QC = PostProcessingDataset(r'T:/ECO/EOServer/data/insitu/hypernets/post_processing_qc/LOBE_2023-05-31_2023-08-11_None_None_None_None_misalign_corrected.csv', + r'T:/ECO/EOServer//data/insitu/hypernets/post_processing_qc/LOBEv4_irradiance.csv', + r'T:/ECO/EOServer//data/insitu/hypernets/post_processing_qc/irradiance/', + 'LOBE', + f'JSIT_{k}', + '0.1') + + new_plane_pipeline(QC, JSIT_maintenance_dates, 70, 10, 0.78, 176.9, False, + fit_method = 'bounds', split_by_date = True, start = v[0], end = v[1]) +''' +QC = PostProcessingDataset(r'T:/ECO/EOServer/data/insitu/hypernets/post_processing_qc/GHNA_2023-10-26_present_None_None_None_None.csv', + r'T:/ECO/EOServer//data/insitu/hypernets/post_processing_qc/GHNAv3_irradiance.csv', + r'T:/ECO/EOServer//data/insitu/hypernets/post_processing_qc/irradiance/', + 'GHNA', + f'GHNA_2023Oct_2025Aug', + 'median', + '550') +GHNA_maintenance_dates = [ + "20231016", + "20231017", + "20231018", + "20231019", + "20231020", + "20231021", + "20231022", + "20231023", + "20231024", + "20231025", + "20240513", + "20240514", + "20240515", + "20240516", + "20240517", + "20240518", + "20240519", + "20240520", +] + + +WWUK_maintenance_dates = [] +JAES_maintenance_dates = [] +LOBE_maintenance_dates = [] + +#ndvi_bound_calculator(QC, LOBE_maintenance_dates, 60, 10) +#misalignment_correction_writer(QC, (1.96, 1.28), (-72.2, -36.4), (1.009, 1.011), True, '20240521') + +#QC = PostProcessingDataset(r'T:/ECO/EOServer/data/insitu/hypernets/post_processing_qc/GHNA_2023-10-26_present_None_None_None_None_misalign_corrected.csv', + # r'T:/ECO/EOServer//data/insitu/hypernets/post_processing_qc/GHNAv3_irradiance.csv', + # r'T:/ECO/EOServer//data/insitu/hypernets/post_processing_qc/irradiance/', + # 'GHNA', + # f'GHNA_Oct23_May24', + # 'median', + # '550') + +#new_plane_pipeline(QC, GHNA_maintenance_dates, 60, 10, 0, #(1.76,1.59), + # 0,# (-80, -77), + # 0, + # False, + # split_by_date = True, start = '20231026', end = '20240521', + # fit_method = 'bounds') +#QC.period_name = f'GHNA_May24_Aug25' +#new_plane_pipeline(QC, GHNA_maintenance_dates, 60, 10, 0, #(1.76,1.59), + # 0,# (-80, -77), + # 0, + # False, + # split_by_date = True, start = '20240521', end = '20250801', + # fit_method = 'bounds') + + +#new_plane_pipeline(QC, JSIT_maintenance_dates, 60, 10, + # 1.67, 8.9, 0.972, + # False, + # split_by_date = True, start = '20241114', end = '20250801', + # fit_method = 'bounds') + +#new_plane_pipeline(QC, GHNA_maintenance_dates, 60, 10, + # 1.01, + # 96.7, + # 0.996, + # False, + # fit_method = 'bounds') + +#new_plane_pipeline(QC, WWUK_maintenance_dates, + # 60, 10, + # 1.58, -163.7, 1.016, False, + # split_by_date = True, start = '20230429', end = '20231101', + # fit_method = 'bounds') + +#irradiance_analysis_writer(QC, LOBE_maintenance_dates, 70, 10, + # 'irradiance_LOBEv3_analysis', + # [415, 490, 550, 665, 675, 705, 740, 765, 842, 870, 1020, 1640]) + +ndvi_bound_calculator(QC, GHNA_maintenance_dates, 60, 10) +#new_plane_pipeline(QC, JAES_maintenance_dates, + # 60, 10, + # 1.23, 42.2, 0.989, + # split_by_date = True, start = '20240605', end = '20240904', + # fit_method = 'bounds') + +#new_plane_pipeline(QC, LOBE_maintenance_dates, + # 60, 10, + # 7.80, -7.71, 0.905, + # False, + # split_by_date = True, start = '20250401', end = '20250503', + # fit_method = 'bounds') + +#aod_plotter(QC, LOBE_maintenance_dates, 60, 10, + # 'irradiance_LOBEv3_analysis', + # [415, 490, 550, 665, 675, 705, 740, 765, 842, 870, 1020, 1640]) + +#alternate_bound_pipeline(QC, LOBE_maintenance_dates, + # 60, 10, + # 0, 0, 0, + # "T:/ECO/EOServer/data/insitu/hypernets/post_processing_qc/bounds/LOBE_2025Apr_2025May_bounds.nc", + # False,) \ No newline at end of file diff --git a/docs/atbd/HYPROCESSOR_ATBD.nlo b/hypernets_processor/post_processing/RadCalNet_conversion/RadCalNet_conversion/__init__.py old mode 100755 new mode 100644 similarity index 100% rename from docs/atbd/HYPROCESSOR_ATBD.nlo rename to hypernets_processor/post_processing/RadCalNet_conversion/RadCalNet_conversion/__init__.py diff --git a/hypernets_processor/post_processing/RadCalNet_conversion/RadCalNet_conversion/convert_radcalnet.py b/hypernets_processor/post_processing/RadCalNet_conversion/RadCalNet_conversion/convert_radcalnet.py new file mode 100644 index 00000000..caf04716 --- /dev/null +++ b/hypernets_processor/post_processing/RadCalNet_conversion/RadCalNet_conversion/convert_radcalnet.py @@ -0,0 +1,241 @@ +"""describe class""" + +"""___Built-In Modules___""" +# import here + +"""___Third-Party Modules___""" +# import here +import pickle +from dateutil.parser import parse +import datetime +import numpy as np +import xarray as xr +from typing import Optional, Union, List, Any + +"""___NPL Modules___""" +# import here +import comet_maths as cm +import matheo.band_integration as bi +from brdf_model import BRDFModelFactory, BRDFParameters + + +"""___Authorship___""" +__author__ = "Pieter De Vis" +__created__ = "01/02/2021" +__maintainer__ = "Pieter De Vis" +__email__ = "pieter.de.vis@npl.co.uk" +__status__ = "Development" + + +class Convert_RadCalNet: + def __init__( + self, + path, + brdf_file, + irr_file, + ): + self.path = path + self.brdf_model_factory = BRDFModelFactory() + with open(brdf_file, "rb") as f: + self.brdf_params = pickle.load(f) + self.irr_all = xr.open_dataset(irr_file) + + def convert(self, refl): + # do processing + # save file + return refl + + def interpolate_direct_diffuse_ratio(self, datetimes, sza=None, wavelength=None): + ds_irr_temp = self.irr_all.isel(datetime=np.where(self.irr_all["sza"] < 90)[0]) + ds_irr_temp["direct_to_diffuse_irradiance_ratio"].values = ( + ds_irr_temp["direct_to_diffuse_irradiance_ratio"].values + / np.cos(ds_irr_temp["sza"].values / 180 * np.pi)[:, None] + ) + ds_irr_temp = ds_irr_temp.interp(datetime=datetimes, method="linear") + if sza is not None: + ds_irr_temp["sza"].values = sza + ds_irr_temp["direct_to_diffuse_irradiance_ratio"].values = ( + ds_irr_temp["direct_to_diffuse_irradiance_ratio"].values + * np.cos(ds_irr_temp["sza"].values / 180 * np.pi)[:, None] + ) + if wavelength is not None: + i_wav = np.argmin(np.abs(wavelength - ds_irr_temp.wavelength.values)) + return ds_irr_temp["direct_to_diffuse_irradiance_ratio"].values[:, i_wav] + else: + return ds_irr_temp["direct_to_diffuse_irradiance_ratio"].values.T + + import pandas as pd + from pysolar.solar import get_altitude + + __author__ = "Pieter De Vis " + + def convert_datetime( + self, + date_time: Union[datetime.datetime, datetime.date, str, float, int, np.ndarray], + ) -> datetime.datetime: + """ + Convert input datetimes to a datetime object + + :param date_time: date time to convert to a datetime object + :return: datetime object corresponding to input date_time + """ + if isinstance(date_time, np.ndarray): + return np.array( + [self.convert_datetime(date_time_i) for date_time_i in date_time] + ) + elif isinstance(date_time, datetime.datetime): + return date_time + elif isinstance(date_time, datetime.date): + return datetime.datetime.combine(date_time, datetime.time()) + elif isinstance(date_time, np.datetime64): + unix_epoch = np.datetime64(0, "s") + one_second = np.timedelta64(1, "s") + seconds_since_epoch = (date_time - unix_epoch) / one_second + return datetime.datetime.utcfromtimestamp(seconds_since_epoch) + elif isinstance( + date_time, + ( + float, + int, + np.uint, + np.float32, + np.float64, + np.int64, + np.int32, + np.uint64, + np.uint32, + ), + ): + return datetime.datetime.utcfromtimestamp(date_time) + else: + print(type(date_time)) + if date_time[-1] == "Z": + date_time = date_time[:-1] + try: + return parse(date_time, fuzzy=False) + except ValueError: + raise ValueError( + "Unable to discern datetime requested: '{}'".format(date_time) + ) + + def correct_to_nadir(self, ds_hyp): + vza = ds_hyp.viewing_zenith_angle.values + sza = ds_hyp.solar_zenith_angle.values + vaa = ds_hyp.viewing_azimuth_angle.values + saa = ds_hyp.solar_azimuth_angle.values + times = [self.convert_datetime(time) for time in ds_hyp.acquisition_time.values] + + brdf_model = self.brdf_model_factory.get_brdf_model( + "RPV", + sza, + vza, + saa, + vaa, + direct_to_diffuse_irr=self.interpolate_direct_diffuse_ratio(times, sza), + ) + + HCRF_model = brdf_model.return_HCRF_for_angle( + sza=sza, + vza=vza, + vaa=vaa, + saa=saa, + brdf_params=self.brdf_params.get_values().T, + ) + HCRF_model_nadir = brdf_model.return_HCRF_for_angle( + sza=sza, + vza=np.zeros_like(vza), + vaa=vaa, + saa=saa, + brdf_params=self.brdf_params.get_values().T, + ) + ds_hyp.reflectance.values = ds_hyp.reflectance.values * cm.interpolate_1d( + self.irr_all.wavelength.values, + (HCRF_model_nadir / HCRF_model), + ds_hyp.wavelength.values, + ) + return ds_hyp + + def spectral_processing( + self, + radcalnet_wavelength, + hypernets_wavelengths, + hypernets_reflectance, + asd_wavelengths, + asd_reflectances, + ): + # do spectral processing here + + # identify last 10nm of HN data and overlap with ASD by 10nm + hyp_mean_last10nm = np.mean( + hypernets_reflectance[ + np.where( + (hypernets_wavelengths > 1670) & (hypernets_wavelengths < 1680) + ) + ] + ) + asd_mean_10nm = np.mean( + asd_reflectances[ + np.where((asd_wavelengths > 1670) & (asd_wavelengths < 1680)) + ] + ) + + # determine correction factor from diff between the two + correction_factor = hyp_mean_last10nm / asd_mean_10nm + print("correction_factor = ", correction_factor) + + # add the asd dataset to the end of the HN dataset, adding correction factor to the asd data so that the join is smooth + combined_wavelengths = np.append( + hypernets_wavelengths[np.where(hypernets_wavelengths < 1680)], + asd_wavelengths[np.where(asd_wavelengths > 1680)], + ) + combined_reflectances = np.append( + hypernets_reflectance[np.where(hypernets_wavelengths < 1680)], + correction_factor * asd_reflectances[np.where(asd_wavelengths > 1680)], + ) + + # band integration using matheo to match RCN 10nm triangular format + # https://matheo.readthedocs.io/en/latest/content/user_guide.html + radcalnet_reflectances = bi.pixel_int( + d=combined_reflectances, + x=combined_wavelengths, + x_pixel=radcalnet_wavelength, + width_pixel=10.0 * np.ones(len(radcalnet_wavelength)), + band_shape="triangle", + ) + + # plots for sanity check + # plt.plot(hypernets_wavelengths,hypernets_reflectance, label='GHNA Instrument') + # plt.plot(asd_wavelengths,asd_reflectances, label='ASD') + # plt.plot(radcalnet_wavelength,radcalnet_reflectances, label='GHNA Interpolated') + # plt.grid() + # plt.legend() + # plt.title('Dataset Comparison', fontsize=20) + # plt.xlabel('Wavelength (nm)', fontsize=20) + # plt.ylabel('Reflectance', fontsize=20) + # plt.savefig('comparison_%s.png'%(rcn_datetime)) + # # plt.show() + return radcalnet_reflectances + + def time_interpolation( + self, + hypernets_datetime_pre, + rcn_refl_pre, + hypernets_datetime_aft, + rcn_refl_aft, + rcn_datetime, + ): + time_measured = np.array([hypernets_datetime_pre, hypernets_datetime_aft]) + refl_measured = np.array([rcn_refl_pre, rcn_refl_aft]) + time_target = rcn_datetime + print(time_measured.shape, refl_measured.shape) + refl_interpolated = cm.interpolate_1d( + time_measured, + refl_measured, + time_target, + u_y_i=None, + method="linear", + return_uncertainties=False, + return_corr=False, + interpolate_axis=0, + ) + return refl_interpolated diff --git a/hypernets_processor/post_processing/RadCalNet_conversion/RadCalNet_conversion/tests/__init__.py b/hypernets_processor/post_processing/RadCalNet_conversion/RadCalNet_conversion/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/hypernets_processor/post_processing/RadCalNet_conversion/RadCalNet_conversion/tests/test_class_template.py b/hypernets_processor/post_processing/RadCalNet_conversion/RadCalNet_conversion/tests/test_class_template.py new file mode 100644 index 00000000..7779717e --- /dev/null +++ b/hypernets_processor/post_processing/RadCalNet_conversion/RadCalNet_conversion/tests/test_class_template.py @@ -0,0 +1,26 @@ +"""Tests for classname module""" + +"""___Built-In Modules___""" +# import here + +"""___Third-Party Modules___""" +import unittest + +"""___NPL Modules___""" +# import here + +"""___Authorship___""" +__author__ = "Pieter De Vis" +__created__ = "01/02/2021" +__maintainer__ = "Pieter De Vis" +__email__ = "pieter.de.vis@npl.co.uk" +__status__ = "Development" + + +class TestClassName(unittest.TestCase): + def test_function1(self): + pass + + +if __name__ == "__main__": + unittest.main() diff --git a/hypernets_processor/post_processing/RadCalNet_conversion/__init__.py b/hypernets_processor/post_processing/RadCalNet_conversion/__init__.py new file mode 100644 index 00000000..025f40f5 --- /dev/null +++ b/hypernets_processor/post_processing/RadCalNet_conversion/__init__.py @@ -0,0 +1,9 @@ +"""GHNA_processing - Process to convert HYPSTAR data GobabebHYPSTAR data to RadCalNet network format.""" + +__author__ = "Morven Sinclair " +__all__ = [] + +from ._version import get_versions + +__version__ = get_versions()["version"] +del get_versions diff --git a/hypernets_processor/post_processing/RadCalNet_conversion/_version.py b/hypernets_processor/post_processing/RadCalNet_conversion/_version.py new file mode 100644 index 00000000..7ec9cd73 --- /dev/null +++ b/hypernets_processor/post_processing/RadCalNet_conversion/_version.py @@ -0,0 +1,556 @@ +# This file helps to compute a version number in source trees obtained from +# git-archive tarball (such as those provided by githubs download-from-tag +# feature). Distribution tarballs (built by setup.py sdist) and build +# directories (produced by setup.py build) will contain a much shorter file +# that just contains the computed version number. + +# This file is released into the public domain. Generated by +# versioneer-0.18 (https://github.com/warner/python-versioneer) + +"""Git implementation of _version.py.""" + +import errno +import os +import re +import subprocess +import sys + + +def get_keywords(): + """Get the keywords needed to look up the version information.""" + # these strings will be replaced by git during git-archive. + # setup.py/versioneer.py will grep for the variable names, so they must + # each be defined on a line of their own. _version.py will just call + # get_keywords(). + git_refnames = "$Format:%d$" + git_full = "$Format:%H$" + git_date = "$Format:%ci$" + keywords = {"refnames": git_refnames, "full": git_full, "date": git_date} + return keywords + + +class VersioneerConfig: + """Container for Versioneer configuration parameters.""" + + +def get_config(): + """Create, populate and return the VersioneerConfig() object.""" + # these strings are filled in when 'setup.py versioneer' creates + # _version.py + cfg = VersioneerConfig() + cfg.VCS = "git" + cfg.style = "pep440" + cfg.tag_prefix = "" + cfg.parentdir_prefix = "" + cfg.versionfile_source = "GHNA_processing/_version.py" + cfg.verbose = False + return cfg + + +class NotThisMethod(Exception): + """Exception raised if a method is not valid for the current scenario.""" + + +LONG_VERSION_PY = {} +HANDLERS = {} + + +def register_vcs_handler(vcs, method): # decorator + """Decorator to mark a method as the handler for a particular VCS.""" + + def decorate(f): + """Store f in HANDLERS[vcs][method].""" + if vcs not in HANDLERS: + HANDLERS[vcs] = {} + HANDLERS[vcs][method] = f + return f + + return decorate + + +def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, env=None): + """Call the given command(s).""" + assert isinstance(commands, list) + p = None + for c in commands: + try: + dispcmd = str([c] + args) + # remember shell=False, so use git.cmd on windows, not just git + p = subprocess.Popen( + [c] + args, + cwd=cwd, + env=env, + stdout=subprocess.PIPE, + stderr=(subprocess.PIPE if hide_stderr else None), + ) + break + except EnvironmentError: + e = sys.exc_info()[1] + if e.errno == errno.ENOENT: + continue + if verbose: + print("unable to run %s" % dispcmd) + print(e) + return None, None + else: + if verbose: + print("unable to find command, tried %s" % (commands,)) + return None, None + stdout = p.communicate()[0].strip() + if sys.version_info[0] >= 3: + stdout = stdout.decode() + if p.returncode != 0: + if verbose: + print("unable to run %s (error)" % dispcmd) + print("stdout was %s" % stdout) + return None, p.returncode + return stdout, p.returncode + + +def versions_from_parentdir(parentdir_prefix, root, verbose): + """Try to determine the version from the parent directory name. + + Source tarballs conventionally unpack into a directory that includes both + the project name and a version string. We will also support searching up + two directory levels for an appropriately named parent directory + """ + rootdirs = [] + + for i in range(3): + dirname = os.path.basename(root) + if dirname.startswith(parentdir_prefix): + return { + "version": dirname[len(parentdir_prefix) :], + "full-revisionid": None, + "dirty": False, + "error": None, + "date": None, + } + else: + rootdirs.append(root) + root = os.path.dirname(root) # up a level + + if verbose: + print( + "Tried directories %s but none started with prefix %s" + % (str(rootdirs), parentdir_prefix) + ) + raise NotThisMethod("rootdir doesn't start with parentdir_prefix") + + +@register_vcs_handler("git", "get_keywords") +def git_get_keywords(versionfile_abs): + """Extract version information from the given file.""" + # the code embedded in _version.py can just fetch the value of these + # keywords. When used from setup.py, we don't want to import _version.py, + # so we do it with a regexp instead. This function is not used from + # _version.py. + keywords = {} + try: + f = open(versionfile_abs, "r") + for line in f.readlines(): + if line.strip().startswith("git_refnames ="): + mo = re.search(r'=\s*"(.*)"', line) + if mo: + keywords["refnames"] = mo.group(1) + if line.strip().startswith("git_full ="): + mo = re.search(r'=\s*"(.*)"', line) + if mo: + keywords["full"] = mo.group(1) + if line.strip().startswith("git_date ="): + mo = re.search(r'=\s*"(.*)"', line) + if mo: + keywords["date"] = mo.group(1) + f.close() + except EnvironmentError: + pass + return keywords + + +@register_vcs_handler("git", "keywords") +def git_versions_from_keywords(keywords, tag_prefix, verbose): + """Get version information from git keywords.""" + if not keywords: + raise NotThisMethod("no keywords at all, weird") + date = keywords.get("date") + if date is not None: + # git-2.2.0 added "%cI", which expands to an ISO-8601 -compliant + # datestamp. However we prefer "%ci" (which expands to an "ISO-8601 + # -like" string, which we must then edit to make compliant), because + # it's been around since git-1.5.3, and it's too difficult to + # discover which version we're using, or to work around using an + # older one. + date = date.strip().replace(" ", "T", 1).replace(" ", "", 1) + refnames = keywords["refnames"].strip() + if refnames.startswith("$Format"): + if verbose: + print("keywords are unexpanded, not using") + raise NotThisMethod("unexpanded keywords, not a git-archive tarball") + refs = set([r.strip() for r in refnames.strip("()").split(",")]) + # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of + # just "foo-1.0". If we see a "tag: " prefix, prefer those. + TAG = "tag: " + tags = set([r[len(TAG) :] for r in refs if r.startswith(TAG)]) + if not tags: + # Either we're using git < 1.8.3, or there really are no tags. We use + # a heuristic: assume all version tags have a digit. The old git %d + # expansion behaves like git log --decorate=short and strips out the + # refs/heads/ and refs/tags/ prefixes that would let us distinguish + # between branches and tags. By ignoring refnames without digits, we + # filter out many common branch names like "release" and + # "stabilization", as well as "HEAD" and "master". + tags = set([r for r in refs if re.search(r"\d", r)]) + if verbose: + print("discarding '%s', no digits" % ",".join(refs - tags)) + if verbose: + print("likely tags: %s" % ",".join(sorted(tags))) + for ref in sorted(tags): + # sorting will prefer e.g. "2.0" over "2.0rc1" + if ref.startswith(tag_prefix): + r = ref[len(tag_prefix) :] + if verbose: + print("picking %s" % r) + return { + "version": r, + "full-revisionid": keywords["full"].strip(), + "dirty": False, + "error": None, + "date": date, + } + # no suitable tags, so version is "0+unknown", but full hex is still there + if verbose: + print("no suitable tags, using unknown + full revision id") + return { + "version": "0+unknown", + "full-revisionid": keywords["full"].strip(), + "dirty": False, + "error": "no suitable tags", + "date": None, + } + + +@register_vcs_handler("git", "pieces_from_vcs") +def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): + """Get version from 'git describe' in the root of the source tree. + + This only gets called if the git-archive 'subst' keywords were *not* + expanded, and _version.py hasn't already been rewritten with a short + version string, meaning we're inside a checked out source tree. + """ + GITS = ["git"] + if sys.platform == "win32": + GITS = ["git.cmd", "git.exe"] + + out, rc = run_command(GITS, ["rev-parse", "--git-dir"], cwd=root, hide_stderr=True) + if rc != 0: + if verbose: + print("Directory %s not under git control" % root) + raise NotThisMethod("'git rev-parse --git-dir' returned error") + + # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] + # if there isn't one, this yields HEX[-dirty] (no NUM) + describe_out, rc = run_command( + GITS, + [ + "describe", + "--tags", + "--dirty", + "--always", + "--long", + "--match", + "%s*" % tag_prefix, + ], + cwd=root, + ) + # --long was added in git-1.5.5 + if describe_out is None: + raise NotThisMethod("'git describe' failed") + describe_out = describe_out.strip() + full_out, rc = run_command(GITS, ["rev-parse", "HEAD"], cwd=root) + if full_out is None: + raise NotThisMethod("'git rev-parse' failed") + full_out = full_out.strip() + + pieces = {} + pieces["long"] = full_out + pieces["short"] = full_out[:7] # maybe improved later + pieces["error"] = None + + # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] + # TAG might have hyphens. + git_describe = describe_out + + # look for -dirty suffix + dirty = git_describe.endswith("-dirty") + pieces["dirty"] = dirty + if dirty: + git_describe = git_describe[: git_describe.rindex("-dirty")] + + # now we have TAG-NUM-gHEX or HEX + + if "-" in git_describe: + # TAG-NUM-gHEX + mo = re.search(r"^(.+)-(\d+)-g([0-9a-f]+)$", git_describe) + if not mo: + # unparseable. Maybe git-describe is misbehaving? + pieces["error"] = "unable to parse git-describe output: '%s'" % describe_out + return pieces + + # tag + full_tag = mo.group(1) + if not full_tag.startswith(tag_prefix): + if verbose: + fmt = "tag '%s' doesn't start with prefix '%s'" + print(fmt % (full_tag, tag_prefix)) + pieces["error"] = "tag '%s' doesn't start with prefix '%s'" % ( + full_tag, + tag_prefix, + ) + return pieces + pieces["closest-tag"] = full_tag[len(tag_prefix) :] + + # distance: number of commits since tag + pieces["distance"] = int(mo.group(2)) + + # commit: short hex revision ID + pieces["short"] = mo.group(3) + + else: + # HEX: no tags + pieces["closest-tag"] = None + count_out, rc = run_command(GITS, ["rev-list", "HEAD", "--count"], cwd=root) + pieces["distance"] = int(count_out) # total number of commits + + # commit date: see ISO-8601 comment in git_versions_from_keywords() + date = run_command(GITS, ["show", "-s", "--format=%ci", "HEAD"], cwd=root)[ + 0 + ].strip() + pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1) + + return pieces + + +def plus_or_dot(pieces): + """Return a + if we don't already have one, else return a .""" + if "+" in pieces.get("closest-tag", ""): + return "." + return "+" + + +def render_pep440(pieces): + """Build up version string, with post-release "local version identifier". + + Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you + get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty + + Exceptions: + 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += plus_or_dot(pieces) + rendered += "%d.g%s" % (pieces["distance"], pieces["short"]) + if pieces["dirty"]: + rendered += ".dirty" + else: + # exception #1 + rendered = "0+untagged.%d.g%s" % (pieces["distance"], pieces["short"]) + if pieces["dirty"]: + rendered += ".dirty" + return rendered + + +def render_pep440_pre(pieces): + """TAG[.post.devDISTANCE] -- No -dirty. + + Exceptions: + 1: no tags. 0.post.devDISTANCE + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"]: + rendered += ".post.dev%d" % pieces["distance"] + else: + # exception #1 + rendered = "0.post.dev%d" % pieces["distance"] + return rendered + + +def render_pep440_post(pieces): + """TAG[.postDISTANCE[.dev0]+gHEX] . + + The ".dev0" means dirty. Note that .dev0 sorts backwards + (a dirty tree will appear "older" than the corresponding clean one), + but you shouldn't be releasing software with -dirty anyways. + + Exceptions: + 1: no tags. 0.postDISTANCE[.dev0] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += ".post%d" % pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + rendered += plus_or_dot(pieces) + rendered += "g%s" % pieces["short"] + else: + # exception #1 + rendered = "0.post%d" % pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + rendered += "+g%s" % pieces["short"] + return rendered + + +def render_pep440_old(pieces): + """TAG[.postDISTANCE[.dev0]] . + + The ".dev0" means dirty. + + Eexceptions: + 1: no tags. 0.postDISTANCE[.dev0] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += ".post%d" % pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + else: + # exception #1 + rendered = "0.post%d" % pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + return rendered + + +def render_git_describe(pieces): + """TAG[-DISTANCE-gHEX][-dirty]. + + Like 'git describe --tags --dirty --always'. + + Exceptions: + 1: no tags. HEX[-dirty] (note: no 'g' prefix) + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"]: + rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) + else: + # exception #1 + rendered = pieces["short"] + if pieces["dirty"]: + rendered += "-dirty" + return rendered + + +def render_git_describe_long(pieces): + """TAG-DISTANCE-gHEX[-dirty]. + + Like 'git describe --tags --dirty --always -long'. + The distance/hash is unconditional. + + Exceptions: + 1: no tags. HEX[-dirty] (note: no 'g' prefix) + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) + else: + # exception #1 + rendered = pieces["short"] + if pieces["dirty"]: + rendered += "-dirty" + return rendered + + +def render(pieces, style): + """Render the given version pieces into the requested style.""" + if pieces["error"]: + return { + "version": "unknown", + "full-revisionid": pieces.get("long"), + "dirty": None, + "error": pieces["error"], + "date": None, + } + + if not style or style == "default": + style = "pep440" # the default + + if style == "pep440": + rendered = render_pep440(pieces) + elif style == "pep440-pre": + rendered = render_pep440_pre(pieces) + elif style == "pep440-post": + rendered = render_pep440_post(pieces) + elif style == "pep440-old": + rendered = render_pep440_old(pieces) + elif style == "git-describe": + rendered = render_git_describe(pieces) + elif style == "git-describe-long": + rendered = render_git_describe_long(pieces) + else: + raise ValueError("unknown style '%s'" % style) + + return { + "version": rendered, + "full-revisionid": pieces["long"], + "dirty": pieces["dirty"], + "error": None, + "date": pieces.get("date"), + } + + +def get_versions(): + """Get version information or return default if unable to do so.""" + # I am in _version.py, which lives at ROOT/VERSIONFILE_SOURCE. If we have + # __file__, we can work backwards from there to the root. Some + # py2exe/bbfreeze/non-CPython implementations don't do __file__, in which + # case we can only use expanded keywords. + + cfg = get_config() + verbose = cfg.verbose + + try: + return git_versions_from_keywords(get_keywords(), cfg.tag_prefix, verbose) + except NotThisMethod: + pass + + try: + root = os.path.realpath(__file__) + # versionfile_source is the relative path from the top of the source + # tree (where the .git directory might live) to this file. Invert + # this to find the root from __file__. + for i in cfg.versionfile_source.split("/"): + root = os.path.dirname(root) + except NameError: + return { + "version": "0+unknown", + "full-revisionid": None, + "dirty": None, + "error": "unable to find root of source tree", + "date": None, + } + + try: + pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose) + return render(pieces, cfg.style) + except NotThisMethod: + pass + + try: + if cfg.parentdir_prefix: + return versions_from_parentdir(cfg.parentdir_prefix, root, verbose) + except NotThisMethod: + pass + + return { + "version": "0+unknown", + "full-revisionid": None, + "dirty": None, + "error": "unable to compute version", + "date": None, + } diff --git a/hypernets_processor/post_processing/RadCalNet_conversion/data_io/__init__.py b/hypernets_processor/post_processing/RadCalNet_conversion/data_io/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/hypernets_processor/post_processing/RadCalNet_conversion/data_io/read_atmospheric_data.py b/hypernets_processor/post_processing/RadCalNet_conversion/data_io/read_atmospheric_data.py new file mode 100644 index 00000000..04e15d08 --- /dev/null +++ b/hypernets_processor/post_processing/RadCalNet_conversion/data_io/read_atmospheric_data.py @@ -0,0 +1,144 @@ +import numpy as np +import xarray as xr +import os +import datetime +import pandas as pd + +# File to read era5 data and aernonet manually downloaded from their appropriate website. +# It is noted that the current model assumes getting H2O, O3 and Pressure data from era5 +# and aerosol data from aeronet, though it is possible to adjust file if other parameters are wanted from these files. +# This file currently requireds the name of the files to be inputted, but is also noted, that this code can be adapted to run from a certain directory (e.g. data file on eo server rather than a particualr file). + +dirname = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + + +def read_era5_data(era5_mean, era5_spread, S2_time, lon, lat): + """ + Function to read in the atmopsheric data form the era5 data. + + Inputs: + era5_mean: str, path to the file containing the means of the era5 data. Currently required to be netcdf file + era5_spread: str, path to the file containing the spread of the era5 data. Currently required to be a netcdf file + S2_time: datetime, the time of the S2 data + lon, lat: float, the coordinates of the hypernets site + + Outputs: + H2O, P, O3, u_H2O, u_P, u_O3: float + """ + # opening the data + mean_ds = xr.open_dataset(era5_mean) + spread_ds = xr.open_dataset(era5_spread) + atm_lon = mean_ds.coords["longitude"].values + atm_lat = mean_ds.coords["latitude"].values + atm_time = mean_ds.coords["time"].values + # Finding the closests coordinate to the point where radiance and reflectance data is measured. + lon_min = (atm_lon - lon) ** 2 + lat_min = (atm_lat - lat) ** 2 + + time_min = [] + for time in range(len(atm_time)): + d_time = np.abs(atm_time[time] - np.datetime64(S2_time)) + time_min.append(d_time) + + lon_min_id = np.where(lon_min == np.min(lon_min)) + lat_min_id = np.where(lat_min == np.min(lat_min)) + time_min_id = np.where(time_min == np.min(time_min)) + # opening the coordinates at this point, tcwv = total column water vapour, tco3 = total column ozone, sp = surface pressure + H2O = mean_ds["tcwv"].values[time_min_id[0][0], lat_min_id[0][0], lon_min_id[0][0]] + u_H2O = spread_ds["tcwv"].values[ + time_min_id[0][0], lat_min_id[0][0], lon_min_id[0][0] + ] + P = ( + mean_ds["sp"].values[time_min_id[0][0], lat_min_id[0][0], lon_min_id[0][0]] + / 100 + ) + u_P = ( + spread_ds["sp"].values[time_min_id[0][0], lat_min_id[0][0], lon_min_id[0][0]] + / 100 + ) + O3 = mean_ds["tco3"].values[ + time_min_id[0][0], lat_min_id[0][0], lon_min_id[0][0] + ] / (2.1415 * 10 ** (-5)) + u_O3 = spread_ds["tco3"].values[ + time_min_id[0][0], lat_min_id[0][0], lon_min_id[0][0] + ] / (2.1415 * 10 ** (-5)) + return H2O, P, O3, u_H2O, u_P, u_O3 + + +def read_aeronet_data(aeronet_data, S2_time): + """ + Function to read in the atmospheric data from the aernoet data. + + Inputs: + aeronet_data: str, path to the aernofile containing the aeronet data. Currently required to be a .txt file + S2_time: datetime, time of the S2 passsover + + Returns: + aerosol: float + """ + # aeronet data is taken for location + # finding the closest coordinate + time_min = [] + aerosol_data = pd.read_csv(aeronet_data, on_bad_lines="skip") + for t in range(len(aerosol_data["Time(hh:mm:ss)"])): + date = datetime.datetime.strptime( + aerosol_data["Date(dd:mm:yyyy)"][t], "%d:%m:%Y" + ).date() + time = datetime.datetime.strptime( + aerosol_data["Time(hh:mm:ss)"][t], "%H:%M:%S" + ).time() + observation = datetime.datetime.combine(date, time) + d_time = np.abs(observation - S2_time) + time_min.append(d_time) + time_min_id = time_min.index(min(time_min)) + # opening the data. It is noted that for aerosol data, the uncertainity is given as 0.001-0.002 for all aerosol data + aerosol = ( + aerosol_data["AOD_500nm"][time_min_id] + * (550 / 500) ** -aerosol_data["500-870_Angstrom_Exponent"][time_min_id] + ) + print( + "aod", + aerosol, + aerosol_data["AOD_500nm"][time_min_id], + aerosol_data["500-870_Angstrom_Exponent"][time_min_id], + ) + u_aerosol = 0.002 + return aerosol, u_aerosol + + +def read_cams_data(cams_data, S2_time, lon, lat): + """ + Function to read in the atmospheric data from the CAMS data. + + Inputs: + cams_data: str, path to the file containing the CAMS dta. Currently requried to be a netcdf file + S2_time: dattime, the time of the S2 data is recorded at. + lon, lat: float, the coordinates of the hypernets site + + Outputs: + aerosol, u_aerosol: float + """ + # opening the data + aerosol_ds = xr.open_dataset(cams_data) + atm_lon = aerosol_ds.coords["longitude"].values + atm_lat = aerosol_ds.coords["latitude"].values + atm_time = aerosol_ds.coords["time"].values + + # Finding the cloest coordinate to the point where radiance and reflectance data is measured. + lon_min = (atm_lon - lon) ** 2 + lat_min = (atm_lat - lat) ** 2 + + time_min = [] + for time in range(len(atm_time)): + d_time = np.abs(atm_time[time] - np.datetime64(S2_time)) + time_min.append(d_time) + + lon_min_id = np.where(lon_min == np.min(lon_min)) + lat_min_id = np.where(lat_min == np.min(lat_min)) + time_min_id = np.where(time_min == np.min(time_min)) + + aerosol = aerosol_ds["aod550"].values[ + time_min_id[0][0], lat_min_id[0][0], lon_min_id[0][0] + ] + u_aerosol = aerosol * 0.1 + return aerosol, u_aerosol diff --git a/hypernets_processor/post_processing/RadCalNet_conversion/data_io/read_hypernets.py b/hypernets_processor/post_processing/RadCalNet_conversion/data_io/read_hypernets.py new file mode 100644 index 00000000..4bad21b1 --- /dev/null +++ b/hypernets_processor/post_processing/RadCalNet_conversion/data_io/read_hypernets.py @@ -0,0 +1,168 @@ +import os.path +import sqlite3 +import xarray as xr +import numpy as np +from obsarray.templater.dataset_util import DatasetUtil + +bad_flags = [ + "pt_ref_invalid", + "half_of_scans_masked", + "not_enough_dark_scans", + "not_enough_rad_scans", + "not_enough_irr_scans", + "no_clear_sky_irradiance", + "variable_irradiance", + "half_of_uncertainties_too_big", + "discontinuity_VNIR_SWIR", + "single_irradiance_used", +] + + +def read_db_hypernets( + hypernets_path_db, + hypernets_path, + site, + start_date, + end_date, + overwrite_product_path=True, + only_passed_qc=False, +): + archive_folder = os.path.abspath(hypernets_path_db) + dbpath = os.path.join(archive_folder, "archive.db") + engine = sqlite3.connect(dbpath) + cursor = engine.cursor() + query = make_query_hypernets( + site_id=site, + date_start=start_date, + date_end=end_date, + product_level="L_L2A", + only_passed_qc=only_passed_qc, + ) + print(query) + cursor.execute(query) + data = cursor.fetchall() + if len(data) == 0: + raise ValueError( + "no data found between the specified dates in the hypernets database" + ) + if overwrite_product_path: + for i in range(len(data)): + data[i] = list(data[i]) + data[i][-1] = os.path.join(hypernets_path, data[i][-2], data[i][-3] + ".nc") + return np.array(data, dtype=object) + + +def read_hypernets_file(filepath, vza=None, vaa=None, nearest=True, filter_flags=True): + ds = xr.open_dataset(filepath) + if filter_flags: + flagged = DatasetUtil.get_flags_mask_or(ds["quality_flag"], bad_flags) + id_series = np.where(~flagged)[0] + print( + len(id_series), + " series selected on quality flag (out of %s total)" + % len(ds.series.values), + ) + ds = ds.isel(series=id_series) + + if len(id_series) == 0: + return None + + if (vza is not None) and (vaa is not None): + if nearest: + angledif_series = (ds["viewing_zenith_angle"].values - vza) ** 2 + ( + np.abs(ds["viewing_azimuth_angle"].values - vaa) + ) ** 2 + id_series = np.where(angledif_series == np.min(angledif_series))[0] + else: + id_series = np.where( + (ds["viewing_zenith_angle"].values == vza) + & (ds["viewing_azimuth_angle"].values == vaa) + )[0] + elif vza is not None: + if nearest: + angledif_series = (ds["viewing_zenith_angle"].values - vza) ** 2 + id_series = np.where(angledif_series == np.min(angledif_series))[0] + else: + id_series = np.where(ds["viewing_zenith_angle"].values == vza)[0] + elif vaa is not None: + if nearest: + angledif_series = (np.abs(ds["viewing_azimuth_angle"].values - vaa)) ** 2 + id_series = np.where(angledif_series == np.min(angledif_series))[0] + else: + id_series = np.where(ds["viewing_azimuth_angle"].values == vaa)[0] + else: + id_series = ds.series.values + + ds = ds.isel(series=id_series) + print( + len(id_series), + " series selected on angle (vza=%s, vaa=%s requested, vza=%s, vaa=%s found)" + % ( + vza, + vaa, + ds["viewing_zenith_angle"].values, + ds["viewing_azimuth_angle"].values, + ), + ) + return ds + + +def make_query_hypernets( + sequence_id=None, + date_start=None, + date_end=None, + time_start=None, + time_end=None, + site_id=None, + system_id=None, + product_level=None, + only_passed_qc=False, +): + if sequence_id: + cond = "sequence_name = '%s'" + args = (sequence_id,) + + else: + if system_id: + cond = "system_id = '%s'" + args = (system_id,) + + elif site_id and not site_id == "all": + cond = "site_id = '%s'" + args = (site_id,) + else: + cond = "" + args = () + + if date_start: + cond += " AND date(datetime_SEQ)>= '%s'" + args += (date_start,) + if date_end: + cond += " AND date(datetime_SEQ)<= '%s'" + args += (date_end,) + + if time_start: + cond += " AND time(datetime_SEQ)>= '%s'" + args += (time_start,) + + if time_end: + cond += " AND time(datetime_SEQ)<= '%s'" + args += (date_end,) + + if product_level: + cond += " AND product_level = '%s'" + if product_level == "L2A": + raise ValueError("product_level should likely be L_L2A") + args += (product_level,) + + if cond[0:4] == " AND": + cond = cond[4:] + + if only_passed_qc: + cond += " AND percent_zero_flags>0" + + return ( + "SELECT sequence_name,site_id,system_id,datetime_SEQ,latitude,longitude,product_name,rel_product_dir,product_path FROM products WHERE " + + cond % args + + " ORDER BY datetime_SEQ" + ) diff --git a/hypernets_processor/post_processing/RadCalNet_conversion/data_io/read_radcalnet.py b/hypernets_processor/post_processing/RadCalNet_conversion/data_io/read_radcalnet.py new file mode 100644 index 00000000..bf5f7185 --- /dev/null +++ b/hypernets_processor/post_processing/RadCalNet_conversion/data_io/read_radcalnet.py @@ -0,0 +1,226 @@ +import os.path +import numpy as np +from datetime import datetime, timedelta +import pandas as pd + + +def read_db_radcalnet_BOA(data_folder, db_folder, file, start_date, end_date): + dbpath = os.path.join(db_folder, file) + db = np.genfromtxt(dbpath, dtype=str) + data = parse_radcalnet_dates(db) + valid_ids = np.where( + (data[:, 1] > start_date) + & (data[:, 1] < end_date) + & ([".input" in dat for dat in data[:, 0]]) + )[0] + data = data[valid_ids, :] + if len(data) == 0: + raise ValueError( + "no data found between the specified dates in the radcalnet database" + ) + data_expanded = np.empty((len(data[:, 0]) * 13, 20), dtype=object) + for i in range(len(data[:, 0])): + data_expanded[i * 13 : (i + 1) * 13, 0] = data[i, 0] + data_expanded[i * 13 : (i + 1) * 13, 1] = data[i, 1] + ( + wav, + times, + refl, + refl_uncs, + doy_l, + local_time, + P, + T, + H2O, + O3, + AOD, + ang, + type, + u_P, + u_T, + u_H2O, + u_O3, + u_AOD, + u_ang, + ) = read_file_radcalnet_BOA(os.path.join(data_folder, data[i, 0])) + for ii in range(len(times)): + time = datetime.strptime(times[ii], "%H:%M") + data_expanded[i * 13 + ii, 1] += timedelta( + hours=time.hour, minutes=time.minute + ) + data_expanded[i * 13 + ii, 2] = wav + data_expanded[i * 13 + ii, 3] = refl[:, ii] + data_expanded[i * 13 + ii, 4] = refl_uncs[:, ii] + data_expanded[i * 13 + ii, 5] = doy_l[ii] + data_expanded[i * 13 + ii, 6] = local_time[ii] + data_expanded[i * 13 + ii, 7] = P[ii] + data_expanded[i * 13 + ii, 8] = T[ii] + data_expanded[i * 13 + ii, 9] = H2O[ii] + data_expanded[i * 13 + ii, 10] = O3[ii] + data_expanded[i * 13 + ii, 11] = AOD[ii] + data_expanded[i * 13 + ii, 12] = ang[ii] + data_expanded[i * 13 + ii, 13] = type[ii] + data_expanded[i * 13 + ii, 14] = u_P[ii] + data_expanded[i * 13 + ii, 15] = u_T[ii] + data_expanded[i * 13 + ii, 16] = u_H2O[ii] + data_expanded[i * 13 + ii, 17] = u_O3[ii] + data_expanded[i * 13 + ii, 18] = u_AOD[ii] + data_expanded[i * 13 + ii, 19] = u_ang[ii] + return data_expanded + + +def read_db_radcalnet_TOA(data_folder, file, start_date, end_date): + dbpath = os.path.join(data_folder, file) + db = np.genfromtxt(dbpath, dtype=str) + data = parse_radcalnet_dates(db) + valid_ids = np.where( + (data[:, 1] > start_date) + & (data[:, 1] < end_date) + & ([".output" in dat for dat in data[:, 0]]) + )[0] + data = data[valid_ids, :] + if len(data) == 0: + raise ValueError( + "no data found between the specified dates in the radcalnet database" + ) + data_expanded = np.empty((len(data[:, 0]) * 13, 13), dtype=object) + for i in range(len(data[:, 0])): + data_expanded[i * 13 : (i + 1) * 13, 0] = data[i, 0] + data_expanded[i * 13 : (i + 1) * 13, 1] = data[i, 1] + ( + wav, + times, + refl, + refl_uncs, + P, + T, + H2O, + O3, + AOD, + u_P, + u_T, + u_H2O, + u_O3, + u_AOD, + ) = read_file_radcalnet_TOA(os.path.join(data_folder, data[i, 0])) + for ii in range(len(times)): + time = datetime.strptime(times[ii], "%H:%M") + data_expanded[i * 13 + ii, 1] += timedelta( + hours=time.hour, minutes=time.minute + ) + data_expanded[i * 13 + ii, 2] = wav + data_expanded[i * 13 + ii, 3] = refl[:, ii] + data_expanded[i * 13 + ii, 4] = refl_uncs[:, ii] + data_expanded[i * 13 + ii, 5] = P[ii] + data_expanded[i * 13 + ii, 6] = H2O[ii] + data_expanded[i * 13 + ii, 7] = O3[ii] + data_expanded[i * 13 + ii, 8] = AOD[ii] + data_expanded[i * 13 + ii, 9] = u_P[ii] + data_expanded[i * 13 + ii, 10] = u_H2O[ii] + data_expanded[i * 13 + ii, 11] = u_O3[ii] + data_expanded[i * 13 + ii, 12] = u_AOD[ii] + return data_expanded + + +def read_file_radcalnet_BOA(filepath): + data2 = np.genfromtxt(filepath, dtype="str", delimiter="\t", skip_header=4) + wav = data2[12:223, 0].astype(float) + times = data2[2, 1:] + refl = data2[12:223, 1:].astype(float) + refl_uncs = data2[229:440, 1:].astype(float) + doy_l = data2[3, 1:].astype(float) + local_time = data2[4, 1:] + P = data2[5, 1:].astype(float) + T = data2[6, 1:].astype(float) + H2O = data2[7, 1:].astype(float) + O3 = data2[8, 1:].astype(float) + AOD = data2[9, 1:].astype(float) + ang = data2[10, 1:].astype(float) + type = data2[11, 1:] + u_P = data2[223, 1:].astype(float) + u_T = data2[224, 1:].astype(float) + u_H2O = data2[225, 1:].astype(float) + u_O3 = data2[226, 1:].astype(float) + u_AOD = data2[227, 1:].astype(float) + u_ang = data2[228, 1:].astype(float) + refl[refl == 9999] = np.nan + refl_uncs[refl_uncs == 9999] = np.nan + u_P[P == 9999] = np.nan + u_T[T == 9999] = np.nan + u_H2O[H2O == 9999] = np.nan + u_O3[O3 == 9999] = np.nan + u_AOD[AOD == 9999] = np.nan + u_ang[ang == 9999] = np.nan + P[P == 9999] = np.nan + T[T == 9999] = np.nan + H2O[H2O == 9999] = np.nan + O3[O3 == 9999] = np.nan + AOD[AOD == 9999] = np.nan + ang[ang == 9999] = np.nan + + return ( + wav, + times, + refl, + refl_uncs, + doy_l, + local_time, + P, + T, + H2O, + O3, + AOD, + ang, + type, + u_P, + u_T, + u_H2O, + u_O3, + u_AOD, + u_ang, + ) + + +def read_file_radcalnet_TOA(filepath): + data2 = np.genfromtxt(filepath, dtype="str", delimiter="\t", skip_header=4) + wav = data2[15:226, 0].astype(float) + times = data2[2, 1:] + refl = data2[15:226, 1:].astype(float) + refl_uncs = data2[232:443, 1:].astype(float) + P = data2[5, 1:].astype(float) + T = data2[6, 1:].astype(float) + H2O = data2[7, 1:].astype(float) + O3 = data2[8, 1:].astype(float) + AOD = data2[9, 1:].astype(float) + u_P = data2[226, 1:].astype(float) + u_T = data2[227, 1:].astype(float) + u_H2O = data2[228, 1:].astype(float) + u_O3 = data2[229, 1:].astype(float) + u_AOD = data2[230, 1:].astype(float) + refl[refl == 9999] = np.nan + refl_uncs[refl_uncs == 9999] = np.nan + P[P == 9999] = np.nan + T[T == 9999] = np.nan + H2O[H2O == 9999] = np.nan + O3[O3 == 9999] = np.nan + AOD[AOD == 9999] = np.nan + u_P[P == 9999] = np.nan + u_T[T == 9999] = np.nan + u_H2O[H2O == 9999] = np.nan + u_O3[O3 == 9999] = np.nan + u_AOD[AOD == 9999] = np.nan + return wav, times, refl, refl_uncs, P, T, H2O, O3, AOD, u_P, u_T, u_H2O, u_O3, u_AOD + + +def parse_radcalnet_dates(db): + dates = np.empty((len(db), 2), dtype=object) + dates[:, 0] = db + for i, file in enumerate(db): + dates[i, 1] = parse_date(file) + return dates + + +def parse_date(filename): + date_string = filename[7:15] + date = datetime.strptime(date_string, "%Y_%j") + return date diff --git a/hypernets_processor/post_processing/RadCalNet_conversion/main.py b/hypernets_processor/post_processing/RadCalNet_conversion/main.py new file mode 100644 index 00000000..7d26b5af --- /dev/null +++ b/hypernets_processor/post_processing/RadCalNet_conversion/main.py @@ -0,0 +1,664 @@ +"""Run MCMC for atmospheric retrieval""" + +import punpy + +"""___Built-In Modules___""" +from GHNA_processing.data_io.read_hypernets import ( + read_db_hypernets, + read_hypernets_file, +) +from GHNA_processing.data_io.read_radcalnet import read_db_radcalnet_BOA +from GHNA_processing.RadCalNet_conversion.convert_radcalnet import Convert_RadCalNet + + +"""___Third-Party Modules___""" +import datetime as dt +from datetime import datetime, timedelta +import numpy as np +import os +import matplotlib.pyplot as plt +import comet_maths as cm + +"""___NPL Modules___""" + + +__author__ = "Pieter De Vis " + +# outputs for GHNA processing +# output_path = r"\\mnt\\t\\ECO\\EOServer\\data\\insitu\\hn_processor" +# paths for windows use +# output_path = os.path.join(r"T:\\", "ECO", "EOServer", "data", "insitu", "hn_processor") +# hypernets_path = r"\\eoserver\home\data\insitu\hypernets\archive" +# db_path = r"\\eoserver\home\data\calvalresults\v2" +# radcalnet_path = os.path.join(r"T:\\", "ECO", "EOServer", "data", "insitu", "radcalnet", "RadCalNetAll-sites-May2024", "GONA") +# asd_path = os.path.join(r"T:\\", "ECO", "EOServer", "data", "insitu", "asd_gobabeb_2020", "wref_corrected.txt") +# BRDF_file = os.path.join(r"T:\\", "ECO", "EOServer", "data", "insitu", "brdf","all_wavelengths" , "BRDFparams_all_20231024T0820_20240516T1143.pkl") +# irr_file = os.path.join(r"T:\\", "ECO", "EOServer", "data", "insitu", "brdf", "irr_all.nc") + +# paths for eoserver use +output_path = r"/mnt/t/data/insitu/hn_processor/v2_vza20" +hypernets_path = r"/home/data/insitu/hypernets/archive" +db_path = "/home/data/calvalresults/v2" +radcalnet_path = r"/mnt/t/data/insitu/radcalnet/RadCalNetAll-sites-May2024/GONA" +# #os.path.join("mnt", "t", "ECO", "EOServer", "data", "insitu", "radcalnet", "Radcalnet_alldata_aug23", "GONA") +asd_path = r"/mnt/t/data/insitu/asd_gobabeb_2020/wref_corrected.txt" +# os.path.join("mnt", "t", "ECO", "EOServer", "data", "insitu", "asd_gobabeb_2020", "wref_corrected.txt") +BRDF_file = r"/mnt/t/data/insitu/brdf/all_wavelengths/BRDFparams_all_20231024T0820_20240516T1143.pkl" +irr_file = r"/mnt/t/data/insitu/brdf/irr_all.nc" + + +conv = Convert_RadCalNet(output_path, BRDF_file, irr_file) +vza_hyp = 20 + + +def identify_matchups(site, radcalnet_db, start_date, end_date, only_passed_qc=False): + """ + Return catalogue product url(s) that satisfy query + + :param query: catalogue query + :return: product urls satisfying query + """ + print("identifying matchups") + start_datetime, end_datetime = parse_startend(start_date, end_date) + hypcat = read_db_hypernets( + db_path, + hypernets_path, + site, + start_date, + end_date, + only_passed_qc=only_passed_qc, + ) + hypcat_datetimes = np.array( + [dt.datetime.strptime(hypdat[3], "%Y-%m-%d %H:%M:%S.%f") for hypdat in hypcat] + ) + + radcat = read_db_radcalnet_BOA( + radcalnet_path, db_path, radcalnet_db, start_datetime, end_datetime + ) + radcat_datetimes = radcat[:, 1] + + matchups = np.empty((len(radcat_datetimes), 5), dtype=object) + for i in range(len(radcat_datetimes)): + matchups[i, 0] = radcat[i] + id_best = np.argmin(np.abs(radcat_datetimes[i] - hypcat_datetimes)) + if radcat_datetimes[i] > hypcat_datetimes[id_best]: + id_best_before = id_best + id_best_after = min(id_best + 1, len(hypcat) - 1) + else: + id_best_before = id_best - 1 + id_best_after = id_best + + matchups[i, 1] = hypcat[id_best_before, :] + matchups[i, 2] = hypcat[id_best_after, :] + diff = radcat_datetimes[i] - hypcat_datetimes[id_best_before] + matchups[i, 3] = diff.days * 24 * 60 + diff.seconds / 60 + diff = radcat_datetimes[i] - hypcat_datetimes[id_best_after] + matchups[i, 4] = diff.days * 24 * 60 + diff.seconds / 60 + return matchups + + +def filter_matchups(matchups, timediff=None): + if timediff is not None: + condition = (abs(matchups[:, 3]) < timediff) & (abs(matchups[:, 4]) < timediff) + matchups = matchups[np.where(condition)[0], :] + print(len(matchups), "matchups found within %s min" % timediff) + return matchups + + +def parse_startend(start_date, end_date): + if not isinstance(start_date, datetime): + try: + start_date = datetime.strptime(start_date, "%Y-%m-%d") + except: + raise ValueError("start_date does not have the right format") + + if not isinstance(end_date, datetime): + try: + end_date = datetime.strptime(end_date, "%Y-%m-%d") + except: + raise ValueError("end_date does not have the right format") + return start_date, end_date + + +def process_matchup(matchup): + prop = punpy.MCPropagation(10, parallel_cores=1) + + rcn_name = matchup[0][0] + rcn_datetime = matchup[0][1].timestamp() + rcn_wav = matchup[0][2] + rcn_refl = matchup[0][3] + u_rcn_refl = matchup[0][4] + atm_prop = matchup[0][5] + u_atm_prop = matchup[0][6] + + hypernets_ds_pre = read_hypernets_file(matchup[1][-1], vza=vza_hyp) + print(hypernets_ds_pre.reflectance.values[100, :]) + hypernets_ds_pre = conv.correct_to_nadir(hypernets_ds_pre) + print(hypernets_ds_pre.reflectance.values[100, :]) + hypernets_datetime_pre = np.mean(hypernets_ds_pre.acquisition_time.values) + + hyp_wav_pre = hypernets_ds_pre.wavelength.values + hyp_refl_pre = np.mean(hypernets_ds_pre.reflectance.values, axis=1) + hyp_std_pre = np.std(hypernets_ds_pre.reflectance.values, axis=1) + u_rand_pre = ( + ( + np.mean(hypernets_ds_pre.u_rel_random_reflectance.values, axis=1) + * 0.01 + * hyp_refl_pre + ) + ** 2 + + hyp_std_pre**2 + ) ** 0.5 / np.sqrt(3) + u_syst_pre = ( + np.mean(hypernets_ds_pre.u_rel_systematic_reflectance.values, axis=1) + * 0.01 + * hyp_refl_pre + ) + u_hyp_refl_pre = np.sqrt(u_rand_pre**2 + u_syst_pre**2) + cov_hyp_refl_pre = cm.convert_corr_to_cov( + np.eye(len(u_rand_pre)), u_rand_pre + ) + cm.convert_corr_to_cov( + hypernets_ds_pre.err_corr_systematic_reflectance.values, u_syst_pre + ) + corr_hyp_refl_pre = cm.convert_cov_to_corr(cov_hyp_refl_pre, u_hyp_refl_pre) + + print( + corr_hyp_refl_pre, + cm.uncertainty_from_covariance(cov_hyp_refl_pre) / u_hyp_refl_pre, + ) + + hypernets_ds_aft = read_hypernets_file(matchup[2][-1], vza=vza_hyp) + hypernets_ds_aft = conv.correct_to_nadir(hypernets_ds_aft) + hypernets_datetime_aft = np.mean(hypernets_ds_aft.acquisition_time.values) + hyp_wav_aft = hypernets_ds_aft.wavelength.values + hyp_refl_aft = np.mean(hypernets_ds_aft.reflectance.values, axis=1) + hyp_std_aft = np.std(hypernets_ds_aft.reflectance.values, axis=1) + u_rand_aft = ( + ( + np.mean(hypernets_ds_aft.u_rel_random_reflectance.values, axis=1) + * 0.01 + * hyp_refl_aft + ) + ** 2 + + hyp_std_aft**2 + ) ** 0.5 / np.sqrt(3) + u_syst_aft = ( + np.mean(hypernets_ds_aft.u_rel_systematic_reflectance.values, axis=1) + * 0.01 + * hyp_refl_aft + ) + u_hyp_refl_aft = np.sqrt(u_rand_aft**2 + u_syst_aft**2) + cov_hyp_refl_aft = cm.convert_corr_to_cov( + np.eye(len(u_rand_aft)), u_rand_aft + ) + cm.convert_corr_to_cov( + hypernets_ds_aft.err_corr_systematic_reflectance.values, u_syst_aft + ) + corr_hyp_refl_aft = cm.convert_cov_to_corr(cov_hyp_refl_aft, u_hyp_refl_aft) + # plt.plot(rcn_wav,rcn_refl,"b") + # plt.plot(hyp_wav_aft,hyp_refl_aft,"r") + # plt.plot(hyp_wav_aft,hyp_refl_aft,"g") + # plt.show() + + asd_corrected = np.loadtxt(asd_path, dtype="str", delimiter="\t") + asd_wvslice = asd_corrected[1:2151, 0] + asd_wavs = asd_wvslice.astype(float) + asd_reflslice = asd_corrected[1:2151, 1:88] + asd_refl_full = asd_reflslice.astype(float) + asd_refl = np.average(asd_refl_full, axis=1) + asd_stdv = np.nanstd(asd_refl_full, axis=1) + u_asd_refl = asd_stdv / np.sqrt(asd_refl_full.shape[1]) + # visual check of asd data + # plt.plot(asd_wavs,asd_refl_full) + # #plt.fill_between(asd_wavs, asd_refl_ - asd_stdv, asd_refl + asd_stdv, + # #alpha=0.3, label='ASD Std Dev') + # plt.title('All ASD Data GHNA March 2020 (WR Corrected)', fontsize=20) + # plt.xlabel('Wavelengths (nm)', fontsize=20) + # plt.ylabel('Reflectances', fontsize=20) + # plt.grid() + # plt.xlim(350,2500) + # plt.legend() + # plt.show() + # plt.plot(asd_wavs, asd_stdv) + # plt.show() + + rcn_refl_pre = conv.spectral_processing( + rcn_wav, hyp_wav_pre, hyp_refl_pre, asd_wavs, asd_refl + ) + u_rcn_refl_pre, corr_rcn_refl_pre = prop.propagate_standard( + conv.spectral_processing, + [rcn_wav, hyp_wav_aft, hyp_refl_pre, asd_wavs, asd_refl], + [None, None, u_hyp_refl_pre, None, u_asd_refl], + corr_x=[None, None, corr_hyp_refl_pre, None, "rand"], + return_corr=True, + ) + rcn_refl_aft = conv.spectral_processing( + rcn_wav, hyp_wav_aft, hyp_refl_aft, asd_wavs, asd_refl + ) + u_rcn_refl_aft, corr_rcn_refl_aft = prop.propagate_standard( + conv.spectral_processing, + [rcn_wav, hyp_wav_aft, hyp_refl_aft, asd_wavs, asd_refl], + [None, None, u_hyp_refl_aft, None, u_asd_refl], + corr_x=[None, None, corr_hyp_refl_aft, None, "rand"], + return_corr=True, + ) + + rcn_refl_interpolated = conv.time_interpolation( + hypernets_datetime_pre, + rcn_refl_pre, + hypernets_datetime_aft, + rcn_refl_aft, + rcn_datetime, + ) # provide the times and hypernets reflectances + u_rcn_refl_interpolated = prop.propagate_standard( + conv.time_interpolation, + [ + hypernets_datetime_pre, + rcn_refl_pre, + hypernets_datetime_aft, + rcn_refl_aft, + rcn_datetime, + ], + [None, u_rcn_refl_pre, None, u_rcn_refl_aft, None], + corr_x=[None, corr_rcn_refl_pre, None, corr_rcn_refl_aft, None], + return_corr=False, + ) + + plt.plot(rcn_wav, rcn_refl, label="GONA RadCalNet") + plt.plot(asd_wavs, asd_refl, label="ASD") + # plt.plot(hyp_wav_pre, hyp_refl_pre, label = 'HYPERNETS Pre') + # plt.plot(hyp_wav_aft, hyp_refl_aft, label='HYPERNETS After') + plt.plot(rcn_wav, rcn_refl_interpolated, label="GHNA Interpolated") + plt.xlabel("Wavelengths (nm)", fontsize=18) + plt.ylabel("Reflectances", fontsize=18) + plt.title("Dataset Comparison", fontsize=20) + plt.grid() + plt.legend() + plt.savefig( + os.path.join( + output_path, + "plots", + "comparison_%s.png" + % (datetime.fromtimestamp(rcn_datetime).strftime("%Y%m%dT%H%M%S")), + ) + ) + plt.clf() + + plt.plot( + rcn_wav, + (rcn_refl_interpolated - rcn_refl) / rcn_refl * 100, + label="(GHNA-GONA)/GONA", + ) + plt.xlabel("Wavelengths (nm)", fontsize=18) + plt.ylabel("relative difference (%)", fontsize=18) + plt.grid() + plt.legend() + plt.savefig( + os.path.join( + output_path, + "plots", + "comparison_%s_reldiff.png" + % (datetime.fromtimestamp(rcn_datetime).strftime("%Y%m%dT%H%M%S")), + ) + ) + plt.clf() + + return rcn_refl_interpolated, u_rcn_refl_interpolated, rcn_wav + + +def main(start_date, end_date): + matchups = identify_matchups( + "GHNA", "GONA_alldata_list_230124.txt", start_date, end_date + ) + + matchups = filter_matchups(matchups, 30) + + with open(os.path.join(output_path, "matchups.txt"), "w") as f: + f.write( + "# radcalnet_name hypernets_name_before hypernets_name_after timediff_before, timediff_after \n" + ) + for i in range(len(matchups)): + f.write( + "%s %s %s %s %s \n" + % ( + matchups[i, 0][0], + matchups[i, 1][0], + matchups[i, 2][0], + matchups[i, 3], + matchups[i, 4], + ) + ) + + output_files = [] + rcn_times = [] + rcn_files = [] + rcn_doy_l = [] + rcn_local_time = [] + rcn_p = [] + rcn_t = [] + rcn_h2o = [] + rcn_o3 = [] + rcn_aod = [] + rcn_ang = [] + rcn_type = [] + + rcn_u_p = [] + rcn_u_t = [] + rcn_u_h2o = [] + rcn_u_o3 = [] + rcn_u_aod = [] + rcn_u_ang = [] + + for i in range(len(matchups)): + rcn_files.append(matchups[i, 0][0]) + rcn_times.append(matchups[i, 0][1]) + rcn_doy_l.append(matchups[i, 0][5]) + rcn_p.append(matchups[i, 0][7]) + rcn_t.append(matchups[i, 0][8]) + rcn_h2o.append(matchups[i, 0][9]) + rcn_o3.append(matchups[i, 0][10]) + rcn_aod.append(matchups[i, 0][11]) + rcn_ang.append(matchups[i, 0][12]) + rcn_type.append(matchups[i, 0][13]) + + rcn_u_p.append(matchups[i, 0][14]) + rcn_u_t.append(matchups[i, 0][15]) + rcn_u_h2o.append(matchups[i, 0][16]) + rcn_u_o3.append(matchups[i, 0][17]) + rcn_u_aod.append(matchups[i, 0][18]) + rcn_u_ang.append(matchups[i, 0][19]) + if matchups[i, 0][0] not in output_files: + output_files.append(matchups[i, 0][0]) + print(output_files) + + rcn_files = np.array(rcn_files) + rcn_times = np.array(rcn_times) + rcn_local_time = np.array(rcn_local_time) + rcn_doy_l = np.array(rcn_doy_l) + rcn_p = np.array(rcn_p) + rcn_t = np.array(rcn_t) + rcn_h2o = np.array(rcn_h2o) + rcn_o3 = np.array(rcn_o3) + rcn_aod = np.array(rcn_aod) + rcn_ang = np.array(rcn_ang) + rcn_type = np.array(rcn_type) + + rcn_u_p = np.array(rcn_u_p) + rcn_u_t = np.array(rcn_u_t) + rcn_u_h2o = np.array(rcn_u_h2o) + rcn_u_o3 = np.array(rcn_u_o3) + rcn_u_aod = np.array(rcn_u_aod) + rcn_u_ang = np.array(rcn_u_ang) + + times_UTC = [ + "08:00", + "08:30", + "09:00", + "09:30", + "10:00", + "10:30", + "11:00", + "11:30", + "12:00", + "12:30", + "13:00", + "13:30", + "14:00", + ] + + for ii in range(0, len(output_files)): + print(np.where(rcn_files == output_files[ii])[0]) + output_refl_day = np.empty( + len(rcn_files[np.where(rcn_files == output_files[ii])[0]]), dtype=object + ) + output_u_refl_day = np.empty( + len(rcn_files[np.where(rcn_files == output_files[ii])[0]]), dtype=object + ) + output_doy_l = rcn_doy_l[np.where(rcn_files == output_files[ii])[0]] + output_times_day = rcn_times[np.where(rcn_files == output_files[ii])[0]] + output_p_day = rcn_p[np.where(rcn_files == output_files[ii])[0]] + output_t_day = rcn_t[np.where(rcn_files == output_files[ii])[0]] + output_h2o_day = rcn_h2o[np.where(rcn_files == output_files[ii])[0]] + output_o3_day = rcn_o3[np.where(rcn_files == output_files[ii])[0]] + output_aod_day = rcn_aod[np.where(rcn_files == output_files[ii])[0]] + output_ang_day = rcn_ang[np.where(rcn_files == output_files[ii])[0]] + output_type_day = rcn_type[np.where(rcn_files == output_files[ii])[0]] + + output_u_p_day = rcn_u_p[np.where(rcn_files == output_files[ii])[0]] + output_u_t_day = rcn_u_t[np.where(rcn_files == output_files[ii])[0]] + output_u_h2o_day = rcn_u_h2o[np.where(rcn_files == output_files[ii])[0]] + output_u_o3_day = rcn_u_o3[np.where(rcn_files == output_files[ii])[0]] + output_u_aod_day = rcn_u_aod[np.where(rcn_files == output_files[ii])[0]] + output_u_ang_day = rcn_u_ang[np.where(rcn_files == output_files[ii])[0]] + + print( + output_refl_day.shape, + output_times_day, + output_t_day, + output_h2o_day, + output_o3_day, + output_aod_day, + output_ang_day, + output_type_day, + ) + + for iii in range(len(output_refl_day)): + for i in range(len(matchups)): + if ( + matchups[i, 0][0] == output_files[ii] + and matchups[i, 0][1] == output_times_day[iii] + ): + ( + output_refl_day[iii], + output_u_refl_day[iii], + rcn_wav, + ) = process_matchup(matchups[i]) + print(output_refl_day) + + with open( + os.path.join(output_path, output_files[ii].replace("GONA01", "GHNA01")), "w" + ) as f: + f.write("Site:\tGHNA01\n") + f.write("Lat:\t-23.60153\n") + f.write("Lon:\t15.12589\n") + f.write("Alt:\t510.0\n\n") + + f.write("Year:") + for iv in range(len(times_UTC)): + f.write("\t%s" % output_times_day[0].strftime("%Y")) + f.write("\n") + + f.write("DOY(U):") + for iv in range(len(times_UTC)): + f.write("\t%s" % output_times_day[0].strftime("%j")) + f.write("\n") + + f.write("UTC:") + for iv in range(len(times_UTC)): + f.write("\t%s" % times_UTC[iv]) + f.write("\n") + + f.write("DOY(L):") + for iv in range(len(times_UTC)): + f.write("\t%s" % output_times_day[0].strftime("%j")) + f.write("\n") + + f.write("Local:") + for iv in range(len(times_UTC)): + time_local = datetime.strptime(times_UTC[iv], "%H:%M") + time_local_corrected = time_local + timedelta(hours=1) + f.write("\t%s" % time_local_corrected.strftime("%H:%M")) + f.write("\n") + + f.write("P:") + for iv in range(len(times_UTC)): + time_present = False + for iii in range(len(output_refl_day)): + if times_UTC[iv] == output_times_day[iii].strftime("%H:%M"): + time_present = True + f.write("\t%.2f" % output_p_day[iii]) + if not time_present: + f.write("\t9999") + f.write("\n") + + f.write("T:") + for iv in range(len(times_UTC)): + time_present = False + for iii in range(len(output_refl_day)): + if times_UTC[iv] == output_times_day[iii].strftime("%H:%M"): + time_present = True + f.write("\t%.1f" % output_t_day[iii]) + if not time_present: + f.write("\t9999") + f.write("\n") + + f.write("WV:") + for iv in range(len(times_UTC)): + time_present = False + for iii in range(len(output_refl_day)): + if times_UTC[iv] == output_times_day[iii].strftime("%H:%M"): + time_present = True + f.write("\t%.2f" % output_h2o_day[iii]) + if not time_present: + f.write("\t9999") + f.write("\n") + + f.write("O3:") + for iv in range(len(times_UTC)): + time_present = False + for iii in range(len(output_refl_day)): + if times_UTC[iv] == output_times_day[iii].strftime("%H:%M"): + time_present = True + f.write("\t%.2f" % output_o3_day[iii]) + if not time_present: + f.write("\t9999") + f.write("\n") + + f.write("AOD:") + for iv in range(len(times_UTC)): + time_present = False + for iii in range(len(output_refl_day)): + if times_UTC[iv] == output_times_day[iii].strftime("%H:%M"): + time_present = True + f.write("\t%.3f" % output_aod_day[iii]) + if not time_present: + f.write("\t9999") + f.write("\n") + + f.write("Ang:") + for iv in range(len(times_UTC)): + time_present = False + for iii in range(len(output_refl_day)): + if times_UTC[iv] == output_times_day[iii].strftime("%H:%M"): + time_present = True + f.write("\t%.3f" % output_ang_day[iii]) + if not time_present: + f.write("\t9999") + f.write("\n") + + f.write("Type:") + for iv in range(len(times_UTC)): + time_present = False + for iii in range(len(output_refl_day)): + if times_UTC[iv] == output_times_day[iii].strftime("%H:%M"): + time_present = True + f.write("\t%s" % output_type_day[iii]) + if not time_present: + f.write("\tR") + f.write("\n") + + for iw in range(len(rcn_wav)): + f.write("%d" % (rcn_wav[iw])) + for iv in range(len(times_UTC)): + time_present = False + for iii in range(len(output_refl_day)): + if times_UTC[iv] == output_times_day[iii].strftime("%H:%M"): + time_present = True + f.write("\t%.4f" % (output_refl_day[iii][iw])) + if not time_present: + f.write("\t9999") + f.write("\n") + + f.write("\n") + + f.write("P:") + for iv in range(len(times_UTC)): + time_present = False + for iii in range(len(output_refl_day)): + if times_UTC[iv] == output_times_day[iii].strftime("%H:%M"): + time_present = True + f.write("\t%.2f" % output_u_p_day[iii]) + if not time_present: + f.write("\t9999") + f.write("\n") + + f.write("T:") + for iv in range(len(times_UTC)): + time_present = False + for iii in range(len(output_refl_day)): + if times_UTC[iv] == output_times_day[iii].strftime("%H:%M"): + time_present = True + f.write("\t%.2f" % output_u_t_day[iii]) + if not time_present: + f.write("\t9999") + f.write("\n") + + f.write("WV:") + for iv in range(len(times_UTC)): + time_present = False + for iii in range(len(output_refl_day)): + if times_UTC[iv] == output_times_day[iii].strftime("%H:%M"): + time_present = True + f.write("\t%.2f" % output_u_h2o_day[iii]) + if not time_present: + f.write("\t9999") + f.write("\n") + + f.write("O3:") + for iv in range(len(times_UTC)): + time_present = False + for iii in range(len(output_refl_day)): + if times_UTC[iv] == output_times_day[iii].strftime("%H:%M"): + time_present = True + f.write("\t%.2f" % output_u_o3_day[iii]) + if not time_present: + f.write("\t9999") + f.write("\n") + + f.write("AOD:") + for iv in range(len(times_UTC)): + time_present = False + for iii in range(len(output_refl_day)): + if times_UTC[iv] == output_times_day[iii].strftime("%H:%M"): + time_present = True + f.write("\t%.3f" % output_u_aod_day[iii]) + if not time_present: + f.write("\t9999") + f.write("\n") + + f.write("Ang:") + for iv in range(len(times_UTC)): + time_present = False + for iii in range(len(output_refl_day)): + if times_UTC[iv] == output_times_day[iii].strftime("%H:%M"): + time_present = True + f.write("\t%.3f" % output_u_ang_day[iii]) + if not time_present: + f.write("\t9999") + f.write("\n") + + for iw in range(len(rcn_wav)): + f.write("%d" % (rcn_wav[iw])) + for iv in range(len(times_UTC)): + time_present = False + for iii in range(len(output_refl_day)): + if times_UTC[iv] == output_times_day[iii].strftime("%H:%M"): + time_present = True + f.write("\t%.4f" % (output_u_refl_day[iii][iw])) + if not time_present: + f.write("\t9999") + if not iw == (len(rcn_wav) - 1): + f.write("\n") + + +if __name__ == "__main__": + main("2022-05-17", "2023-11-08") diff --git a/hypernets_processor/post_processing/anomaly_reporting.py b/hypernets_processor/post_processing/anomaly_reporting.py index a4182b16..560d9a10 100644 --- a/hypernets_processor/post_processing/anomaly_reporting.py +++ b/hypernets_processor/post_processing/anomaly_reporting.py @@ -1,17 +1,27 @@ import sqlite3 from sqlite3 import Error -from datetime import datetime -import seaborn as sns import matplotlib.pyplot as plt import matplotlib.dates as mdates from matplotlib.gridspec import GridSpec import pandas as pd import matplotlib.pyplot as plt import matplotlib.dates as md -path2figs="/home/data/insitu/hypernets/QC/" -path2files="/home/data/insitu/hypernets/archive/" -import sqlite3 -from sqlite3 import Error + +# path2files = r"/archive/test" +# path2figs = r"/home/admin/post_processing/" +# path2files = r"T:/ECO/EOServer/data/insitu/hypernets/archive" +path2figs = r"T:/ECO/EOServer/data/insitu/hypernets/post_processing_qc" + +path2files = r"C:\Users\pdv\OneDrive - National Physical Laboratory\Documents\MobaXterm\home" + +include_sites = None +include_sites = ["GHNA","WWUK","JSIT","JAES","LOBE"] + +include_anomalies = None +include_anomalies = ["cl","scl","nu", "nos", "val", "x"] +start_date = "2022-01-01" #include all dates +stop_date = None #include all dates + def create_connection(path): connection = None @@ -23,89 +33,47 @@ def create_connection(path): return connection -def closest_idx(xlist, xval): - idx, xret = min(enumerate(xlist), key=lambda x: abs(float(x[1]) - float(xval))) - return (idx, xret) - -def get_flag_encoding(da): - """ - Returns flag encoding for flag type data array - :type da: xarray.DataArray - :param da: data array - :return: flag meanings - :rtype: list - :return: flag masks - :rtype: list - """ - - try: - flag_meanings = da.attrs["flag_meanings"].split() - flag_masks = [int(fm) for fm in da.attrs["flag_masks"].split(",")] - except KeyError: - raise KeyError(da.name + " not a flag variable") - - return flag_meanings, flag_masks - - -def get_flags(da): - """ - Returns flag encoding for flag type data array - :type da: xarray.DataArray - :param da: data array - :return: flag meanings - :rtype: list - :return: flag masks - :rtype: list - """ - - try: - flag_meanings = da.attrs["flag_meanings"].split() - flag_masks = [int(fm) for fm in da.attrs["flag_masks"].split(",")] - except KeyError: - raise KeyError(da.name + " not a flag variable") - - return [([i],flag_meanings[i],flag_masks[i]) for i in range(0,len(flag_meanings)) - if ds['quality_flag'].values[i]!=0] - -def flags(da): - qf=da["quality_flag"].values - flagsname=get_flag_encoding(da["quality_flag"])[0] - flagsval=get_flag_encoding(da["quality_flag"])[1] - flags=np.full((len(qf),len(flagsname)),False,dtype=bool) - for i in range(0,len(qf)): - n=qf[i] - while n > 0: - r=2**round(np.log2(n)) - for j in range(0,len(flagsval)): - flags[i,j]=(r==flagsval[j]) - n=n-r - flags=pd.DataFrame(flags, columns=flagsname) - return flags - - - -dbcon=create_connection("/{}/anomaly.db".format(path2files)) -SQL_Query = pd.read_sql_query( - '''select * from anomalies''', dbcon) +dbcon = create_connection("/{}/anomaly.db".format(path2files)) +SQL_Query = pd.read_sql_query("""select * from anomalies""", dbcon) df = pd.DataFrame(SQL_Query) -df.index=pd.to_datetime(df.datetime) -df['date']=df.index.floor('D') +df.index = pd.to_datetime(df.datetime) +df["date"] = df.index.floor("D") print(df.datetime) -print(df['site_id'].unique()) +print(df["site_id"].unique()) -dbcon=create_connection("/{}/archive.db".format(path2files)) -SQL_Query = pd.read_sql_query( - '''select * from products''', dbcon) +dbcon = create_connection("/{}/archive.db".format(path2files)) +SQL_Query = pd.read_sql_query("""select * from products""", dbcon) prods = pd.DataFrame(SQL_Query) -all_dates = pd.date_range("2023-01-01", "2023-12-31", freq='M').date -for site in df['site_id'].unique(): +#only select relevant sites and product levels +if include_sites is None: + prodsel = prods +else: + prodsel = prods[ + prods["site_id"].str.contains( + "|".join(include_sites) + ) + ] +#only select valid date range +prodsel.index = pd.to_datetime(prodsel.datetime_start, format='mixed') +prodsel["date"] = prodsel.index.floor("D") +prodsel["date"] = pd.to_datetime(prodsel.date) +if start_date is not None: + prodsel = prodsel[prodsel.index >= pd.to_datetime(start_date)] +if stop_date is not None: + prodsel = prodsel[prodsel.index <= pd.to_datetime(stop_date)] + +for site in prodsel["site_id"].unique(): print(site) - prodsel = prods[prods['site_id'] == site] - nseq = len(prodsel['sequence_name'].unique()) - dfsel = df[df['site_id'] == site] + prodsel = prods[prods["site_id"] == site] + nseq = len(prodsel["sequence_name"].unique()) + dfsel = df[df["site_id"] == site] + + dates=prodsel.datetime_start.unique() + dates.sort() + sel_dates = pd.date_range(dates[2], dates[-1], freq="ME").date - fig = plt.figure(figsize=(10, 15)) + fig = plt.figure(figsize=(15, 10)) gs = GridSpec(nrows=2, ncols=2) ax1 = fig.add_subplot(gs[0, 0]) ax2 = fig.add_subplot(gs[1, :]) @@ -114,35 +82,46 @@ def flags(da): # print number of anomalies for site df1 = pd.crosstab(dfsel.date, dfsel.anomaly_id) df1 = df1.rename_axis(None, axis=1).reset_index() - df1.index = pd.to_datetime(df1['date']) - df1 = df1.drop(columns='date') + df1.index = pd.to_datetime(df1["date"]) + df1 = df1.drop(columns="date") # df1.plot.bar() dfsum = df1.sum() dfsum.plot(kind="bar", ax=ax1) ax1.set_ylabel("Number of sequences") - ax1.set_title('Anomalies for {} seqs. at {}'.format(nseq, site)) - + ax1.set_title("Anomalies for {} seqs. at {}".format(nseq, site)) + + if include_anomalies is not None: + dfsel = dfsel[ + dfsel["anomaly_id"].str.contains( + "|".join(include_anomalies) + ) + ] dfsel.index = pd.to_datetime(dfsel.datetime) - dfsel['date'] = dfsel.index.floor('D') - dfsel['date'] = pd.to_datetime(dfsel.date) + dfsel["date"] = dfsel.index.floor("D") + dfsel["date"] = pd.to_datetime(dfsel.date) cross = pd.crosstab(dfsel.date, dfsel.anomaly_id) # ,margins=True, dropna=False) - cross['date'] = cross.index - cross = cross.resample('M', on='date').sum().reindex(all_dates) + cross["date"] = cross.index + cross = cross.resample("ME", on="date").sum().reindex(sel_dates) cross.plot(kind="bar", ax=ax2, stacked=False) - ax2.legend(loc='center left', bbox_to_anchor=(1, 0.5), title='Anomalies') + ax2.legend(loc="center left", bbox_to_anchor=(1, 0.5), title="Anomalies") ax2.set_ylabel("Number of sequences") - filtered_df = prodsel[prodsel["product_level"].str.contains("L0A_RAD|L0A_IRR|L_L1A_RAD|L_L1A_IRR|L_L1B|L_L1C|L_L2A")] + filtered_df = prodsel[ + prodsel["product_level"].str.contains( + "L0A_RAD|L0A_IRR|L_L1A_RAD|L_L1A_IRR|L_L1B|L_L1C|L_L2A|L_L2B" + ) + ] filtered_df.index = pd.to_datetime(filtered_df.datetime_SEQ) - filtered_df['date'] = filtered_df.index.floor('D') - cross = pd.crosstab(filtered_df.date, filtered_df.product_level) # ,margins=True, dropna=False) - cross['date'] = cross.index - cross = cross.resample('M', on='date').sum().reindex(all_dates) + filtered_df["date"] = filtered_df.index.floor("D") + cross = pd.crosstab( + filtered_df.date, filtered_df.product_level + ) # ,margins=True, dropna=False) + cross["date"] = cross.index + cross = cross.resample("M", on="date").sum().reindex(sel_dates) cross.plot(kind="bar", ax=ax3, stacked=False) - ax3.legend(loc='center left', bbox_to_anchor=(1, 0.5), title='Proc. level') + ax3.legend(loc="center left", bbox_to_anchor=(1, 0.5), title="Proc. level") ax3.set_ylabel("Number of sequences") plt.tight_layout() - plt.savefig('{}/{}_nseq{}_anomalies_archive.png'.format(path2figs, site, nseq)) - + plt.savefig("{}/{}_nseq{}_anomalies_archive.png".format(path2figs, site, nseq)) diff --git a/hypernets_processor/post_processing/archive_reporting.py b/hypernets_processor/post_processing/archive_reporting.py new file mode 100644 index 00000000..a6b4b10b --- /dev/null +++ b/hypernets_processor/post_processing/archive_reporting.py @@ -0,0 +1,91 @@ +import sqlite3 +from sqlite3 import Error +import matplotlib.pyplot as plt +import matplotlib.dates as mdates +from matplotlib.gridspec import GridSpec +import pandas as pd +import matplotlib.pyplot as plt +import matplotlib.dates as md + +# path2files = r"/archive/test" +# path2figs = r"/home/admin/post_processing/" +path2files = r"T:/ECO/EOServer/data/insitu/hypernets/archive" +path2figs = r"T:/ECO/EOServer/data/insitu/hypernets/post_processing_qc" + +# path2files = r"C:\Users\pdv\OneDrive - National Physical Laboratory\Documents\MobaXterm\home" + +include_sites = ["GHNA", "JSIT", "WWUK", "LOBE", "JAES"] +start_date = "2022-01-01" #include all dates +stop_date = None #include all dates + +# start_date = "2022-01-01" +# stop_date = "2025-12-31" + +import sqlite3 +from sqlite3 import Error + + +def create_connection(path): + connection = None + try: + connection = sqlite3.connect(path) + print("Connection to SQLite DB successful") + except Error as e: + print(f"The error '{e}' occurred") + + return connection + +dbcon = create_connection("/{}/archive.db".format(path2files)) +SQL_Query = pd.read_sql_query("""select * from products""", dbcon) +prods = pd.DataFrame(SQL_Query) +#only select relevant sites and product levels +prodsel = prods[ + prods["product_level"].str.contains( + "L_L1B|L_L2A|L_L2B" + ) +] +prodsel = prodsel[ + prodsel["site_id"].str.contains( + "|".join(include_sites) + ) +] +#only select valid date range +prodsel.index = pd.to_datetime(prodsel.datetime_start, format='mixed') +prodsel["date"] = prodsel.index.floor("D") +prodsel["date"] = pd.to_datetime(prodsel.date) +if start_date is not None: + prodsel = prodsel[prodsel.index >= pd.to_datetime(start_date)] +if stop_date is not None: + prodsel = prodsel[prodsel.index <= pd.to_datetime(stop_date)] + +fig = plt.figure(figsize=(15, 5)) +ax = fig.add_subplot(1, 1, 1) +cross = pd.crosstab(prodsel.site_id, prodsel.product_level) # ,margins=True, dropna=False) +cross.plot(kind="bar", ax=ax, stacked=False) +ax.legend(loc="center left", bbox_to_anchor=(1, 0.5), title="Products") +ax.set_ylabel("Number of sequences") + +plt.tight_layout() +plt.savefig("{}/all_products_archive.png".format(path2figs)) + +for site in prodsel["site_id"].unique(): + prodsel_s = prodsel[prodsel["site_id"] == site] + + sel_dates = pd.date_range(str(prodsel_s.date.min()), str(prodsel_s.date.max()), freq="ME").date + + fig = plt.figure(figsize=(15, 5)) + ax = fig.add_subplot(1, 1, 1) + print(sel_dates) + + cross = pd.crosstab(prodsel_s.date, prodsel_s.product_level) # ,margins=True, dropna=False) + cross["date"] = cross.index + cross = cross.resample("ME", on="date").sum().reindex(sel_dates) + cross.plot(kind="bar", ax=ax, stacked=False) + ax.legend(loc="center left", bbox_to_anchor=(1, 0.5), title="Products") + ax.set_ylabel("Number of sequences") + + plt.tight_layout() + plt.savefig("{}/{}_products_archive.png".format(path2figs, site)) + + + \ No newline at end of file diff --git a/hypernets_processor/post_processing/bounds_reformatting.py b/hypernets_processor/post_processing/bounds_reformatting.py new file mode 100644 index 00000000..253b142e --- /dev/null +++ b/hypernets_processor/post_processing/bounds_reformatting.py @@ -0,0 +1,21 @@ +import xarray as xr +import numpy as np + +filename = "GHNAv1_bounds" + +data_path = ( + f"T:/ECO/EOServer/data/insitu/hypernets/post_processing_qc/bounds/{filename}.nc" +) + +data = xr.open_dataset(data_path) +raa_vals = [ + np.mean([float(x) for x in data.raa.values[i].split("_")]) + for i in range(len(data.raa.values)) +] +raa_vals[-1] = 0 +data["raa"] = raa_vals + + +data.to_netcdf( + f"T:/ECO/EOServer/data/insitu/hypernets/post_processing_qc/bounds/{filename}_corrected.nc" +) diff --git a/hypernets_processor/post_processing/check_valid_db.py b/hypernets_processor/post_processing/check_valid_db.py index fd6046a1..eb4d1c23 100644 --- a/hypernets_processor/post_processing/check_valid_db.py +++ b/hypernets_processor/post_processing/check_valid_db.py @@ -1,11 +1,13 @@ """ post processing """ + import numpy as np import warnings import os import xarray as xr import matplotlib as mpl + mpl.use("Agg") import sqlite3 import matplotlib.pyplot as plt @@ -31,26 +33,57 @@ plot_path = r"/home/data/insitu/hypernets/archive_qc_Jan2024/qc_plots" plotter = Plotting("", plot_path, ".png") -bad_flags=["pt_ref_invalid", "half_of_scans_masked", "not_enough_dark_scans", "not_enough_rad_scans", - "not_enough_irr_scans", "no_clear_sky_irradiance", "variable_irradiance", - "half_of_uncertainties_too_big", "discontinuity_VNIR_SWIR", "single_irradiance_used"] +bad_flags = [ + "pt_ref_invalid", + "half_of_scans_masked", + "not_enough_dark_scans", + "not_enough_rad_scans", + "not_enough_irr_scans", + "no_clear_sky_irradiance", + "variable_irradiance", + "half_of_uncertainties_too_big", + "discontinuity_VNIR_SWIR", + "single_irradiance_used", +] check_flags = ["single_irradiance_used"] -colors = ["magenta", "yellow", "cyan", "red", "green", "blue", "black", "orange", "navy", "gray", "brown", - "greenyellow", "purple"] - +colors = [ + "magenta", + "yellow", + "cyan", + "red", + "green", + "blue", + "black", + "orange", + "navy", + "gray", + "brown", + "greenyellow", + "purple", +] def read_db_hypernets( - hypernets_path_db, hypernets_path, site, start_date, end_date, overwrite_product_path=True, only_passed_qc=False, + hypernets_path_db, + hypernets_path, + site, + start_date, + end_date, + overwrite_product_path=True, + only_passed_qc=False, ): archive_folder = os.path.abspath(hypernets_path_db) dbpath = os.path.join(archive_folder, "archive.db") - print("dbpath:",dbpath) + print("dbpath:", dbpath) engine = sqlite3.connect(dbpath) cursor = engine.cursor() query = make_query_hypernets( - site_id=site, date_start=start_date, date_end=end_date, product_level="L_L2A", only_passed_qc=only_passed_qc + site_id=site, + date_start=start_date, + date_end=end_date, + product_level="L_L2A", + only_passed_qc=only_passed_qc, ) print(query) cursor.execute(query) @@ -65,6 +98,7 @@ def read_db_hypernets( data[i][-1] = os.path.join(hypernets_path, data[i][-2], data[i][-3] + ".nc") return np.array(data, dtype=object) + def read_hypernets_file( filepath, vza=None, @@ -119,6 +153,7 @@ def read_hypernets_file( # print(len(id_series), " series selected on angle (vza=%s, vaa=%s requested, vza=%s, vaa=%s found)"%(vza,vaa,ds["viewing_zenith_angle"].values,ds["viewing_azimuth_angle"].values)) return ds + if __name__ == "__main__": sites = [ "GHNA", @@ -132,7 +167,7 @@ def read_hypernets_file( "PEAN", # "DEGE", "IFAR", - "JAES" + "JAES", ] hypernets_path_db = r"/home/data/calvalresults/v2" hypernets_path = r"/home/data/insitu/hypernets/archive" @@ -171,28 +206,26 @@ def read_hypernets_file( # engine3.close() for isite in range(len(sites)): - query = "select * FROM products WHERE site_id='%s'"%(sites[isite]) + query = "select * FROM products WHERE site_id='%s'" % (sites[isite]) print(query) cursor.execute(query) data = cursor.fetchall() print(len(data)) for i in range(len(data)): - if i%100==0: - print(sites[isite],i*100/len(data),"% complete") - row=data[i] - path=row[8] - product=row[1] - #print(os.path.join(hypernets_path,path,product)) - if os.path.exists(os.path.join(hypernets_path,path,product+".nc")): + if i % 100 == 0: + print(sites[isite], i * 100 / len(data), "% complete") + row = data[i] + path = row[8] + product = row[1] + # print(os.path.join(hypernets_path,path,product)) + if os.path.exists(os.path.join(hypernets_path, path, product + ".nc")): # ds = xr.open_dataset(os.path.join(hypernets_path,path,product+".nc")) # #print(min(ds.acquisition_time.values),max(ds.acquisition_time.values)) # query = "update products set datetime_start = '%s', datetime_end = '%s' WHERE product_name='%s'" % (dt.fromtimestamp(np.nanmin(ds["acquisition_time"].values)),dt.fromtimestamp(np.nanmax(ds["acquisition_time"].values)),product) # cursor.execute(query) continue else: - query = "delete FROM products WHERE product_name='%s'"%(product) + query = "delete FROM products WHERE product_name='%s'" % (product) cursor.execute(query) engine.commit() engine.close() - - diff --git a/hypernets_processor/post_processing/cloud_check.py b/hypernets_processor/post_processing/cloud_check.py new file mode 100644 index 00000000..a02b8385 --- /dev/null +++ b/hypernets_processor/post_processing/cloud_check.py @@ -0,0 +1,328 @@ +import numpy as np +import pandas as pd +import os +import matplotlib.pyplot as plt +from matplotlib import colors +from matplotlib.pyplot import tight_layout +import scipy.optimize as spo +import xarray as xr +from ast import literal_eval + +# read in data +windows_results_path = r"T:/ECO/EOServer/data/insitu/hypernets/post_processing_qc/joe" + +linux_data_path = r"/mnt/t/data/insitu/hypernets/post_processing_qc" +windows_data_path = r"T:/ECO/EOServer/data/insitu/hypernets/post_processing_qc" + +dataframe = pd.read_csv(windows_data_path + r"/LOBEv4_irradiance.csv") +wav_df = xr.open_dataset( + "T:/ECO/EOServer/data/insitu/hypernets/archive/LOBE/2025/07/07/SEQ20250707T143048/HYPERNETS_L_LOBE_L1B_IRR_20250707T1430_20250707T1623_v2.1.nc" +) +wav = wav_df.wavelength.values + +def find_nearest_to_wav(array, wv, value): + wv = np.asarray(wv) + idx = (np.abs(wv - value)).argmin() + mean = np.mean( + [array[idx - 2], array[idx - 1], array[idx], array[idx + 1], array[idx + 2]] + ) + return mean + + +def interpolate_irradiance_sza(sza, ds_irr): + ds_irr_temp = ds_irr.copy() + ds_irr_temp["solar_irradiance_BOA"].values = ( + ds_irr_temp["solar_irradiance_BOA"].values + / np.cos(ds_irr_temp["sza"].values / 180 * np.pi)[:, None] + ) + ds_irr_temp = ds_irr_temp.interp(sza=sza, wavelength=wav, method="linear") + ds_irr_temp["solar_irradiance_BOA"].values = ds_irr_temp[ + "solar_irradiance_BOA" + ].values * np.cos(ds_irr_temp["sza"].values / 180 * np.pi) + return ds_irr_temp["solar_irradiance_BOA"].values + + +def modelled_data_read_and_interp(sza, aod): + mod_data = xr.open_dataset( + r"T:\ECO\EOServer\data\insitu\hypernets\post_processing_qc\irradiance\LOBE_clear_sky_medianaod.nc".format( + aod + ) + ) + sza = float(sza) + interp_mod_data = interpolate_irradiance_sza(sza, mod_data) + return interp_mod_data + + +IDs = dataframe["ID"] +flags = dataframe["Flag"] + +##get dates and times of measurements + + +def get_dates_times(data): + dates = [] + times = [] + for i in range(len(data)): + date = int(IDs[i][3:11]) + time = int(IDs[i][12:16]) + dates.append(date) + times.append(time) + new_data = data.assign(date=dates, time=times) + return new_data + + +##remove flagged data + +bad_inds = [] +bad_flags = [] +for i in range(len(flags)): + if flags.values[i] != 0: + bad_inds.append(i) + bad_flags.append(flags.values[i]) + +df_clean = dataframe.drop(bad_inds) +df_clean.reset_index(drop=True, inplace=True) +df_clean = get_dates_times(df_clean) + +IDs = df_clean.iloc[:, 0] + +IDs.to_csv( + r"T:\ECO\EOServer\data\insitu\hypernets\post_processing_qc\joe\clean_irr_IDs.csv" +) +szas = df_clean["SZA"] + + +def read_irr(df): + data = np.zeros((df.shape[0], len(df.columns) - 6)) + + for j in range(df.shape[0]): + for i in range(len(df.columns) - 6): + data[j, i] = df["{}".format(i)][j] + + return data + + +data = read_irr(df_clean) +""" +outliers = pd.read_csv( + r"T:\ECO\EOServer\data\insitu\hypernets\post_processing_qc\joe\GHNA_2022_outliers.csv" +) +good_data = pd.read_csv( + r"T:\ECO\EOServer\data\insitu\hypernets\post_processing_qc\joe\GHNA_2022_good.csv" +) + +outliers_IDs = outliers["# id"] +good_IDs = good_data["# id"] + +is_outlier = IDs.isin(outliers_IDs) +print(len(is_outlier)) +""" + +# plotting +print(len(szas)) +print(np.argwhere(IDs == 'SEQ20250707T143048')) +ID_num = np.argwhere(IDs == 'SEQ20250707T143048')[1] +aod_1_data = modelled_data_read_and_interp(szas[ID_num], 0.1) + +plt.plot(wav, data[ID_num,:].T, label = 'Measured Data') +plt.plot(wav, aod_1_data, label = 'Model') + +plt.ylabel("Irradiance") +plt.xlabel("Wavelength (nm)") +plt.legend() +plt.show() + + +def check(data, tolerance, wavelength): + passes = np.zeros((data.shape[0], 4)) + values = np.zeros((data.shape[0], 4)) + for i in range(data.shape[0]): + aod_0 = find_nearest_to_wav( + modelled_data_read_and_interp(szas[i], 0.0), wav, wavelength + ) + aod_1 = find_nearest_to_wav( + modelled_data_read_and_interp(szas[i], 0.1), wav, wavelength + ) + aod_2 = find_nearest_to_wav( + modelled_data_read_and_interp(szas[i], 0.2), wav, wavelength + ) + aod_3 = find_nearest_to_wav( + modelled_data_read_and_interp(szas[i], 0.3), wav, wavelength + ) + meas = find_nearest_to_wav(data[i, :], wav, wavelength) + + values[i, 0] = aod_0 * tolerance - meas + values[i, 1] = aod_1 * tolerance - meas + values[i, 2] = aod_2 * tolerance - meas + values[i, 3] = aod_3 * tolerance - meas + + for j in range(4): + if values[i, j] > 0: + passes[i, j] = 1 # cloudy + else: + passes[i, j] = 0 # clear + + print(i, passes[i, :]) + return values, passes + + +""" +val, pas = check(data, 0.9, 550) + +out_pas = pas[is_outlier == True] +good_pas = pas[is_outlier == False] + +print(len(out_pas)) + +outs_removed_0 = list(out_pas[:, 0]).count(1)/len(out_pas) +good_removed_0 = list(good_pas[:, 0]).count(1)/len(good_pas) + +outs_removed_1 = list(out_pas[:, 1]).count(1)/len(out_pas) +good_removed_1 = list(good_pas[:, 1]).count(1)/len(good_pas) + +outs_removed_2 = list(out_pas[:, 2]).count(1)/len(out_pas) +good_removed_2 = list(good_pas[:, 2]).count(1)/len(good_pas) + +outs_removed_3 = list(out_pas[:, 3]).count(1)/len(out_pas) +good_removed_3 = list(good_pas[:, 3]).count(1)/len(good_pas) + +print('For AOD = 0.0, the percentage of outliers removed is {:.2f}% and the percentage of good data removed is {:.2f}%'.format(outs_removed_0*100, good_removed_0*100)) +print('For AOD = 0.1, the percentage of outliers removed is {:.2f}% and the percentage of good data removed is {:.2f}%'.format(outs_removed_1*100, good_removed_1*100)) +print('For AOD = 0.2, the percentage of outliers removed is {:.2f}% and the percentage of good data removed is {:.2f}%'.format(outs_removed_2*100, good_removed_2*100)) +print('For AOD = 0.3, the percentage of outliers removed is {:.2f}% and the percentage of good data removed is {:.2f}%'.format(outs_removed_3*100, good_removed_3*100)) +""" + + +def tol_checks(data, tol_start, tol_stop, tol_step, wavelength): + tol = np.arange(tol_start, tol_stop, tol_step) + aod_0_outs = [] + aod_0_good = [] + aod_1_outs = [] + aod_1_good = [] + aod_2_outs = [] + aod_2_good = [] + aod_3_outs = [] + aod_3_good = [] + for i in tol: + val, pas = check(data, i, wavelength) + # out_pas = pas[is_outlier == True] + # good_pas = pas[is_outlier == False] + + cut_IDs_0 = IDs[pas[:, 0] == 1] + out_cut_0 = outliers[outliers["# id"].isin(cut_IDs_0)] + good_cut_0 = good_data[good_data["# id"].isin(cut_IDs_0)] + aod_0_outs.append(len(out_cut_0) * 100 / len(outliers)) + aod_0_good.append(len(good_cut_0) * 100 / len(good_data)) + + cut_IDs_1 = IDs[pas[:, 1] == 1] + out_cut_1 = outliers[outliers["# id"].isin(cut_IDs_1)] + good_cut_1 = good_data[good_data["# id"].isin(cut_IDs_1)] + aod_1_outs.append(len(out_cut_1) * 100 / len(outliers)) + aod_1_good.append(len(good_cut_1) * 100 / len(good_data)) + + cut_IDs_2 = IDs[pas[:, 2] == 1] + out_cut_2 = outliers[outliers["# id"].isin(cut_IDs_2)] + good_cut_2 = good_data[good_data["# id"].isin(cut_IDs_2)] + aod_2_outs.append(len(out_cut_2) * 100 / len(outliers)) + aod_2_good.append(len(good_cut_2) * 100 / len(good_data)) + + cut_IDs_3 = IDs[pas[:, 3] == 1] + out_cut_3 = outliers[outliers["# id"].isin(cut_IDs_3)] + good_cut_3 = good_data[good_data["# id"].isin(cut_IDs_3)] + aod_3_outs.append(len(out_cut_3) * 100 / len(outliers)) + aod_3_good.append(len(good_cut_3) * 100 / len(good_data)) + + return np.vstack( + ( + tol, + aod_0_outs, + aod_0_good, + aod_1_outs, + aod_1_good, + aod_2_outs, + aod_2_good, + aod_3_outs, + aod_3_good, + ) + ) + + +def wav_checks(data, tol, wav_start, wav_stop, wav_step): + wav = np.arange(wav_start, wav_stop, wav_step) + aod_0_outs = [] + aod_0_good = [] + aod_1_outs = [] + aod_1_good = [] + aod_2_outs = [] + aod_2_good = [] + aod_3_outs = [] + aod_3_good = [] + for i in wav: + val, pas = check(data, tol, i) + out_pas = pas[is_outlier == True] + good_pas = pas[is_outlier == False] + + aod_0_outs.append(list(out_pas[:, 0]).count(1) * 100 / len(out_pas)) + aod_0_good.append(list(good_pas[:, 0]).count(1) * 100 / len(good_pas)) + + aod_1_outs.append(list(out_pas[:, 1]).count(1) * 100 / len(out_pas)) + aod_1_good.append(list(good_pas[:, 1]).count(1) * 100 / len(good_pas)) + + aod_2_outs.append(list(out_pas[:, 2]).count(1) * 100 / len(out_pas)) + aod_2_good.append(list(good_pas[:, 2]).count(1) * 100 / len(good_pas)) + + aod_3_outs.append(list(out_pas[:, 3]).count(1) * 100 / len(out_pas)) + aod_3_good.append(list(good_pas[:, 3]).count(1) * 100 / len(good_pas)) + + return np.vstack( + ( + wav, + aod_0_outs, + aod_0_good, + aod_1_outs, + aod_1_good, + aod_2_outs, + aod_2_good, + aod_3_outs, + aod_3_good, + ) + ) + + +# np.savetxt(r'C:/Users/jr20/code/hypernets_cloud_checks/na.csv', tol_checks(data, 0.8, 1, 0.02, 550), delimiter = ',') + + +def make_new_csvs(data_refl, passes, filepath=None): + cut_IDs = IDs[passes == 1] + + new = data_refl[~data_refl["# id"].isin(cut_IDs)] + new.reset_index(drop=True, inplace=True) + print(len(new), len(data_refl)) + if filepath is not None: + new.to_csv(filepath, index=False) + + +refl_data = pd.read_csv( + r"T:\ECO\EOServer\data\insitu\hypernets\post_processing_qc\joe\GHNA_2022_prelim_and_raa10.csv" +) +""" +val_0_875, pass_0_875 = check(data, 0.875, 550) +make_new_csvs(refl_data, pass_0_875[:,0], r'T:\ECO\EOServer\data\insitu\hypernets\post_processing_qc\joe\prelim_cc0875.csv') +""" +val_1_9, pass_1_9 = check(data, 0.9, 550) +make_new_csvs( + good_data, + pass_1_9[:, 1], + r"T:\ECO\EOServer\data\insitu\hypernets\post_processing_qc\joe\GHNA_2022_cc_good.csv", +) +make_new_csvs( + outliers, + pass_1_9[:, 1], + r"T:\ECO\EOServer\data\insitu\hypernets\post_processing_qc\joe\GHNA_2022_cc_outliers.csv", +) + + +JSIT = xr.open_dataset( + r"T:\ECO\EOServer\data\insitu\hypernets\post_processing_qc\irradiance\irr_clear_sky_JSIT_0.0_10.nc" +) +print(JSIT) diff --git a/hypernets_processor/post_processing/cloud_check_plots.py b/hypernets_processor/post_processing/cloud_check_plots.py new file mode 100644 index 00000000..244c8e8d --- /dev/null +++ b/hypernets_processor/post_processing/cloud_check_plots.py @@ -0,0 +1,60 @@ +import numpy as np +import pandas as pd +import matplotlib.pyplot as plt + +data = np.loadtxt( + r"C:/Users/jr20/code/hypernets_cloud_checks/first_cc_0875.csv", delimiter="," +) + +tols = data[0, :] + +aod_0_outs = data[1, :] +aod_0_good = data[2, :] + +aod_1_outs = data[3, :] +aod_1_good = data[4, :] + +aod_2_outs = data[5, :] +aod_2_good = data[6, :] + +aod_3_outs = data[7, :] +aod_3_good = data[8, :] + +fig, ax = plt.subplots(1, 1, figsize=(6, 4), dpi=150) + + +ax.plot(tols, aod_0_outs, label="AOD = 0.0: Outliers", color="r", linestyle="solid") +ax.plot(tols, aod_0_good, label="AOD = 0.0: Good Data", color="r", linestyle="dotted") +ax.plot( + tols, aod_1_outs, label="AOD = 0.1: Outliers", color="orange", linestyle="solid" +) +ax.plot( + tols, aod_1_good, label="AOD = 0.1: Good Data", color="orange", linestyle="dotted" +) +ax.plot(tols, aod_2_outs, label="AOD = 0.2: Outliers", color="blue", linestyle="solid") +ax.plot( + tols, aod_2_good, label="AOD = 0.2: Good Data", color="blue", linestyle="dotted" +) +ax.plot(tols, aod_3_outs, label="AOD = 0.3: Outliers", color="green", linestyle="solid") +ax.plot( + tols, aod_3_good, label="AOD = 0.3: Good Data", color="green", linestyle="dotted" +) + +ax.set_xlabel("Tolerance") +ax.set_ylabel("Percentage of Data Removed") +ax.set_title("Cloud Check Tolerance Analysis for 550nm") +""" + +ax.plot(tols, aod_0_outs - aod_0_good, label = 'AOD = 0.0', color = 'r', linestyle = 'solid') +ax.plot(tols, aod_1_outs - aod_1_good, label = 'AOD = 0.1', color = 'orange', linestyle = 'solid') +ax.plot(tols, aod_2_outs - aod_2_good, label = 'AOD = 0.2', color = 'blue', linestyle = 'solid') +ax.plot(tols, aod_3_outs - aod_3_good, label = 'AOD = 0.3', color ='green', linestyle = 'solid') + + +ax.set_xlabel('Tolerance') +ax.set_ylabel('Difference in Percentage of Data Removed') +ax.set_title('Cloud Check Tolerance Analysis for 550nm') +""" + +plt.legend() +plt.show() diff --git a/hypernets_processor/post_processing/hypernets_brdf_data_io.py b/hypernets_processor/post_processing/hypernets_brdf_data_io.py new file mode 100644 index 00000000..2bee5bcf --- /dev/null +++ b/hypernets_processor/post_processing/hypernets_brdf_data_io.py @@ -0,0 +1,399 @@ +"""hypernets_brdf.subpackage_template.module_template - describe class""" + +__author__ = [ + "Pieter De Vis ", +] +__all__ = [] + +from brdf_model import BRDFMeasurementsSpectra +import glob +import os +import xarray as xr +from dateutil.parser import parse +import datetime +import numpy as np +from obsarray.templater.dataset_util import DatasetUtil +from typing import Optional, Union, List, Any + +bad_flags = [ + "pt_ref_invalid", + "half_of_scans_masked", + "not_enough_dark_scans", + "not_enough_rad_scans", + "not_enough_irr_scans", + "no_clear_sky_irradiance", + "variable_irradiance", + "half_of_uncertainties_too_big", + "discontinuity_VNIR_SWIR", + "single_irradiance_used", +] + + +def read_data_hypernets( + files, + spatial_var_percent=0, + i=None, + mask=True, + vzamax=None, + vza=None, + vaa=None, + filter_flags=True, +) -> BRDFMeasurementsSpectra: + """ + Function to read full HYPERNETS dataset + + :param path: + :param start_time: + :param stop_time: + :return: + """ + if i is not None: + ds_HYP = read_hypernets_file( + files[i], + vza=vza, + vaa=vaa, + vzamax=vzamax, + mask=mask, + filter_flags=filter_flags, + ) + else: + for ii in range(len(files)): + ds_HYP = read_hypernets_file( + files[ii], + vza=vza, + vaa=vaa, + vzamax=vzamax, + mask=mask, + filter_flags=filter_flags, + ) + if ds_HYP is not None: + break + reflectance = ds_HYP["reflectance"].values + u_rand_refl = ( + ds_HYP["u_rel_random_reflectance"].values * reflectance / 100 + + spatial_var_percent * reflectance / 100 + ) + u_syst_refl = ds_HYP["u_rel_systematic_reflectance"].values * reflectance / 100 + corr_syst_refl = ds_HYP["err_corr_systematic_reflectance"].values + geometries = np.array( + [ + ds_HYP["solar_zenith_angle"].values, + ds_HYP["viewing_zenith_angle"].values, + ds_HYP["solar_azimuth_angle"].values, + ds_HYP["viewing_azimuth_angle"].values, + ] + ) + corr_reflectances_wav = np.mean( + [ + ds_HYP.unc["reflectance"][:, i].total_err_corr_matrix() + for i in range(len(ds_HYP["solar_zenith_angle"].values)) + ], + axis=0, + ) + + meas = BRDFMeasurementsSpectra( + ds_HYP["wavelength"].values, + geometries, + ds_HYP["acquisition_time"].values, + reflectance, + u_rand_refl, + u_syst_refl, + corr_reflectances_wav=corr_reflectances_wav, + ids=ds_HYP.attrs["sequence_id"], + ) + + if i is None: + for ii in range(1, len(files)): + ds_HYP = read_hypernets_file( + files[ii], + vza=vza, + vaa=vaa, + vzamax=vzamax, + mask=mask, + filter_flags=filter_flags, + ) + + if ds_HYP is not None: + reflectance = ds_HYP["reflectance"].values + u_rand_refl = ( + ds_HYP["u_rel_random_reflectance"].values * reflectance / 100 + + spatial_var_percent * reflectance / 100 + ) + u_syst_refl = ( + ds_HYP["u_rel_systematic_reflectance"].values * reflectance / 100 + ) + geometries = np.array( + [ + ds_HYP["solar_zenith_angle"].values, + ds_HYP["viewing_zenith_angle"].values, + ds_HYP["solar_azimuth_angle"].values, + ds_HYP["viewing_azimuth_angle"].values, + ] + ) + try: + meas.add_measurement( + geometries, + ds_HYP["acquisition_time"].values, + reflectance, + u_rand_reflectance=u_rand_refl, + u_syst_reflectance=u_syst_refl, + id=ds_HYP.attrs["sequence_id"], + ) + except: + print("Could not add measurement from file: " + files[ii]) + + return meas + + +def read_hypernets_file( + filepath, vza=None, vaa=None, vzamax=None, mask=True, filter_flags=True +): + ds_HYP = xr.open_dataset(filepath) + if filter_flags: + flagged = DatasetUtil.get_flags_mask_or(ds_HYP["quality_flag"], bad_flags) + id_series = np.where(~flagged)[0] + # print(len(id_series), " series selected on quality flag (out of %s total)"%len(ds.series.values)) + ds_HYP = ds_HYP.isel(series=id_series) + if len(ds_HYP.series) == 0: + return None + + if (vza is not None) and (vaa is not None): + vzadiff = ds_HYP["viewing_zenith_angle"].values - vza + if vza <= 2.5: + vaadiff = np.abs(ds_HYP["viewing_azimuth_angle"].values - vaa % 180) % 180 + else: + vaadiff = np.abs(ds_HYP["viewing_azimuth_angle"].values - vaa % 360) + angledif_series = vzadiff**2 + vaadiff**2 + id_series = np.where(angledif_series == np.min(angledif_series))[0] + + elif vza is not None: + angledif_series = (ds_HYP["viewing_zenith_angle"].values - vza) ** 2 + id_series = np.where(angledif_series == np.min(angledif_series))[0] + + elif vaa is not None: + angledif_series = ( + np.abs(ds_HYP["viewing_azimuth_angle"].values - vaa % 360) + ) ** 2 + id_series = np.where(angledif_series == np.min(angledif_series))[0] + + else: + id_series = ds_HYP.series.values + ds_HYP = ds_HYP.isel(series=id_series) + + if vzamax is not None: + ds_HYP = ds_HYP.isel( + series=np.where(ds_HYP["viewing_zenith_angle"].values < vzamax)[0] + ) + + if mask: + ds_HYP = mask_HYPERNETS_BRDF(ds_HYP) + + if len(ds_HYP.series) > 0: + return ds_HYP + + +def filter_files_start_stop( + files, start_time, stop_time, tod_start=None, tod_stop=None +): + start_time = convert_datetime(start_time) + stop_time = convert_datetime(stop_time) + files_out = [] + + if tod_start is not None: + tod_start = datetime.datetime.strptime(tod_start, "%H%M").time() + + if tod_stop is not None: + tod_stop = datetime.datetime.strptime(tod_stop, "%H%M").time() + + for file in files: + try: + ds_HYP = xr.open_dataset(file) + times = [ + datetime.datetime.utcfromtimestamp(timestamp) + for timestamp in ds_HYP["acquisition_time"].values + ] + if min(times) > start_time and max(times) < stop_time: + if (tod_start is None or min(times).time() > tod_start) and ( + tod_stop is None or max(times).time() < tod_stop + ): + files_out.append(file) + except: + continue + return files_out + +def filter_files_quick( + files, start_time, stop_time, tod_start=None, tod_stop=None +): + start_time = convert_datetime(start_time) + stop_time = convert_datetime(stop_time) + files_out = [] + + if tod_start is not None: + tod_start = datetime.datetime.strptime(tod_start, "%H%M").time() + + if tod_stop is not None: + tod_stop = datetime.datetime.strptime(tod_stop, "%H%M").time() + + for file in files: + time_stamp = file.split('\\')[5][3:11] + times = datetime.datetime.strptime(time_stamp, '%Y%m%d') + + if times > start_time and times < stop_time: + if (tod_start is None or times.time() > tod_start) and ( + tod_stop is None or times.time() < tod_stop + ): + files_out.append(file) + return files_out + + +def mask_HYPERNETS_BRDF(ds_HYP): + + # mask_boom + mask_shadow = np.where( + ~( + (ds_HYP["viewing_azimuth_angle"].values > 345) + | (ds_HYP["viewing_azimuth_angle"].values < 15) + ) + )[0] + ds_HYP = ds_HYP.isel(series=mask_shadow) + + # mask feet and mast here using bitwise operators such as: | for or , & for and + # mask_shadow = np.where( + # ~( + # ( + # (ds_HYP["viewing_azimuth_angle"].values > 225) + # | (ds_HYP["viewing_azimuth_angle"].values < 90) + # ) + # & (ds_HYP["viewing_zenith_angle"].values < 15) + # ) + # )[0] + # ds_HYP = ds_HYP.isel(series=mask_shadow) + + mask_shadow = np.where( + ~( + ( + (ds_HYP["viewing_azimuth_angle"].values > 300) + | (ds_HYP["viewing_azimuth_angle"].values < 60) + ) + & (ds_HYP["viewing_zenith_angle"].values < 25) + ) + )[0] + ds_HYP = ds_HYP.isel(series=mask_shadow) + + # mask sun shadow + + mask_shadow = np.where( + ~( + ( + np.abs( + ds_HYP["viewing_azimuth_angle"].values + - ds_HYP["solar_azimuth_angle"].values + ) + < 10 + ) + & ( + ds_HYP["viewing_zenith_angle"].values + < ds_HYP["solar_zenith_angle"].values + 5 + ) + ) + )[0] + ds_HYP = ds_HYP.isel(series=mask_shadow) + + # additional mask for specific additional mast anomalies (AM) + # mask_shadow = np.where( + # ~( + # ( + # (ds_HYP["viewing_azimuth_angle"].values > 15) + # & (ds_HYP["viewing_azimuth_angle"].values < 30) + # ) + # & (ds_HYP["viewing_zenith_angle"].values < 30) + # ) + # )[0] + # ds_HYP = ds_HYP.isel(series=mask_shadow) + # + # #additional mask for single additional high anomaly (PM) + # mask_shadow = np.where( + # ~( + # ( + # (ds_HYP["viewing_azimuth_angle"].values > 285) + # & (ds_HYP["viewing_azimuth_angle"].values < 300) + # ) + # & (ds_HYP["viewing_zenith_angle"].values < 15) + # ) + # )[0] + # ds_HYP = ds_HYP.isel(series=mask_shadow) + + return ds_HYP + + +def read_data_hypernets_sequence(self, path): + """ + Function to read single HYPERNETS sequence + + :param path: + :return: + """ + + pass + + +def read_data_CIMEL(self, path, start_time, stop_time) -> BRDFMeasurementsSpectra: + """ + Function to read CIMEL dataset + + :param path: + :param start_time: + :param stop_time: + :return: + """ + + pass + + +def read_data_CIMEL_ASCII(self, path): + """ + Function to read single CIMEL ASCII file + + :param path: + :return: + """ + + pass + + +def convert_datetime( + date_time: Union[datetime.datetime, datetime.date, str, float, int, np.ndarray], +) -> datetime.datetime: + """ + Convert input datetimes to a datetime object + + :param date_time: date time to convert to a datetime object + :return: datetime object corresponding to input date_time + """ + if isinstance(date_time, np.ndarray): + return np.array([convert_datetime(date_time_i) for date_time_i in date_time]) + elif isinstance(date_time, datetime.datetime): + return date_time + elif isinstance(date_time, datetime.date): + return datetime.datetime.combine(date_time, datetime.time()) + elif isinstance(date_time, np.datetime64): + unix_epoch = np.datetime64(0, "s") + one_second = np.timedelta64(1, "s") + seconds_since_epoch = (date_time - unix_epoch) / one_second + return datetime.datetime.utcfromtimestamp(seconds_since_epoch) + elif isinstance(date_time, (float, int, np.uint)): + return datetime.datetime.utcfromtimestamp(date_time) + else: + if date_time[-1] == "Z": + date_time = date_time[:-1] + try: + return parse(date_time, fuzzy=False) + except ValueError: + raise ValueError( + "Unable to discern datetime requested: '{}'".format(date_time) + ) + + +if __name__ == "__main__": + pass diff --git a/hypernets_processor/post_processing/hypernets_merge_parallel_db.py b/hypernets_processor/post_processing/hypernets_merge_parallel_db.py index 9816afb5..d241f410 100644 --- a/hypernets_processor/post_processing/hypernets_merge_parallel_db.py +++ b/hypernets_processor/post_processing/hypernets_merge_parallel_db.py @@ -17,31 +17,31 @@ # "archive_ATGE.db", # "archive_BASP.db", "archive_DEGE.db", - #"archive_GHNA.db", - #"archive_IFAR.db", - #"archive_LOBE.db", - #"archive_PEAN.db", + # "archive_GHNA.db", + # "archive_IFAR.db", + # "archive_LOBE.db", + # "archive_PEAN.db", # "archive_WWUK.db", - #"archive_JAES.db", + # "archive_JAES.db", ] files_anomaly = [ # "anomaly_ATGE.db", # "anomaly_BASP.db", "anomaly_DEGE.db", - #"anomaly_GHNA.db", - #"anomaly_IFAR.db", - #"anomaly_LOBE.db", - #"anomaly_PEAN.db", + # "anomaly_GHNA.db", + # "anomaly_IFAR.db", + # "anomaly_LOBE.db", + # "anomaly_PEAN.db", # "anomaly_WWUK.db", - #"anomaly_JAES.db", + # "anomaly_JAES.db", ] files_metadata = [ # "metadata_ATGE.db", # "metadata_BASP.db", "metadata_DEGE.db", # "metadata_GHNA.db", - #"metadata_IFAR.db", - #"metadata_LOBE.db", + # "metadata_IFAR.db", + # "metadata_LOBE.db", # "metadata_PEAN.db", # "metadata_WWUK.db", # "metadata_JAES.db", @@ -81,7 +81,7 @@ def merge(outfile, file, table_name): ) id_prev = list(cursor_a.execute("SELECT id FROM " + table_name)) if len(id_prev) > 0: - #index_prev = id_prev[-1] + # index_prev = id_prev[-1] index_prev = [len(id_prev)] for i, row in enumerate(cursor_b.execute("SELECT * FROM " + table_name)): row = ["none" if (is_invalid(v)) else v for v in row] diff --git a/hypernets_processor/post_processing/hypernets_remove_L2B_from_db.py b/hypernets_processor/post_processing/hypernets_remove_L2B_from_db.py new file mode 100644 index 00000000..ead65117 --- /dev/null +++ b/hypernets_processor/post_processing/hypernets_remove_L2B_from_db.py @@ -0,0 +1,230 @@ +import glob +import sqlite3 +import os +import sys +import numpy as np +from hypernets_processor.version import __version__ +import shutil + +"""___Authorship___""" +__author__ = "Pieter De Vis" +__created__ = "29/3/2023" +__version__ = __version__ +__maintainer__ = "Pieter De Vis" +__email__ = "pieter.de.vis@npl.co.uk" +__status__ = "Development" + +# set your archive path, and database names +archive_path = r"/archive/" +# archive_path = r"C:\Users\pdv\data\insitu\hypernets\archive" + +archive_db = "archive.db" +anomaly_db = "anomaly.db" +metadata_db = "metadata.db" + +# the following fields can be used to control the behavious of the script. +# If these arfe set to none, the user will be prompted for input through the command line. +# To avoid being prompted, set these to the appropriate values for what you want to remove +# (see examples below) + +bad_sequence_list = None # instead of using a SQL query, it is also possible to provide manually a list of Sequence_names +sql_query = None # Here provide using SQLite the conditions (i.e. the WHERE clause in SQL) for which you would like to remove sequences +list_info = False # If True, show list of all the products/anomalies that will be removed (this is slower). If False, only sequence_name and product_dir will be shown. + +# Here are some example sql queries you can use to filter sequences (you can connecty these by AND/OR) +# --- Archive DB (products table) --- +# sql_query = "site_id = 'ATGE'" +# sql_query = "datetime_SEQ >= '2025-01-01'" +# sql_query = "datetime_start >= '2025-01-01' AND datetime_end <= '2025-11-01'" +# sql_query = "sequence_name LIKE 'SEQ2025%'" +# sql_query = "product_level = 'L2A'" +# sql_query = "solar_zenith_angle_max < 70.0" +# sql_query = "percent_zero_flags > 0.1" + +# --- Anomaly DB (anomalies table) --- +# sql_query = "anomaly_id = 'x'" +# sql_query = "anomaly_id IN ('x', 'ms')" +# sql_query = "datetime >= '2025-01-01' AND datetime <= '2025-11-01'" +# sql_query = "site_id = 'ATGE'" +# sql_query = "sequence_name LIKE 'SEQ2025%'" +# sql_query = "product_level_last = 'L2A'" +# sql_query = "datetime >= '2025-01-01' AND datetime <= '2025-11-01'" + +# Set your params here is you want to avoid specifying them through the command line. +db_choice = "anomaly" +# db_choice = "archive" + +#bad_sequence_list = ["SEQ20251114T110045"] + +sql_query = "site_id = 'GHNA' and datetime_SEQ >= '2025-09-01'" + +def get_products(db_path, sql_query): + conn = sqlite3.connect(db_path) + cursor = conn.cursor() + sequences = [] + for row in cursor.execute(f"SELECT sequence_name,rel_product_dir,product_name FROM products WHERE product_level='L_L2B' AND {sql_query}"): + sequences.append(row) + conn.close() + sequences=np.array(sequences) + if len(sequences)==0: + return sequences + unique_ids=np.unique(sequences[:,2],return_index=True)[1] + sequences=sequences[unique_ids] + return sequences + +def get_amomalies_from_anomaly_db(db_path, sql_query): + conn = sqlite3.connect(db_path) + cursor = conn.cursor() + sequences = [] + print(f"SELECT sequence_name,rel_product_dir,anomaly_id FROM anomalies WHERE anomaly_id IN ('per','val','tod','hsn','scl','npr','man','wns','nos','hos') AND {sql_query.replace('datetime_SEQ','datetime')}") + for row in cursor.execute(f"SELECT sequence_name,rel_product_dir,anomaly_id FROM anomalies WHERE anomaly_id IN ('per','val','tod','hsn','scl','npr','man','wns','nos','hos') AND {sql_query.replace('datetime_SEQ','datetime')}"): + sequences.append(row) + conn.close() + return sequences + +def get_all_info(db_path, table_name, sequence_id): + conn = sqlite3.connect(db_path) + cursor = conn.cursor() + sequence_info = [] + if table_name=="anomalies": + return_key = "anomaly_id" + else: + return_key = "product_name" + for row in cursor.execute(f"SELECT {return_key} FROM {table_name} WHERE sequence_name=?", (sequence_id,)): + sequence_info.append(row) + conn.close() + return sequence_info + +def remove_from_anomaly_db(db_path, seq_name,sql_query): + conn = sqlite3.connect(db_path) + cursor = conn.cursor() + cursor.execute(f"DELETE FROM anomalies WHERE anomaly_id IN ('per','val','tod','hsn','scl','npr','man','wns','nos','hos') AND sequence_name='{seq_name}' AND {sql_query.replace('datetime_SEQ','datetime')}") + conn.commit() + conn.close() + +def remove_all_from_anomaly_db(db_path,sql_query): + conn = sqlite3.connect(db_path) + cursor = conn.cursor() + cursor.execute(f"DELETE FROM anomalies WHERE anomaly_id IN ('per','val','tod','hsn','scl','npr','man','wns','nos','hos') AND {sql_query.replace('datetime_SEQ','datetime')}") + conn.commit() + conn.close() + +def remove_all_from_metadata_db(db_path,sql_query): + conn = sqlite3.connect(db_path) + cursor = conn.cursor() + cursor.execute(f"DELETE FROM L_L2B WHERE {sql_query}") + conn.commit() + conn.close() + +def remove_all_from_archive_db(db_path,sql_query): + conn = sqlite3.connect(db_path) + cursor = conn.cursor() + cursor.execute(f"DELETE FROM products WHERE product_level='L_L2B' AND {sql_query}") + conn.commit() + conn.close() + +def remove_product_from_db(cursor, table_name, product_name, metadata=False): + if metadata: + try: + cursor.execute(f"DELETE FROM {table_name} WHERE product_name=?", (product_name,)) + except: + print(f"product_name not found in {table_name} in metadata.db") + else: + cursor.execute(f"DELETE FROM {table_name} WHERE product_name=?", (product_name,)) + + +def load_table_names(db_a): + cursor_a = db_a.cursor() + cursor_a.execute("SELECT name FROM sqlite_master WHERE type='table';") + table_names = [table_item[0] for table_item in cursor_a.fetchall()] + return table_names + +def delete_files(file_path): + if os.path.exists(file_path): + try: + os.remove(file_path) + print(f"Deleted file: {file_path}") + except Exception as e: + print(f"Error deleting {file_path}: {e}") + else: + print(f"File/Folder does not exist: {file_path}") + +def main(archive_path=None, bad_sequence_list=None, sql_query=None): + if archive_path is None: + archive_path = input("Enter the archive path (where the .db files are located): ").strip() + db_file = archive_db + db_path = os.path.join(archive_path, db_file) + if bad_sequence_list is not None: + sql_query = f"product_level = 'L_L2B' AND sequence_name IN {str(tuple(bad_sequence_list))}" + elif sql_query is None: + sql_query = input("Enter SQL WHERE clause to filter sequences (e.g., date < '2023-01-01'): ").strip() + sequences = get_products(db_path, sql_query) + print(f"Found {len(sequences)} products.") + for idx, seq in enumerate(sequences): + print(f"[{idx}] ID: {seq[0]}, product_path: {seq[1]}, product: {seq[2]}") + + to_remove = input("Enter comma-separated indices of sequences to remove (or `*' for all): ").strip() + if to_remove == "*": + indices = range(len(sequences)) + else: + indices = [int(i) for i in to_remove.split(",") if i.isdigit()] + + for i in indices: + seq = sequences[i] + # remove file + delete_files(os.path.join(archive_path,seq[1],seq[2]+".nc")) + #remove plots + glob_files = glob.glob(os.path.join(archive_path,seq[1],"plots/*L2B*.nc")) + if len(glob_files)>0: + for gf in glob_files: + delete_files(gf) + + #removem product from metadata db + db_path = os.path.join(archive_path, metadata_db) + + if to_remove == "*" and "datetime" not in sql_query: + remove_all_from_metadata_db(db_path, sql_query) + else: + conn = sqlite3.connect(db_path) + cursor = conn.cursor() + for i in indices: + seq = sequences[i] + remove_product_from_db(cursor, "L_L2B", seq[2], metadata=True) + conn.commit() + conn.close() + + #remove product from archive db + db_path = os.path.join(archive_path, archive_db) + + if to_remove == "*": + remove_all_from_archive_db(db_path, sql_query) + else: + conn = sqlite3.connect(db_path) + cursor = conn.cursor() + for i in indices: + seq = sequences[i] + remove_product_from_db(cursor, "products", seq[2], metadata=False) + conn.commit() + conn.close() + print(f"L2B products have been removed from archive db and files deleted.") + + db_path = os.path.join(archive_path, anomaly_db) + sequences = get_amomalies_from_anomaly_db(db_path,sql_query) + + print(f"Found {len(sequences)} anomalies.") + for idx, seq in enumerate(sequences): + print(f"[{idx}] ID: {seq[0]}, product_path: {seq[1]}, anomaly: {seq[2]}") + to_remove = input("Enter comma-separated indices of sequences to remove (or `*' for all): ").strip() + if to_remove == "*": + indices = range(len(sequences)) + else: + indices = [int(i) for i in to_remove.split(",") if i.isdigit()] + + for i in indices: + seq = sequences[i] + remove_from_anomaly_db(db_path, seq[0], sql_query) + + print(f"Sequences have been removed from anomaly db") + +if __name__ == "__main__": + main(archive_path, bad_sequence_list, sql_query) \ No newline at end of file diff --git a/hypernets_processor/post_processing/hypernets_remove_duplicates.py b/hypernets_processor/post_processing/hypernets_remove_duplicates.py index 5f8eb816..30004000 100644 --- a/hypernets_processor/post_processing/hypernets_remove_duplicates.py +++ b/hypernets_processor/post_processing/hypernets_remove_duplicates.py @@ -1,3 +1,5 @@ +import glob + from hypernets_processor.version import __version__ import sys, sqlite3 import os @@ -14,20 +16,19 @@ archive_path = r"/home/data/insitu/hypernets/archive" files_archive = [ - "archive_ATGE.db", - "archive_BASP.db", - "archive_WWUK.db", -] -files_anomaly = [ - "anomaly_ATGE.db", - "anomaly_BASP.db", - "anomaly_WWUK.db", -] -files_metadata = [ - "metadata_ATGE.db", - "metadata_BASP.db", - "metadata_WWUK.db", + "archive.db", ] +# files_anomaly = [ +# "anomaly_ATGE.db", +# "anomaly_BASP.db", +# "anomaly_WWUK.db", +# ] +# files_metadata = [ +# "metadata_ATGE.db", +# "metadata_BASP.db", +# "metadata_WWUK.db", +# ] + def remove_files(archive_path, file, table_name): db_b = sqlite3.connect(os.path.join(archive_path, file)) @@ -35,36 +36,38 @@ def remove_files(archive_path, file, table_name): for i, row in enumerate(cursor_b.execute("SELECT * FROM " + table_name)): row = ["none" if (is_invalid(v)) else v for v in row] row = tuple(row) - date=row[1][39:52] - try: - os.remove(row[7]) - except: - print(row[7], "file does not exist") - - dir_name = os.path.join(os.path.dirname(row[7]), "plots") - test = os.listdir(dir_name) - - for item in test: - if date in item: - try: - os.remove(os.path.join(dir_name, item)) - except: - print(os.path.join(dir_name, item), "file does not exist") - - dir_name = os.path.join(os.path.dirname(row[7]), "image") - test = os.listdir(dir_name) - - for item in test: - if date in item: - try: - os.remove(os.path.join(dir_name, item)) - except: - print(os.path.join(dir_name, item), "file does not exist") - - try: - os.remove(row[7]) - except: - print(row[7], "file does not exist") + date = row[1][39:52] + print(row[7]) + # try: + # os.remove(row[7]) + # except: + # print(row[7], "file does not exist") + + # dir_name = os.path.join(os.path.dirname(row[7]), "plots") + # test = os.listdir(dir_name) + # + # for item in test: + # if date in item: + # try: + # os.remove(os.path.join(dir_name, item)) + # except: + # print(os.path.join(dir_name, item), "file does not exist") + # + # dir_name = os.path.join(os.path.dirname(row[7]), "image") + # test = os.listdir(dir_name) + # + # for item in test: + # if date in item: + # try: + # os.remove(os.path.join(dir_name, item)) + # except: + # print(os.path.join(dir_name, item), "file does not exist") + # + # try: + # os.remove(row[7]) + # except: + # print(row[7], "file does not exist") + def is_invalid(v): if v is None: @@ -90,12 +93,44 @@ def load_table_names(db_a): if __name__ == "__main__": - print(os.path.join(archive_path, files_archive[0])) - db_a = sqlite3.connect(os.path.join(archive_path, files_archive[0])) - table_names = load_table_names(db_a) - db_a.close() - print(table_names) - for file in files_archive: - print(file) - for table_name in table_names: - remove_files(archive_path, file, table_name) + # print(os.path.join(archive_path, files_archive[0])) + # db_a = sqlite3.connect(os.path.join(archive_path, files_archive[0])) + # table_names = load_table_names(db_a) + # db_a.close() + # print(table_names) + # for file in files_archive: + # print(file) + # for table_name in table_names: + # remove_files(archive_path, file, table_name) + + files = [ + "HYPERNETS_L_GHNA_L2B_REF_20231026T0652_20250620T2025_v2.2", + "HYPERNETS_L_GHNA_L2B_REF_20231026T0730_20250620T2025_v2.2", + "HYPERNETS_L_GHNA_L2B_REF_20231026T0801_20250620T2025_v2.2", + "HYPERNETS_L_GHNA_L2B_REF_20231026T0831_20250620T2026_v2.2", + "HYPERNETS_L_GHNA_L2B_REF_20231026T0900_20250620T2026_v2.2", + "HYPERNETS_L_GHNA_L2B_REF_20231026T1000_20250620T2026_v2.2", + "HYPERNETS_L_GHNA_L2B_REF_20231026T1030_20250620T2026_v2.2", + "HYPERNETS_L_GHNA_L2B_REF_20231026T1101_20250620T2026_v2.2", + "HYPERNETS_L_GHNA_L2B_REF_20231026T1131_20250620T2026_v2.2", + "HYPERNETS_L_GHNA_L2B_REF_20231026T1300_20250620T2026_v2.2", + "HYPERNETS_L_GHNA_L2B_REF_20231026T1330_20250620T2026_v2.2", + "HYPERNETS_L_GHNA_L2B_REF_20231026T1400_20250620T2027_v2.2", + "HYPERNETS_L_GHNA_L2B_REF_20231026T1430_20250620T2027_v2.2", + "HYPERNETS_L_GHNA_L2B_REF_20231026T1500_20250620T2027_v2.2", + "HYPERNETS_L_GHNA_L2B_REF_20231026T1531_20250620T2027_v2.2", + "HYPERNETS_L_GHNA_L2B_REF_20231028T0701_20250620T2027_v2.2", + "HYPERNETS_L_GHNA_L2B_REF_20231028T0730_20250620T2027_v2.2", + "HYPERNETS_L_GHNA_L2B_REF_20231028T0801_20250620T2027_v2.2", + "HYPERNETS_L_GHNA_L2B_REF_20231028T0830_20250620T2027_v2.2", + "HYPERNETS_L_GHNA_L2B_REF_20231028T0901_20250620T2027_v2.2", + "HYPERNETS_L_GHNA_L2B_REF_20231028T0930_20250620T2028_v2.2", + "HYPERNETS_L_GHNA_L2B_REF_20231028T1000_20250620T2028_v2.2", + "HYPERNETS_L_GHNA_L2B_REF_20231028T1031_20250620T2028_v2.2", + "HYPERNETS_L_GHNA_L2B_REF_20231028T1100_20250620T2028_v2.2", + ] + for file in files: + file_b = glob.glob( + "/home/data/insitu/hypernets/archive/GHNA/2023/10/*/*/%s*" % file + ) + os.remove(file_b[0]) diff --git a/hypernets_processor/post_processing/hypernets_remove_sequences_from_db.py b/hypernets_processor/post_processing/hypernets_remove_sequences_from_db.py new file mode 100644 index 00000000..9fd527f7 --- /dev/null +++ b/hypernets_processor/post_processing/hypernets_remove_sequences_from_db.py @@ -0,0 +1,164 @@ +import glob +import sqlite3 +import os +import sys +import numpy as np +from hypernets_processor.version import __version__ +import shutil + +"""___Authorship___""" +__author__ = "Pieter De Vis" +__created__ = "29/3/2023" +__version__ = __version__ +__maintainer__ = "Pieter De Vis" +__email__ = "pieter.de.vis@npl.co.uk" +__status__ = "Development" + +# set your archive path, and database names +archive_path = r"/archive/" +#archive_path = r"C:\Users\pdv\data\insitu\hypernets\archive" +archive_db = "archive.db" +anomaly_db = "anomaly.db" +metadata_db = "metadata.db" + +# the following fields can be used to control the behavious of the script. +# If these arfe set to none, the user will be prompted for input through the command line. +# To avoid being prompted, set these to the appropriate values for what you want to remove +# (see examples below) + +db_choice = None # which db should be queried to identify sequences that should be removed (the selected sequences wil be removed from all three db) +bad_sequence_list = None # instead of using a SQL query, it is also possible to provide manually a list of Sequence_names +sql_query = None # Here provide using SQLite the conditions (i.e. the WHERE clause in SQL) for which you would like to remove sequences +list_info = False # If True, show list of all the products/anomalies that will be removed (this is slower). If False, only sequence_name and product_dir will be shown. + +# Here are some example sql queries you can use to filter sequences (you can connecty these by AND/OR) +# --- Archive DB (products table) --- +# sql_query = "site_id = 'ATGE'" +# sql_query = "datetime_SEQ >= '2025-01-01'" +# sql_query = "datetime_start >= '2025-01-01' AND datetime_end <= '2025-11-01'" +# sql_query = "sequence_name LIKE 'SEQ2025%'" +# sql_query = "product_level = 'L2A'" +# sql_query = "solar_zenith_angle_max < 70.0" +# sql_query = "percent_zero_flags > 0.1" + +# --- Anomaly DB (anomalies table) --- +# sql_query = "anomaly_id = 'x'" +# sql_query = "anomaly_id IN ('x', 'ms')" +# sql_query = "datetime >= '2025-01-01' AND datetime <= '2025-11-01'" +# sql_query = "site_id = 'ATGE'" +# sql_query = "sequence_name LIKE 'SEQ2025%'" +# sql_query = "product_level_last = 'L2A'" +# sql_query = "datetime >= '2025-01-01' AND datetime <= '2025-11-01'" + +# Set your params here is you want to avoid specifying them through the command line. +db_choice = "anomaly" +# db_choice = "archive" + +# bad_sequence_list = ["SEQ20231026T080128", "SEQ20231031T073028"] + +sql_query = "anomaly_id IN ('x', 'ms') AND site_id = 'WWUK'" + + + +def get_sequences(db_path, table_name, sql_query): + conn = sqlite3.connect(db_path) + cursor = conn.cursor() + sequences = [] + for row in cursor.execute(f"SELECT sequence_name,rel_product_dir FROM {table_name} WHERE {sql_query}"): + sequences.append(row) + conn.close() + sequences=np.array(sequences) + if len(sequences)==0: + return sequences + unique_ids=np.unique(sequences[:,0],return_index=True)[1] + sequences=sequences[unique_ids] + return sequences + +def get_all_info(db_path, table_name, sequence_id): + conn = sqlite3.connect(db_path) + cursor = conn.cursor() + sequence_info = [] + if table_name=="anomalies": + return_key = "anomaly_id" + else: + return_key = "product_name" + for row in cursor.execute(f"SELECT {return_key} FROM {table_name} WHERE sequence_name=?", (sequence_id,)): + sequence_info.append(row) + conn.close() + return sequence_info + +def remove_sequence_from_db(db_path, table_name, sequence_id, metadata=False): + conn = sqlite3.connect(db_path) + cursor = conn.cursor() + if metadata: + try: + cursor.execute(f"DELETE FROM {table_name} WHERE sequence_id=?", (sequence_id,)) + except: + print(f"sequence_id not found in {table_name} in metadata.db") + else: + cursor.execute(f"DELETE FROM {table_name} WHERE sequence_name=?", (sequence_id,)) + conn.commit() + conn.close() + +def load_table_names(db_a): + cursor_a = db_a.cursor() + cursor_a.execute("SELECT name FROM sqlite_master WHERE type='table';") + table_names = [table_item[0] for table_item in cursor_a.fetchall()] + return table_names + +def delete_files(file_path): + if os.path.exists(file_path): + try: + shutil.rmtree(file_path) + print(f"Deleted folder: {file_path}") + except Exception as e: + print(f"Error deleting {file_path}: {e}") + else: + print(f"File/Folder does not exist: {file_path}") + +def main(archive_path=None, db_choice=None, bad_sequence_list=None, sql_query=None): + if archive_path is None: + archive_path = input("Enter the archive path (where the .db files are located): ").strip() + if db_choice is None: + db_choice = input("Select database (archive/anomaly): ").strip().lower() + db_file = archive_db if db_choice == "archive" else anomaly_db + db_path = os.path.join(archive_path, db_file) + conn = sqlite3.connect(db_path) + table_names = load_table_names(conn) + conn.close() + if bad_sequence_list is not None: + sql_query = f"sequence_name IN {str(tuple(bad_sequence_list))}" + elif sql_query is None: + sql_query = input("Enter SQL WHERE clause to filter sequences (e.g., date < '2023-01-01'): ").strip() + sequences = get_sequences(db_path, table_names[0], sql_query) + print(f"Found {len(sequences)} sequences.") + for idx, seq in enumerate(sequences): + if list_info: + print(f"[{idx}] ID: {seq[0]}, product_path: {seq[1]}, ids: {get_all_info(db_path, table_names[0], seq[0])}") + else: + print(f"[{idx}] ID: {seq[0]}, product_path: {seq[1]}") + + to_remove = input("Enter comma-separated indices of sequences to remove (or `*' for all): ").strip() + if to_remove == "*": + indices = range(len(sequences)) + else: + indices = [int(i) for i in to_remove.split(",") if i.isdigit()] + + for i in indices: + seq = sequences[i] + delete_files(os.path.join(archive_path,seq[1])) + for db in [metadata_db,anomaly_db,archive_db]: + db_path = os.path.join(archive_path, db) + conn = sqlite3.connect(db_path) + table_names = load_table_names(conn) + conn.close() + for table_name in table_names: + for i in indices: + seq = sequences[i] + remove_sequence_from_db(db_path, table_name, seq[0], metadata=(db==metadata_db)) + print(f"Sequence {seq[0]} removed from {table_name} in {db}.") + + print(f"Sequences have been removed from all the db and files deleted.") + +if __name__ == "__main__": + main(archive_path, db_choice, bad_sequence_list, sql_query) \ No newline at end of file diff --git a/hypernets_processor/post_processing/irr_generate_csv.py b/hypernets_processor/post_processing/irr_generate_csv.py new file mode 100644 index 00000000..284d9e95 --- /dev/null +++ b/hypernets_processor/post_processing/irr_generate_csv.py @@ -0,0 +1,171 @@ +import numpy as np +import hypernets_brdf_data_io as data_io +import os +import glob +import matplotlib + +matplotlib.use("Qt5Agg") +import matplotlib.pyplot as plt +import xarray as xr +import pandas as pd + +# set appropriate folders (different for linux or windows) and settings +data_path = r"T:\ECO\EOServer\data\insitu\hypernets\archive" +results_path = r"T:\ECO\EOServer\data\insitu\hypernets\post_processing_qc" +site = "PEAN" + +# results_path = os.path.join(results_path,brdf_model) +if not os.path.exists(results_path): + os.mkdir(results_path) +start_time = "20231230T0000" +stop_time = "20240123T0000" +wavelength = 550 + +# start_tod = ["0900","0930","1000","1030","1100"] +# stop_tod = ["0930","1000","1030","1100","1130"] +# +# start_tod = ["0900","1000","1100"] +# stop_tod = ["1000","1100","1200"] + +start_tod = [None] +stop_tod = [None] + +# #set minimum and maximum reflectance for colourbar plots (Set to None to let it work it out automatically) +vmin = 0.18 +vmax = 0.28 + +vzas = [20] +vaas = [83] + + +def find_nearest_to_wav(array, wv, value): + + wv = np.asarray(wv) + idx = (np.abs(wv - value)).argmin() + return array[idx] + + +data_clear = xr.open_dataset( + "T:/ECO/EOServer/data/insitu/hypernets/archive/PEAN/2023/12/30/SEQ20231230T133041/HYPERNETS_L_PEAN_L1B_IRR_20231230T1330_20241028T1648_v2.1.nc") +wav_ori = data_clear.wavelength.values +print("Got wavelengths") + + +def get_irr(data, wav_val): + data = xr.open_dataset(r"%s" % data) + wav = data.wavelength.values + irr = data.irradiance.to_pandas() + szas = data.solar_zenith_angle.values + saas = data.solar_azimuth_angle.values + if wav_val is None: + if irr.shape[1] != 2 or szas.shape[0] != 2: + irr1 = np.zeros(len(wav)) + irr2 = np.zeros(len(wav)) + qual = [999, 999] + sza = [999, 999] + saa = [999, 999] + else: + irr1 = np.array(irr.iloc[:, 0].values) + irr2 = np.array(irr.iloc[:, 1].values) + qual = data.quality_flag.values + sza = szas + saa = saas + else: + if irr.shape[1] != 2 or szas.shape[0] != 2: + irr1 = 0 + irr2 = 0 + qual = [999, 999] + sza = [999, 999] + else: + irr1 = find_nearest_to_wav(irr.iloc[:, 0].values, wav, wav_val) + irr2 = find_nearest_to_wav(irr.iloc[:, 1].values, wav, wav_val) + qual = data.quality_flag.values + sza = szas + + return irr1, irr2, wav, qual, sza, saa + + +def make_irrs(file_list, wav_val): + if wav_val is None: + irrs = pd.DataFrame(columns=["ID", "Flag", "SZA", "SAA"]) + refl = np.zeros((len(file_list) * 2, len(wav_ori))) + for j in range(len(file_list)): + try: + irr1, irr2, wav, qual, sza, saa = get_irr(file_list[j], wav_val=wav_val) + new_row1 = pd.Series( + { + "ID": file_list[j][62:80], + "Flag": qual[0], + "SZA": sza[0], + "SAA": saa[0], + } + ) + new_row2 = pd.Series( + { + "ID": file_list[j][62:80], + "Flag": qual[1], + "SZA": sza[1], + "SAA": saa[1], + } + ) + irrs = pd.concat([irrs, new_row1.to_frame().T], ignore_index=True) + irrs = pd.concat([irrs, new_row2.to_frame().T], ignore_index=True) + refl[2 * j, :] = irr1 + refl[2 * j + 1, :] = irr2 + except Exception as e: + print("failed due to", e) + refl[2 * j, :] = np.ones(len(wav_ori)) * -999 + refl[2 * j + 1, :] = np.ones(len(wav_ori)) * -999 + for k in range(len(wav_ori)): + irrs.insert(len(irrs.columns), "{}".format(k), refl[:, k]) + else: + irrs = pd.DataFrame(columns=["ID", "Flag", "SZA", "Data"]) + for j in range(len(file_list)): + irr1, irr2, wav, qual, sza = get_irr(file_list[j], wav_val=wav_val) + new_row1 = pd.Series( + { + "ID": file_list[j][62:80], + "Flag": qual[0], + "SZA": sza[0], + "Data": irr1, + } + ) + new_row2 = pd.Series( + { + "ID": file_list[j][62:80], + "Flag": qual[1], + "SZA": sza[1], + "Data": irr2, + } + ) + irrs = pd.concat([irrs, new_row1.to_frame().T], ignore_index=True) + irrs = pd.concat([irrs, new_row2.to_frame().T], ignore_index=True) + + irrs.to_csv(results_path + '\PEANv3_irradiance.csv', index=False) + return irrs + + +##running + +files = glob.glob(os.path.join(data_path, "PEAN", "*", "*", "*", "*", "*L1B_IRR*.nc")) + +print("Files In") + +for ii in range(len(start_tod)): + files = data_io.filter_files_start_stop( + files, start_time, stop_time, tod_start=start_tod[ii], tod_stop=stop_tod[ii] + ) +print("Files Sorted") + +irrs = make_irrs(files, None) +print(irrs.iloc[1]) +""" +data_cloud = 'T:\ECO\EOServer\data\insitu\hypernets\\archive\GHNA\\2024\\03\\12\SEQ20240312T070125\HYPERNETS_L_GHNA_L1B_IRR_20240312T0701_20240416T2306_v2.0.nc' +data_clear = 'T:\ECO\EOServer\data\insitu\hypernets\\archive\GHNA\\2024\\03\\14\SEQ20240314T070025\HYPERNETS_L_GHNA_L1B_IRR_20240314T0700_20240416T1138_v2.0.nc' + +irr_clear1, irr_clear2, wav_clear, qual_clear, sza_clear = get_irr(data_clear, 550) +irr_cloud1, irr_cloud2, wav_cloud, qual_cloud, sza_cloud = get_irr(data_cloud, 550) + +plt.plot(wav_clear - wav_cloud) +plt.show() +""" diff --git a/hypernets_processor/post_processing/joe_quality_checks.py b/hypernets_processor/post_processing/joe_quality_checks.py new file mode 100644 index 00000000..26b23997 --- /dev/null +++ b/hypernets_processor/post_processing/joe_quality_checks.py @@ -0,0 +1,429 @@ +import numpy as np +import pandas as pd +import os +import matplotlib.pyplot as plt +from matplotlib import colors +from matplotlib.pyplot import tight_layout +import scipy.optimize as spo +from setuptools.windows_support import windows_only + +# read in data +windows_results_path = r"T:/ECO/EOServer/data/insitu/hypernets/post_processing_qc/joe" + +linux_data_path = r"/mnt/t/data/insitu/hypernets/post_processing_qc" +windows_data_path = r"T:/ECO/EOServer/data/insitu/hypernets/post_processing_qc" + +data = pd.read_csv(windows_data_path + r"/JSIT_None_None_None_None.csv") + + +maintenance_dates = [ + "20240513", + "20240514", + "20240515", + "20240516", + "20240517", + "20240518", + "20240519", + "20240520", +] + + +##get outlier dataset + + +def get_outliers(data, sigmas): + mean = data[" refl_550nm"].mean() + stde = data[" refl_550nm"].std() + print(mean) + print(stde) + max_range = mean + (stde * sigmas) + min_range = mean - (stde * sigmas) + outliers = data[ + (data[" refl_550nm"] > max_range) | (data[" refl_550nm"] < min_range) + ] + good_data = data[ + (data[" refl_550nm"] < max_range) & (data[" refl_550nm"] > min_range) + ] + good_data.reset_index(drop=True, inplace=True) + outliers.reset_index(drop=True, inplace=True) + + fig, axs = plt.subplots(1, 3, figsize=(6, 4), sharey=True) + for i in range(3): + axs[i].axhline(y=mean, color="blue") + axs[i].axhline(y=mean - (stde * sigmas), color="green") + axs[i].axhline(y=mean + (stde * sigmas), color="orange") + axs[i].set_title(f"SZA = {(i + 1) * 20}") + axs[i].scatter( + data[data[" sza"].between((i + 1) * 20 - 1, (i + 1) * 20 + 1)][ + " vza" + ].values, + data[data[" sza"].between((i + 1) * 20 - 1, (i + 1) * 20 + 1)][ + " refl_550nm" + ].values, + color="black", + s=5, + ) + axs[i].set_xlabel("VZA") + axs[0].set_ylabel("Reflectance") + plt.show() + return outliers, good_data + + +##get dates and times of measurements + + +def get_dates_times(data): + dates = [] + times = [] + for i in range(len(data)): + date = data["# id"][i][3:11] + time = data["# id"][i][12:16] + dates.append(date) + times.append(time) + new_data = data.assign(date=dates, time=times) + return new_data + + +##get relative azimuthal angle + + +def get_raa(data): + data.loc[:, "raa"] = (data.loc[:, " vaa"] - data.loc[:, " saa"] + 360) % 360 + + +##manual instrument maintenance day removals + + +def maintenance_check(data, bad_dates): + ### note bad_dates need to be in speech marks### + bad_inds = [] + # get dates of measurements + for i in range(len(data)): + date = data["date"][i] + + # remove maintenance dates + if date in bad_dates: + bad_index = i + bad_inds.append(bad_index) + else: + continue + + cleaned_data = data.drop(bad_inds) + cleaned_data.reset_index(drop=True, inplace=True) + return cleaned_data + + +##sunrise check + + +def sza_check(data, sza_limit): + bad_inds = [] + for i in range(len(data)): + sza = data[" sza"][i] + if sza > sza_limit: + bad_index = i + bad_inds.append(bad_index) + else: + continue + + cleaned_data = data.drop(bad_inds) + cleaned_data.reset_index(drop=True, inplace=True) + return cleaned_data + + +def cutter(data, column, type, cut_value): + if type == "above": + clean_data = data[data[column] < cut_value] + + if type == "below": + clean_data = data[data[column] > cut_value] + + if type == "between": + clean_data = data[(data[column] > cut_value[0]) & (data[column] < cut_value[1])] + + percent = ((len(data) - len(clean_data)) / len(data)) * 100 + print("Checks remove {:.2f}% of data".format(percent)) + return clean_data + + +##cleaning + + +def run_checks( + data, + maintenance=False, + bad_dates=None, + sza=False, + sza_limit=None, + sza_cut_type=None, + vza=False, + vza_limit=None, + vza_cut_type=None, +): + clean_data = get_dates_times(data) + get_raa(clean_data) + if maintenance is True: + clean_data = maintenance_check(clean_data, bad_dates) + else: + pass + + if sza is True: + clean_data = cutter(clean_data, " sza", sza_cut_type, sza_limit) + else: + pass + + if vza is True: + clean_data = cutter(clean_data, " vza", vza_cut_type, vza_limit) + else: + pass + + percent = ((len(data) - len(clean_data)) / len(data)) * 100 + print("Checks remove {:.2f}% of data".format(percent)) + return clean_data + + +##cutting tool + + +##cut 1 dimension and plot +def cut_and_plot_1d(outliers, good_data, cut1, cut1_typ, cut1_ran, x_lab): + + var1 = np.arange(cut1_ran[0], cut1_ran[1], 1) + + out_cuts = [] + good_cuts = [] + diff = [] + + for i in range(len(var1)): + outliers_cut = cutter(outliers, cut1, cut1_typ, var1[i]) + good_cut = cutter(good_data, cut1, cut1_typ, var1[i]) + + o_percent = ((len(outliers) - len(outliers_cut)) / len(outliers)) * 100 + g_percent = ((len(good_data) - len(good_cut)) / len(good_data)) * 100 + + out_cuts.append(o_percent) + good_cuts.append(g_percent) + diff.append(o_percent - g_percent) + + plt.plot(var1, good_cuts, color="g", label="Good Data") + plt.plot(var1, out_cuts, color="r", label="Outliers") + plt.xlabel(x_lab) + plt.ylabel("Percent of Data Cut") + plt.axis([cut1_ran[0], cut1_ran[1], 0, 100]) + plt.grid(alpha=0.5, color="black") + plt.legend() + plt.show() + + +##cut 2 dimensions and plot a 2d colormap +def cut_and_plot_2d( + outliers, + good_data, + cut1, + cut1_typ, + cut1_ran, + cut2, + cut2_typ, + cut2_ran, + x_lab, + y_lab, +): + + var1 = np.arange(cut1_ran[0], cut1_ran[1], 1) + var2 = np.arange(cut2_ran[0], cut2_ran[1], 1) + + out_cuts = np.zeros((len(var1), len(var2))) + good_cuts = np.zeros((len(var1), len(var2))) + + for i in range(len(var1)): + outlier_cut = cutter(outliers, cut1, cut1_typ, var1[i]) + goo_cut = cutter(good_data, cut1, cut1_typ, var1[i]) + for j in range(len(var2)): + outliers_cut = cutter(outlier_cut, cut2, cut2_typ, var2[j]) + o_percent = ((len(outliers) - len(outliers_cut)) / len(outliers)) * 100 + + good_cut = cutter(goo_cut, cut2, cut2_typ, var2[j]) + g_percent = ((len(good_data) - len(good_cut)) / len(good_data)) * 100 + + out_cuts[i, j] = o_percent + good_cuts[i, j] = g_percent + + fig, axs = plt.subplots(1, 3, figsize=(6, 4), layout="constrained") + axs[0].set_xlabel(x_lab) + axs[0].set_ylabel(y_lab) + axs[1].set_xlabel(x_lab) + axs[1].set_ylabel(y_lab) + axs[0].set_title("Outliers") + axs[1].set_title("Good Data") + axs[2].set_xlabel(x_lab) + axs[2].set_ylabel(y_lab) + axs[2].set_title("Difference") + images = [] + norm = colors.Normalize( + vmin=np.min(np.append(out_cuts, good_cuts)), + vmax=np.max(np.append(out_cuts, good_cuts)), + ) + images.append( + axs[0].imshow( + out_cuts, + interpolation="nearest", + origin="lower", + extent=[cut2_ran[0], cut2_ran[1], cut1_ran[0], cut1_ran[1]], + norm=norm, + ) + ) + images.append( + axs[1].imshow( + good_cuts, + interpolation="nearest", + origin="lower", + extent=[cut2_ran[0], cut2_ran[1], cut1_ran[0], cut1_ran[1]], + norm=norm, + ) + ) + images.append( + axs[2].imshow( + out_cuts - good_cuts, + interpolation="nearest", + origin="lower", + extent=[cut2_ran[0], cut2_ran[1], cut1_ran[0], cut1_ran[1]], + norm=norm, + ) + ) + fig.colorbar(images[0], ax=axs, label="Percent of Data Cut") + plt.show() + + +##outputting results +def write_csv(data, filename): + data.to_csv(windows_results_path + filename) + + +def plot_data(filename, x, y, x_label=None, y_label=None, title=None): + fig, ax = plt.subplots(1, 1, figsize=(6, 4), dpi=150) + ax.plot(x, y) + ax.set_xlabel(x_label) + ax.set_ylabel(y_label) + ax.set_title(title) + fig.savefig(windows_results_path + filename) + + +# more_wav_data = pd.read_csv(windows_data_path + r'/GHNA_refl_2022_None_None_None_None.csv') +data = get_dates_times(data) +get_raa(data) +clean = run_checks( + data, + maintenance=False, + bad_dates=maintenance_dates, + sza=False, + sza_limit=70, + sza_cut_type="above", +) + +more_wav_raa = cutter(cutter(clean, "raa", "below", 10), "raa", "above", 350) +print(len(clean), len(data)) + +o, g = get_outliers(clean, 3) +print(len(o), len(g)) + +# write_csv(clean, '/JSIT_dates_times_raa.csv') +""" +write_csv(clean_more_wav, '/more_wav_prelim_checks_2024_GHNA.csv') +""" +# print(len(get_outliers(clean_more_wav, 3)[0]), len(get_outliers(clean_more_wav, 3)[1])) +##running cuts and plotting +""" +data = get_dates_times(data) +get_raa(data) +clean = run_checks(data, maintenance = True, bad_dates = maintenance_dates, + sza = True, sza_limit = 70, sza_cut_type = 'above') + +good_data = pd.read_csv(windows_data_path + r'/joe/good_tol01_cloud_checked_GHNA_2024.csv') +outliers = pd.read_csv(windows_data_path + r'/joe/outliers_tol01_cloud_checked_GHNA_2024.csv') + + +good_clean_19 = run_checks(good_data, maintenance = False, bad_dates = maintenance_dates, + sza = False, sza_limit = 20, sza_cut_type = 'below', + vza = True, vza_limit = 15, vza_cut_type = 'below') + +out_clean_19 = run_checks(outliers, maintenance = False, bad_dates = maintenance_dates, + sza = False, sza_limit = 10, sza_cut_type = 'below', + vza = True, vza_limit = 15, vza_cut_type = 'below') + + +clean_raa = cutter(cutter(clean, 'raa', 'below', 15), 'raa', 'above', 345) +print(len(clean), len(clean_raa)) +#clean_o_raa = cutter(cutter(outliers, 'raa', 'below', 5), 'raa', 'above', 355) +#print(len(outliers), len(clean_o_raa)) + +#write_csv(clean, '/maintenance_sza70_2024_GHNA.csv') + +#cut_and_plot_1d(outliers, good_data, ' vza', 'below', (0,30), 'VZA Cut') + +#cut_and_plot_2d(outliers, good_data, 'raa', 'below', (0,20), 'raa', 'above', (340,360), 'RAA Cut Above', 'RAA Cut Below') +""" + +# outliers = pd.read_csv(r'T:\ECO\EOServer\data\insitu\hypernets\post_processing_qc\joe\GHNA_2022_outliers.csv') +# good_data = pd.read_csv(r'T:\ECO\EOServer\data\insitu\hypernets\post_processing_qc\joe\GHNA_2022_good.csv') + + +""" +saa_and_sza_good = good_data[(good_data[' saa'] > 70) + & (good_data[' saa'] < 85) + & (good_data[' vaa'] > 260) + & (good_data[' vaa'] < 300) + & (good_data[' sza'] > 60) + & (good_data[' sza'] < 70)] + +print(len(good_data), len(saa_and_sza_good)) + +saa_and_sza_out = outliers[(outliers[' saa'] > 70) + & (outliers[' saa'] < 85) + & (outliers[' vaa'] > 260) + & (outliers[' vaa'] < 300) + & (outliers[' sza'] > 60) + & (outliers[' sza'] < 70)] + +print(len(outliers), len(saa_and_sza_out)) +""" +""" +saa_and_sza_good = good_data[(good_data[' saa'] > 255) + & (good_data[' saa'] < 265) + & (good_data[' vaa'] > 290) + & (good_data[' sza'] > 65) + & (good_data[' sza'] < 70)] + +print(len(good_data), len(saa_and_sza_good)) + +saa_and_sza_out = outliers[(outliers[' saa'] > 255) + & (outliers[' saa'] < 265) + & (outliers[' vaa'] > 290) + & (outliers[' sza'] > 65) + & (outliers[' sza'] < 70)] + +print(len(outliers), len(saa_and_sza_out)) + +saa_and_vza_good = good_data[(good_data[' saa'] < 180) + & (good_data[' vza'] < 1)] + +print(len(good_data), len(saa_and_vza_good)) + +saa_and_vza_out = outliers[(outliers[' saa'] < 180) + & (outliers[' vza'] < 1)] + +print(len(outliers), len(saa_and_vza_out)) + +saa_sza_vza_good = good_data[(good_data[' saa'] < 180) + & (good_data[' vza'] < 8) + & (good_data[' vza'] > 4) + & (good_data[' sza'] > 55)] + +print(len(good_data), len(saa_sza_vza_good)) + +saa_sza_vza_out = outliers[(outliers[' saa'] < 180) + & (outliers[' vza'] < 8) + & (outliers[' vza'] > 4) + & (outliers[' sza'] > 55)] + +print(len(outliers), len(saa_sza_vza_out)) +""" diff --git a/hypernets_processor/post_processing/misalignment_calculator.py b/hypernets_processor/post_processing/misalignment_calculator.py new file mode 100644 index 00000000..1c2773a4 --- /dev/null +++ b/hypernets_processor/post_processing/misalignment_calculator.py @@ -0,0 +1,81 @@ +import numpy as np +import pandas as pd +import matplotlib.pyplot as plt +from scipy.optimize import curve_fit +import xarray as xr +from scipy.stats import chisquare +from scipy.stats import binned_statistic +import datetime +from curepy import MCMCRetrieval, plot_trace, plot_corner +import os +import matplotlib + +matplotlib.use("Qt5Agg") + + +def ratio_calculator(vza, vaa, sza, saa, direct_to_diffuse): + if vaa < 0: + vza = -vza + vaa += 180 + + if sza.shape == (): + if saa < 0: + sza = -sza + saa += 180 + else: + sza[saa < 0] = -sza[saa < 0] + saa[saa < 0] += 180 + + sza = np.radians(sza) + saa = np.radians(saa) + vza = np.radians(vza) + vaa = np.radians(vaa) + # new_sza = np.cos(sza + vza*np.cos((saa - vaa + 360) % 360)) + new_sza = np.arccos( + np.cos(sza) * np.cos(vza) + np.sin(sza) * np.sin(vza) * np.cos((saa - vaa)) + ) + new_direct_to_diffuse = direct_to_diffuse * np.cos(new_sza) / np.cos(sza) + return (new_direct_to_diffuse + 1) / (direct_to_diffuse + 1) + + +class MisalignmentModel: + + def __init__(self, data_path, sza_data, saa_data, ratio_rand_unc, ratio_syst_unc): + + data = pd.read_csv(data_path, delimiter=",") + data.drop("Unnamed: 0", axis="columns") + + self.wavelengths = [] + ratio_dict = {} + dir_diff_dict = {} + + for key in data.keys(): + if "model" in key: + wv = "".join(filter(str.isdigit, key)) + idx = np.argwhere(data.keys() == key)[0] + model_key = data.keys()[idx] + obs_key = data.keys()[idx + 1] + dir_diff_ratio_key = data.keys()[idx + 2] + dir_diff_dict[wv] = data[dir_diff_ratio_key].values.reshape(-1) + ratio = data[model_key].values / data[obs_key].values + ratio_dict[wv] = ratio.reshape(-1) + self.wavelengths.append(wv) + + self.dataset = xr.Dataset( + data_vars=dict( + sza=(["date"], sza_data), + saa=(["date"], saa_data), + dir_diff_ratio=(["wv", "date"], list(dir_diff_dict.values())), + ratio=(["wv", "date"], list(ratio_dict.values())), + ), + coords=dict( + date=( + "date", + [ + datetime.datetime.strptime(str(x), "%Y%m%d") + for x in data["Date"] + ], + ), + wv=("wv", list(ratio_dict.keys())), + ), + ) diff --git a/hypernets_processor/post_processing/misalignment_model.py b/hypernets_processor/post_processing/misalignment_model.py new file mode 100644 index 00000000..0667bbca --- /dev/null +++ b/hypernets_processor/post_processing/misalignment_model.py @@ -0,0 +1,1937 @@ +import numpy as np +import pandas as pd +import matplotlib.pyplot as plt +from numpy.random import normal +from scipy.optimize import curve_fit +import xarray as xr +from scipy.stats import chisquare +from scipy.stats import binned_statistic, binned_statistic_2d +import datetime +from curepy import MCMCRetrieval, plot_trace, plot_corner +import os +import matplotlib + +matplotlib.use("Qt5Agg") + +results_path = r"T:\ECO\EOServer\joe\hypernets_plots\misalignment" +# read in data +data = pd.read_csv( + r"T:\ECO\EOServer\data\insitu\hypernets\post_processing_qc\joe\irradiance_GHNA_v3_analysis.csv", + delimiter=",", +) +data.drop("Unnamed: 0", axis="columns") + +site = 'LOBE_2025Apr_2025May' +site_file = 'LOBEv4' +data_v1 = pd.read_csv(f'T:\ECO\EOServer\data\insitu\hypernets\post_processing_qc\joe\irradiance_{site_file}_analysis.csv', delimiter = ',') +data_v1.drop('Unnamed: 0', axis = 'columns') + +data_v1 = data_v1.sort_values(by="Date") + +# cloud check and flag removal +data = data[data["model_550nm"] * 0.9 < data["obs_550nm"]] +data = data[data["model_550nm"] * 1.1 > data["obs_550nm"]] +data = data[data["Flag"] == 0] + +# format data +sza_measured = data["SZA"].values +saa_measured = data["SAA"].values +wavelengths = [] + +for i in range(len(saa_measured)): + if saa_measured[i] > 180: + saa_measured[i] = saa_measured[i] - 360 + + +ratio_dict = {} +dir_diff_dict = {} +for key in data.keys(): + if "model" in key: + wv = "".join(filter(str.isdigit, key)) + idx = np.argwhere(data.keys() == key)[0] + model_key = data.keys()[idx] + obs_key = data.keys()[idx + 1] + dir_diff_ratio_key = data.keys()[idx + 2] + dir_diff_dict[wv] = data[dir_diff_ratio_key].values.reshape(-1) + ratio = data[model_key].values / data[obs_key].values + ratio_dict[wv] = ratio.reshape(-1) + wavelengths.append(wv) + +data = xr.Dataset( + data_vars=dict( + sza=(["date"], sza_measured), + saa=(["date"], saa_measured), + dir_diff_ratio=(["wv", "date"], list(dir_diff_dict.values())), + ratio=(["wv", "date"], list(ratio_dict.values())), + time=(["date"], [str(x).zfill(4) for x in data["Time"]]), + ), + coords=dict( + date=( + "date", + [datetime.datetime.strptime(str(x), "%Y%m%d") for x in data["Date"]], + ), + wv=("wv", list(ratio_dict.keys())), + ), +) + +data_before_may24 = data.sel(date=slice("2023-10-01", "2024-05-23")) +data_after_may24 = data.sel(date=slice("2024-05-24", "2025-01-01")) + + +# cloud check and flag removal +data_v1 = data_v1[data_v1["model_550nm"] * 0.9 < data_v1["obs_550nm"]] +data_v1 = data_v1[data_v1["model_550nm"] * 1.2 > data_v1["obs_550nm"]] +data_v1 = data_v1[data_v1["Flag"] == 0] +data_v1 = data_v1[data_v1["Time"] > 830] + +#cloud check and flag removal +data_v1 = data_v1[data_v1['model_550nm']*0.9 < data_v1['obs_550nm']] +data_v1 = data_v1[data_v1['model_550nm']*1.1 > data_v1['obs_550nm']] +data_v1 = data_v1[data_v1['Flag'] == 0] +#if site == 'LOBEv4' or site == 'LOBEv2': + # data_v1 = data_v1[data_v1['SZA'] < 50] +if site == 'WWUK_May22_Oct22': + data_v1 = data_v1[data_v1['SZA'] < 50] + +if 'GHNA' in site: + data_v1 = data_v1[data_v1['Time'] > 830] +#if site == 'LOBEv2': + # data_v1 = data_v1[data_v1['Time'] > 800] + +#format data_v1 +sza_measured_v1 = data_v1['SZA'].values +saa_measured_v1 = data_v1['SAA'].values +wavelengths = [] + +for i in range(len(saa_measured_v1)): + if saa_measured_v1[i] > 180: + saa_measured_v1[i] = saa_measured_v1[i] - 360 + + +ratio_dict = {} +dir_diff_dict = {} +for key in data_v1.keys(): + if "model" in key: + wv = "".join(filter(str.isdigit, key)) + idx = np.argwhere(data_v1.keys() == key)[0] + model_key = data_v1.keys()[idx] + obs_key = data_v1.keys()[idx + 1] + dir_diff_ratio_key = data_v1.keys()[idx + 2] + dir_diff_dict[wv] = data_v1[dir_diff_ratio_key].values.reshape(-1) + ratio = data_v1[model_key].values / data_v1[obs_key].values + ratio_dict[wv] = ratio.reshape(-1) + wavelengths.append(wv) + +data_v1 = xr.Dataset( + data_vars=dict( + sza=(["date"], sza_measured_v1), + saa=(["date"], saa_measured_v1), + dir_diff_ratio=(["wv", "date"], list(dir_diff_dict.values())), + ratio=(["wv", "date"], list(ratio_dict.values())), + time=(["date"], [str(x).zfill(4) for x in data_v1["Time"]]), + ), + coords=dict( + date=( + "date", + [datetime.datetime.strptime(str(x), "%Y%m%d") for x in data_v1["Date"]], + ), + wv=("wv", list(ratio_dict.keys())), + ), +) + +if site == 'LOBEv4': + data_v1 = data_v1.sel(date = slice('2025-06-03', '2025-09-02')) +if site == 'LOBE_2025Apr_2025May': + data_v1 = data_v1.sel(date = slice('2025-04-03', '2025-05-03')) +if site == 'WWUK_May22_Oct22': + data_v1 = data_v1.sel(date = slice('2022-05-05', '2022-10-11')) +if site == 'WWUK_Apr23_Nov23': + data_v1 = data_v1.sel(date = slice('2023-04-29', '2023-11-01')) + + + +def ratio_calculator(vza, vaa, sza, saa, direct_to_diffuse=1000): + """ + This ratio is observed as the measured irradiance/ divided by the clear sky model. + I.e. equal to (direct_measured+diffuse_measured)/(direct_model+diffuse_model) + diffuse irradiance is assumed not to depend on vza and diffuse_model_vza=diffuse_model. We can divide both sides by this diffuse contribution: + (direct_model_vza/diffuse_model+1)*offset/(direct_model/diffuse_model+1) + The direct_model_vza is proportional to cosine of vza. + """ + if vza < 0: + vza = -vza + vaa += -180 + + sza = np.radians(sza) + saa = np.radians(saa) + vza = np.radians(vza) + vaa = np.radians(vaa) + # new_sza = np.cos(sza + vza*np.cos((saa - vaa + 360) % 360)) + new_sza = np.arccos( + np.cos(sza) * np.cos(vza) + np.sin(sza) * np.sin(vza) * np.cos((saa - vaa)) + ) + new_direct_to_diffuse = direct_to_diffuse * np.cos(new_sza) / np.cos(sza) + return (new_direct_to_diffuse + 1) / (direct_to_diffuse + 1) + + +def ratio_calculator_offset(vza, vaa, offset, sza, saa, direct_to_diffuse=1000): + if vza < 0: + vza = -vza + vaa += -180 + + sza = np.radians(sza) + saa = np.radians(saa) + vza = np.radians(vza) + vaa = np.radians(vaa) + # new_sza = np.cos(sza + vza*np.cos((saa - vaa + 360) % 360)) + new_sza = np.arccos( + np.cos(sza) * np.cos(vza) + np.sin(sza) * np.sin(vza) * np.cos((saa - vaa)) + ) + new_direct_to_diffuse = direct_to_diffuse * np.cos(new_sza) / np.cos(sza) + return (new_direct_to_diffuse + 1) / (direct_to_diffuse + 1) + offset + + +def ratio_calculator_10offsets( + vza, + vaa, + offset, + offset2, + offset3, + offset4, + offset5, + offset6, + offset7, + offset8, + offset9, + offset10, + offset11, + offset12, + sza, + saa, + direct_to_diffuse=1000, +): + if vza < 0: + vza = -vza + vaa += -180 + + sza = np.radians(sza) + saa = np.radians(saa) + vza = np.radians(vza) + vaa = np.radians(vaa) + # new_sza = np.cos(sza + vza*np.cos((saa - vaa + 360) % 360)) + new_sza = np.arccos( + np.cos(sza) * np.cos(vza) + np.sin(sza) * np.sin(vza) * np.cos((saa - vaa)) + ) + new_direct_to_diffuse = direct_to_diffuse * np.cos(new_sza) / np.cos(sza) + offsets = np.array( + [ + offset, + offset2, + offset3, + offset4, + offset5, + offset6, + offset7, + offset8, + offset9, + offset10, + offset11, + offset12, + ] + ) + return (new_direct_to_diffuse + 1) / (direct_to_diffuse + 1) + offsets[:, None] + + +def ratio_calculator_only_offsets( + offset, + offset2, + offset3, + offset4, + offset5, + offset6, + offset7, + offset8, + offset9, + offset10, + offset11, + offset12, + sza, + saa, + direct_to_diffuse, +): + vza = 0 + vaa = 0 + sza = np.radians(sza) + saa = np.radians(saa) + + # new_sza = np.cos(sza + vza*np.cos((saa - vaa + 360) % 360)) + new_sza = np.arccos( + np.cos(sza) * np.cos(vza) + np.sin(sza) * np.sin(vza) * np.cos((saa - vaa)) + ) + new_direct_to_diffuse = direct_to_diffuse * np.cos(new_sza) / np.cos(sza) + offsets = np.array( + [ + offset, + offset2, + offset3, + offset4, + offset5, + offset6, + offset7, + offset8, + offset9, + offset10, + offset11, + offset12, + ] + ) + return (new_direct_to_diffuse + 1) / (direct_to_diffuse + 1) + offsets[:, None] + + +def ratio_uncertainty_calculator(vza, vaa, sza, saa, u_vza, u_vaa): + if vza < 0: + vza = -vza + vaa += -180 + + sza = np.radians(sza) + saa = np.radians(saa) + vza = np.radians(vza) + vaa = np.radians(vaa) + + unc = np.sqrt( + (np.sin(sza) * np.sin(vza) * np.sin(saa - vaa) * u_vaa) ** 2 + + ( + (-np.cos(sza) * np.sin(sza) + np.sin(sza) * np.cos(vza) * np.cos(saa - vaa)) + * u_vza + ) + ** 2 + ) + new_sza = np.cos(sza) * np.cos(vza) + np.sin(sza) * np.sin(vza) * np.cos( + (saa - vaa) + ) + return unc / new_sza * 100 + + +def convergence_test(d, window_length, threshold): + series = pd.Series(d) + std = list(series.rolling(window_length).std())[::-1] + turning_point = [i for i, v in enumerate(std) if v > threshold][0] + return turning_point + + +def quad_plane(xy, a, b, c, d, e, f): + x, y = xy + return a + b * x + c * y + d * x**2 + e * y**2 + f * x * y + + +def plane_fit(input_sza, input_saa, ratio): + x = input_sza + y = input_saa + z = ratio + + popt, pcov = curve_fit(quad_plane, (x, y), z, nan_policy="omit") + + return popt, pcov + + +def plane_plotter(ax, input_sza, input_saa, ratio, color, dot_color): + coeff, cov = plane_fit(input_sza, input_saa, ratio) + coeff = [round(x, 9) for x in coeff] + sza, saa = np.meshgrid( + np.linspace(np.min(input_sza), np.max(input_sza), 1000), + np.linspace(np.min(input_saa), np.max(input_saa), 1000), + ) + plane = quad_plane((sza, saa), *coeff) + + # fig = plt.figure() + # ax = fig.add_subplot(111, projection = '3d') + ax = ax + ax.scatter(input_sza, input_saa, ratio, marker="x", alpha=0.7, s=2, color=dot_color) + ax.plot_surface(sza, saa, plane, alpha=0.5, color=color) + ax.set_xlabel("SZA") + ax.set_ylabel("SAA") + ax.set_zlabel("Model to Measurement Ratio") + + +def comparison_plotter(sza, saa, vza, vaa, ratio): + + sza_range = np.linspace(np.min(sza), np.max(sza), 100) + saa_range = np.linspace(np.min(saa), np.max(saa), 100) + + sza_mesh, saa_mesh = np.meshgrid(sza_range, saa_range) + + vzas = vza * np.ones(sza_mesh.shape) + vaas = vaa * np.ones(saa_mesh.shape) + + fig = plt.figure() + ax = fig.add_subplot(111, projection="3d") + plane_plotter( + ax, + sza_mesh, + saa_mesh, + ratio_calculator(vzas, vaas, sza_mesh, saa_mesh), + "green", + "black", + ) + plane_plotter(ax, sza, saa, ratio, "red", "blue") + plt.show() + + +def chi_square_minimiser(sza, saa, measured_ratio): + chi = 10000000 + vza = 999 + vaa = 999 + max_sza = max(sza) + min_sza = min(sza) + max_saa = max(saa) + min_saa = min(saa) + + sza_range = np.linspace(min_sza, max_sza, 10) + saa_range = np.linspace(min_saa, max_saa, 10) + + sza_mesh, saa_mesh = np.meshgrid(sza_range, saa_range) + coeff, cov = plane_fit(sza, saa, measured_ratio) + coeff = [round(x, 9) for x in coeff] + + measured_fit = quad_plane((sza_mesh, saa_mesh), *coeff) + for i in np.arange(0, 5, 0.1): + for j in range(-180, 180): + vzas = i * np.ones(sza.shape) + vaas = j * np.ones(saa.shape) + calculated_fit = ratio_calculator(vzas, vaas, sza, saa) + + # new_chi = chisquare(measured_fit, (np.sum(measured_fit)/np.sum(calculated_fit))*calculated_fit).statistic + new_chi = np.sum((measured_ratio - calculated_fit) ** 2 / calculated_fit) + if new_chi < chi: + chi = new_chi + vza = i + vaa = j + + best_fit = ratio_calculator( + vza * np.ones(sza_mesh.shape), vaa * np.ones(saa_mesh.shape), sza_mesh, saa_mesh + ) + + fig = plt.figure() + ax = fig.add_subplot(111, projection="3d") + ax.scatter(sza, saa, measured_ratio, marker="x", alpha=0.7, s=2) + ax.plot_surface(sza_mesh, saa_mesh, best_fit, alpha=0.5, color="green") + ax.plot_surface(sza_mesh, saa_mesh, measured_fit, alpha=0.5, color="red") + plt.show() + + print(chi, vza, vaa, np.size(sza_mesh) - 6) + return chi, vza, vaa, np.size(sza_mesh) - 6 + + +def chi_square_minimiser_mesh(sza, saa, measured_ratio): + chi = 10000000 + vza = 999 + vaa = 999 + max_sza = max(sza) + min_sza = min(sza) + max_saa = max(saa) + min_saa = min(saa) + + sza_range = np.linspace(min_sza, max_sza, 10) + saa_range = np.linspace(min_saa, max_saa, 10) + + sza_mesh, saa_mesh = np.meshgrid(sza_range, saa_range) + coeff, cov = plane_fit(sza, saa, measured_ratio) + coeff = [round(x, 9) for x in coeff] + + measured_fit = quad_plane((sza_mesh, saa_mesh), *coeff) + best_fit = np.ones(sza_mesh.shape) + for i in np.arange(0, 10, 0.1): + for j in range(0, 360): + vzas = i * np.ones(sza_mesh.shape) + vaas = j * np.ones(saa_mesh.shape) + calculated_fit = ratio_calculator(vzas, vaas, sza_mesh, saa_mesh) + + # new_chi = chisquare(measured_fit, (np.sum(measured_fit)/np.sum(calculated_fit))*calculated_fit).statistic + new_chi = np.sum((measured_fit - calculated_fit) ** 2 / calculated_fit) + if new_chi < chi: + chi = new_chi + vza = i + vaa = j + best_fit = calculated_fit + + fig = plt.figure() + ax = fig.add_subplot(111, projection="3d") + ax.scatter(sza, saa, measured_ratio, marker="x", alpha=0.7, s=2) + ax.plot_surface(sza_mesh, saa_mesh, best_fit, alpha=0.5, color="green") + ax.plot_surface(sza_mesh, saa_mesh, measured_fit, alpha=0.5, color="red") + plt.show() + plt.close() + + print(chi, vza, vaa, np.size(sza_mesh) - 6) + return chi, vza, vaa, np.size(sza_mesh) - 6 + + +def chi_square_minimiser_MCMC(sza, saa, dir_diff_ratio, measured_ratio, plot_name=""): + + mcmc = MCMCRetrieval( + ratio_calculator, + measured_ratio, + rand_uncertainty=0.03, + syst_uncertainty=0.03, + initial_guess=[2, -40], + downlims=[-90, -180], + uplims=[90, 180], + n_input=2, + b=[sza, saa, dir_diff_ratio], + u_b=[0, 0, 0], + b_iter=1, + circular=True, + ) + mean, unc, corr, samples = mcmc.run_retrieval( + 100, 1000, 100, return_corr=True, return_samples=True + ) + chisq = mcmc.find_chisum(mean) + print(len(sza), chisq) + print(mean, unc) + + plot_corner( + samples, + os.path.join(results_path, "plot_corner_%s.png" % plot_name), + labels=["vza", "vaa", "offset"], + ) + plot_trace( + samples, labels=["vza", "vaa", "offset"], path=results_path, tag=plot_name + ) + + return mean, unc, chisq / (len(sza) - 2) + + +def chi_square_minimiser_MCMC_offset( + sza, saa, dir_diff_ratio, measured_ratio, plot_name="" +): + + mcmc = MCMCRetrieval( + ratio_calculator_offset, + measured_ratio, + rand_uncertainty=0.03, + syst_uncertainty=0.03, + initial_guess=[2, -40, 0.01], + downlims=[-90, -180, -1], + uplims=[90, 180, 1], + n_input=3, + b=[sza, saa, dir_diff_ratio], + u_b=[0, 0, 0], + b_iter=1, + circular=True, + ) + mean, unc, corr, samples = mcmc.run_retrieval( + 1000, 1000, 100, return_corr=True, return_samples=True + ) + chisq = mcmc.find_chisum(mean) + print(len(sza), chisq) + print(mean, unc) + + plot_corner( + samples, + os.path.join(results_path, "plot_corner_%s.png" % plot_name), + labels=["vza", "vaa", "offset"], + ) + plot_trace( + samples, labels=["vza", "vaa", "offset"], path=results_path, tag=plot_name + ) + + return mean, unc + + +def chi_square_minimiser_MCMC_10offsets( + sza, saa, dir_diff_ratio, measured_ratio, plot_name="" +): + mcmc = MCMCRetrieval( + ratio_calculator_10offsets, + measured_ratio, + rand_uncertainty=0.05, + syst_uncertainty=0.05, + initial_guess=[ + 2, + -40, + 0.01, + 0.01, + 0.01, + 0.01, + 0.01, + 0.01, + 0.01, + 0.01, + 0.01, + 0.01, + 0.01, + 0.01, + ], + downlims=[-90, -360, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], + uplims=[90, 360, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], + # initial_guess=[2, -40, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], + # downlims=[-90, -180, 0.01, 0.01, 0.01, 0.01, 0.01, 0.01, 0.01, 0.01, 0.01, 0.01], + # uplims=[90, 180, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2], + n_input=14, + b=[sza, saa, dir_diff_ratio], + u_b=[0, 0, 0], + b_iter=1, + circular=True, + ) + mean, unc, corr, samples = mcmc.run_retrieval( + 100, 10000, 1000, return_corr=True, return_samples=True + ) + chisq = mcmc.find_chisum(mean) + print("CHI", len(sza) - 14, chisq, chisq / (len(sza) - 14)) + + # plot_corner( + # samples, + # os.path.join(results_path, "plot_corner_%s.png" % plot_name), + # labels=['vza','vaa', 'offset', 'offset2', 'offset3', 'offset4', 'offset5', 'offset6', 'offset7', 'offset8', 'offset9', 'offset10', 'offset11', 'offset12'], + # ) + # plot_trace( + # samples, + # labels=['vza','vaa', 'offset', 'offset2', 'offset3', 'offset4', 'offset5', 'offset6', 'offset7', 'offset8', 'offset9', 'offset10'], + # path=results_path, + # tag = plot_name + # ) + + return mean, unc, chisq / (len(sza) - 14) + + +def chi_square_minimiser_MCMC_only_offsets( + sza, saa, dir_diff_ratio, measured_ratio, plot_name="" +): + mcmc = MCMCRetrieval( + ratio_calculator_only_offsets, + measured_ratio, + rand_uncertainty=0.05, + syst_uncertainty=0.05, + initial_guess=[ + 0.01, + 0.01, + 0.01, + 0.01, + 0.01, + 0.01, + 0.01, + 0.01, + 0.01, + 0.01, + 0.01, + 0.01, + ], + downlims=[-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], + uplims=[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], + # initial_guess=[2, -40, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], + # downlims=[-90, -180, 0.01, 0.01, 0.01, 0.01, 0.01, 0.01, 0.01, 0.01, 0.01, 0.01], + # uplims=[90, 180, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2], + n_input=12, + b=[sza, saa, dir_diff_ratio], + u_b=[0, 0, 0], + b_iter=1, + circular=True, + ) + mean, unc, corr, samples = mcmc.run_retrieval( + 100, 10000, 1000, return_corr=True, return_samples=True + ) + chisq = mcmc.find_chisum(mean) + print("Only Offsets CHI", len(sza) - 10, chisq, chisq / (len(sza) - 10)) + + return mean, unc, chisq / (len(sza) - 10) + + +def plot_misalign_change(dataset): + ma_dict = {"vza": {}, "unc_vza": {}, "num": {}, "sza": {}, "saa": {}} + earliest = min(dataset.date.values) + recent = max(dataset.date.values) + earliest_year = np.datetime64(earliest, "Y").astype(int) + 1970 + recent_year = np.datetime64(recent, "Y").astype(int) + 1970 + for y in range(earliest_year, recent_year + 1): + for m in range(0, 12): + m = str(m).zfill(2) + try: + ds_month = dataset.loc[dict(date=f"{y}-{m}")] + mean, unc = chi_square_minimiser_MCMC_10offsets( + ds_month.sza.values, + ds_month.saa.values, + ds_month.dir_diff_ratio.values, + np.array(list(ds_month.ratio.values)), + ) + # comparison_plotter(ds_month.sza.values, ds_month.saa.values, vza, vaa, list(ds_month.ratio.values)) + ma_dict["vza"][f"{y}-{m}"] = mean[0] + ma_dict["unc_vza"][f"{y}-{m}"] = unc[0] + ma_dict["num"][f"{y}-{m}"] = len(ds_month.sza.values) + ma_dict["sza"][f"{y}-{m}"] = ds_month.sza.values + ma_dict["saa"][f"{y}-{m}"] = ds_month.saa.values + + except: + ma_dict[f"{y}-{m}"] = np.nan + + fig, axs = plt.subplots(4, 1) + axs[0].errorbar( + ma_dict["vza"].keys(), + list(ma_dict["vza"].values()), + yerr=list(ma_dict["unc_vza"].values()), + marker="", + linestyle="", + capsize=5, + ) + axs[1].bar(ma_dict["vza"].keys(), list(ma_dict["num"].values())) + axs[2].violinplot(ma_dict["sza"].values()) + axs[3].violinplot(ma_dict["saa"].values()) + + axs[0].set_ylabel("VZA") + axs[1].set_ylabel("Number of Measurements") + axs[2].set_ylabel("SZA") + axs[3].set_ylabel("SAA") + plt.show() + + +def plot_misalign_sza(dataset, wav, ret=False): + ma_dict = { + "vza": {}, + "unc_vza": {}, + "vaa": {}, + "ratio": {}, + "unc_ratio": {}, + "unc_vaa": {}, + "num": {}, + "sza": {}, + "saa": {}, + "obs_ratio": {}, + } + # sza_bounds = [0,7.5,12.5,17.5,22.5,27.5,32.5,37.5,42.5,47.5,52.5,57.5,62.5,67.5] + val, bins = pd.qcut(dataset.sza, 10, retbins=True) + bins.round(1) + bins[0] = 0 + sza_bounds = bins + # sza_bounds = [0,20,40,60,80] + ratio_0 = 999 + for i in range(len(sza_bounds) - 1): + sza_min = sza_bounds[i] + sza_max = sza_bounds[i + 1] + sza_center = np.mean([sza_min, sza_max]).round(1) + + ds = ( + dataset.loc[dict(wv=wav)] + .set_coords("sza") + .where((dataset.sza < sza_max) & (dataset.sza > sza_min)) + ) + ds = ds.dropna(dim="date") + try: + mean, unc = chi_square_minimiser_MCMC( + ds.sza.values, + ds.saa.values, + ds.dir_diff_ratio.values, + np.array(list(ds.ratio.values)), + wav + f"_{sza_center}", + ) + + mean = list(mean) + unc = list(unc) + if mean[0] < 0: + mean[0] = -mean[0] + mean[1] += -180 + + ma_dict["vza"][f"{sza_center}"] = mean[0] + ma_dict["unc_vza"][f"{sza_center}"] = unc[0] + ma_dict["vaa"][f"{sza_center}"] = mean[1] + ma_dict["unc_vaa"][f"{sza_center}"] = unc[1] + ma_dict["num"][f"{sza_center}"] = len(ds.sza.values) + ma_dict["sza"][f"{sza_center}"] = ds.sza.values + ma_dict["saa"][f"{sza_center}"] = ds.saa.values + + ratio_grid = ratio_calculator_offset( + mean[0], + mean[1], + mean[2], + sza_center, + ds.saa.values, + ds.dir_diff_ratio.values, + ) + ratio_unc = ( + ratio_uncertainty_calculator( + mean[0], mean[1], sza_center, ds.saa.values, unc[0], unc[1] + ) + * ratio_grid + / 100 + ) + if ratio_0 == 999: + ratio_0 = np.mean(ratio_grid) + + ma_dict["ratio"][f"{sza_center}"] = np.mean(ratio_grid) # /ratio_0 + ma_dict["unc_ratio"][f"{sza_center}"] = np.sqrt( + (np.std(ratio_grid) ** 2 + np.sum(ratio_unc**2) / len(ratio_unc) ** 2) + ) + ma_dict["obs_ratio"][f"{sza_center}"] = np.mean( + np.array(list(ds.ratio.values)) + ) + + except: + ma_dict[f"{sza_center}"] = np.nan + + fig, axs = plt.subplots(4, 1, sharex=True) + axs[0].errorbar( + ma_dict["vza"].keys(), + list(ma_dict["vza"].values()), + yerr=list(ma_dict["unc_vza"].values()), + marker="", + linestyle="", + capsize=5, + ) + axs[1].errorbar( + ma_dict["vza"].keys(), + list(ma_dict["vaa"].values()), + yerr=list(ma_dict["unc_vaa"].values()), + marker="", + linestyle="", + capsize=5, + ) + axs[2].errorbar( + ma_dict["vza"].keys(), + list(ma_dict["ratio"].values()), + yerr=list(ma_dict["unc_ratio"].values()), + marker="x", + linestyle="", + capsize=5, + ) + axs[2].scatter( + ma_dict["vza"].keys(), list(ma_dict["obs_ratio"].values()), marker="o" + ) + axs[2].hlines(y=1, color="black", alpha=0.5, xmin=0, xmax=15) + axs[3].bar(ma_dict["vza"].keys(), list(ma_dict["num"].values())) + + axs[0].grid(color="black", alpha=0.5, axis="y") + axs[1].grid(color="black", alpha=0.5, axis="y") + + axs[2].set_ylim(0.9, 1.1) + + axs[0].set_ylabel("VZA") + axs[1].set_ylabel("VAA") + axs[2].set_ylabel("Ratio") + axs[3].set_ylabel("Number of \nMeasurements") + fig.suptitle(wav) + fig.tight_layout() + fig.savefig(os.path.join(results_path, f"sza_plot_{wav}_after.png")) + + if ret: + return ma_dict + + +def plot_misalign_sza_all_wavs(dataset): + ma_dict = {"vza": {}, "unc_vza": {}, "num": {}, "sza": {}, "saa": {}} + # sza_bounds = [0,7.5,12.5,17.5,22.5,27.5,32.5,37.5,42.5,47.5,52.5,57.5,62.5,67.5] + sza_bounds = [0, 20, 40, 60, 80] + fig1, ax1 = plt.subplots(1, 1) + fig2, ax2 = plt.subplots(4, 1, sharey=True) + for wav in dataset.wv.values: + + for i in range(len(sza_bounds) - 1): + sza_min = sza_bounds[i] + sza_max = sza_bounds[i + 1] + sza_center = np.mean([sza_min, sza_max]) + + ds = ( + dataset.loc[dict(wv=wav)] + .set_coords("sza") + .where((dataset.sza < sza_max) & (dataset.sza > sza_min)) + ) + ds = ds.dropna(dim="date") + try: + mean, unc = chi_square_minimiser_MCMC( + ds.sza.values, + ds.saa.values, + ds.dir_diff_ratio.values, + np.array(list(ds.ratio.values)), + ) + + ma_dict["vza"][f"{sza_center}"] = mean[0] + ma_dict["unc_vza"][f"{sza_center}"] = unc[0] + ma_dict["num"][f"{sza_center}"] = len(ds.sza.values) + ma_dict["sza"][f"{sza_center}"] = ds.sza.values + ma_dict["saa"][f"{sza_center}"] = ds.saa.values + + except: + ma_dict[f"{sza_center}"] = np.nan + + ax2[i].errorbar( + wav, mean[0], yerr=unc[0], marker="", linestyle="", capsize=5, label=wav + ) + ax2[i].set_ylabel("VZA") + ax2[i].set_title(f"{sza_center}") + ax2[i].set_ylim(0, 10) + + ax1.errorbar( + ma_dict["vza"].keys(), + list(ma_dict["vza"].values()), + yerr=list(ma_dict["unc_vza"].values()), + marker="", + linestyle="", + capsize=5, + label=wav, + ) + + ax1.set_ylabel("VZA") + ax1.legend() + + fig2.tight_layout() + plt.show() + + +def plot_misalign_saa(dataset, wav): + ma_dict = {"vza": {}, "unc_vza": {}, "num": {}, "sza": {}, "saa": {}} + saa_bounds = [ + -180, + -160, + -140, + -120, + -100, + -80, + -60, + -40, + -20, + 0, + 20, + 40, + 60, + 80, + 100, + 120, + 140, + 160, + 180, + ] + for i in range(len(saa_bounds) - 1): + saa_min = saa_bounds[i] + saa_max = saa_bounds[i + 1] + saa_center = np.mean([saa_min, saa_max]) + + ds = ( + dataset.loc[dict(wv=wav)] + .set_coords("saa") + .where((dataset.saa < saa_max) & (dataset.saa > saa_min)) + ) + ds = ds.dropna(dim="date") + try: + mean, unc = chi_square_minimiser_MCMC( + ds.sza.values, + ds.saa.values, + ds.dir_diff_ratio.values, + np.array(list(ds.ratio.values)), + ) + + ma_dict["vza"][f"{saa_center}"] = mean[0] + ma_dict["unc_vza"][f"{saa_center}"] = unc[0] + ma_dict["num"][f"{saa_center}"] = len(ds.sza.values) + ma_dict["sza"][f"{saa_center}"] = ds.sza.values + ma_dict["saa"][f"{saa_center}"] = ds.saa.values + + except: + ma_dict[f"{saa_center}"] = np.nan + + for key in ma_dict["vza"].keys(): + if ma_dict["num"][key] < 100: + del ma_dict["vza"][key] + del ma_dict["unc_vza"][key] + + fig, axs = plt.subplots(2, 1) + axs[0].errorbar( + ma_dict["vza"].keys(), + list(ma_dict["vza"].values()), + yerr=list(ma_dict["unc_vza"].values()), + marker="", + linestyle="", + capsize=5, + ) + axs[1].bar(ma_dict["vza"].keys(), list(ma_dict["num"].values())) + + axs[0].set_ylabel("VZA") + axs[1].set_ylabel("Number of Measurements") + plt.show() + + +def normalise_ratios(dataset): + val, bins = pd.qcut(dataset.sza, 15, retbins=True) + bins.round(1) + bins[0] = 0 + sza_bounds = bins + sza_min = sza_bounds[0] + sza_max = sza_bounds[1] + sza_center = np.mean([sza_min, sza_max]).round(1) + + for wav in wavelengths: + ds = ( + dataset.loc[dict(wv=wav)] + .set_coords("sza") + .where((dataset.sza < sza_max) & (dataset.sza > sza_min)) + ) + ds = ds.dropna(dim="date") + mean, unc = chi_square_minimiser_MCMC( + ds.sza.values, + ds.saa.values, + ds.dir_diff_ratio.values, + np.array(list(ds.ratio.values)), + wav + f"_{sza_center}", + ) + + ratio_grid = ratio_calculator_offset( + mean[0], + mean[1], + mean[2], + dataset.sza.values, + dataset.saa.values, + dataset.dir_diff_ratio.loc[dict(wv=wav)].values, + ) + + dataset.ratio.loc[dict(wv=wav)] = ( + dataset.ratio.loc[dict(wv=wav)].values / ratio_grid + ) + + return dataset + + +def find_misalignment_angles(dataset, ratio_thresh): + retrieval_dict = {"vza": {}, "unc_vza": {}, "vaa": {}, "unc_vaa": {}} + # dataset = normalise_ratios(dataset) + + for wv in wavelengths: + ma_dict = plot_misalign_sza(dataset, wv, True) + + converged_vza = convergence_test(ma_dict["vza"].values(), 3, 1) + converged_vaa = convergence_test(ma_dict["vaa"].values(), 3, 10) + + converged_keys = list(ma_dict["ratio"].keys())[ + -(min(converged_vza, converged_vaa)) : + ] + + thresh_keys = [ + k + for k, v in ma_dict["ratio"].items() + if abs(v - list(ma_dict["ratio"].values())[0]) > ratio_thresh + ] + + sza_keys = [ + k for k in ma_dict["ratio"].keys() if k in converged_keys and thresh_keys + ] + + retrieval_dict["vza"][f"{wv}"] = np.mean( + [v for k, v in ma_dict["vza"].items() if k in sza_keys] + ) + retrieval_dict["unc_vza"][f"{wv}"] = np.std( + [v for k, v in ma_dict["vza"].items() if k in sza_keys] + ) + retrieval_dict["vaa"][f"{wv}"] = np.mean( + [v for k, v in ma_dict["vaa"].items() if k in sza_keys] + ) + retrieval_dict["unc_vaa"][f"{wv}"] = np.std( + [v for k, v in ma_dict["vaa"].items() if k in sza_keys] + ) + + fig, axs = plt.subplots(2, 1, sharex=True) + axs[0].errorbar( + retrieval_dict["vza"].keys(), + list(retrieval_dict["vza"].values()), + yerr=list(retrieval_dict["unc_vza"].values()), + marker="", + linestyle="", + capsize=5, + ) + axs[1].errorbar( + retrieval_dict["vza"].keys(), + list(retrieval_dict["vaa"].values()), + yerr=list(retrieval_dict["unc_vaa"].values()), + marker="", + linestyle="", + capsize=5, + ) + + axs[0].grid(color="black", alpha=0.5, axis="y") + axs[1].grid(color="black", alpha=0.5, axis="y") + + axs[0].set_ylabel("VZA") + axs[1].set_ylabel("VAA") + + fig.tight_layout() + fig.savefig(os.path.join(results_path, f"retrieval_plot_after.png")) + + +def wav_separated_misalignment_calculator(dataset, wavelength, site): + ma_dict = { + "vza": {}, + "unc_vza": {}, + "vaa": {}, + "ratio": {}, + "unc_ratio": {}, + "unc_vaa": {}, + "num": {}, + "obs_ratio": {}, + "offset": {}, + "obs_grid": {}, + "ratio_grid": {}, + "obs_grid": {}, + "time": {}, + "unc_grid": {}, + } + + residuals = [] + + for wav in wavelength: + ds = dataset.loc[dict(wv=wav)].set_coords("sza").where(dataset.sza > 0) + ds = ds.dropna(dim="date") + try: + mean, unc = chi_square_minimiser_MCMC( + ds.sza.values, + ds.saa.values, + ds.dir_diff_ratio.values, + np.array(list(ds.ratio.values)), + ) + + mean = list(mean) + unc = list(unc) + + if mean[0] < 0: + mean[0] = -mean[0] + mean[1] += -180 + + ma_dict["vza"][f"{wav}"] = mean[0] + ma_dict["unc_vza"][f"{wav}"] = unc[0] + ma_dict["vaa"][f"{wav}"] = mean[1] + ma_dict["unc_vaa"][f"{wav}"] = unc[1] + ma_dict["num"][f"{wav}"] = len(ds.sza.values) + + ratio_grid = ratio_calculator_offset( + mean[0], + mean[1], + mean[2], + ds.sza.values, + ds.saa.values, + ds.dir_diff_ratio.values, + ) + ratio_unc = ( + ratio_uncertainty_calculator( + mean[0], mean[1], ds.sza.values, ds.saa.values, unc[0], unc[1] + ) + * ratio_grid + / 100 + ) + residuals += [ + float(x) + for x in list( + (np.array(list(ds.ratio.values)) - ratio_grid) / ratio_unc + ) + ] + + ma_dict["ratio"][f"{wav}"] = np.mean(ratio_grid) + ma_dict["unc_ratio"][f"{wav}"] = np.sqrt( + (np.std(ratio_grid) ** 2 + np.sum(ratio_unc**2) / len(ratio_unc) ** 2) + ) + ma_dict["obs_ratio"][f"{wav}"] = np.mean(np.array(list(ds.ratio.values))) + ma_dict["offset"][f"{wav}"] = mean[2] + ma_dict["obs_grid"][f"{wav}"] = np.array(list(ds.ratio.values)) + ma_dict["ratio_grid"][f"{wav}"] = ratio_grid + ma_dict["time"][f"{wav}"] = np.array(list(ds.time.values)) + + except: + ma_dict[f"{wav}"] = np.nan + + fig, axs = plt.subplots(4, 1, sharex=True) + axs[0].errorbar( + ma_dict["vza"].keys(), + list(ma_dict["vza"].values()), + yerr=list(ma_dict["unc_vza"].values()), + marker="", + linestyle="", + capsize=5, + ) + axs[1].errorbar( + ma_dict["vza"].keys(), + list(ma_dict["vaa"].values()), + yerr=list(ma_dict["unc_vaa"].values()), + marker="", + linestyle="", + capsize=5, + ) + axs[2].errorbar( + ma_dict["vza"].keys(), + list(ma_dict["ratio"].values()), + yerr=list(ma_dict["unc_ratio"].values()), + marker="x", + linestyle="", + capsize=5, + ) + axs[2].scatter( + ma_dict["vza"].keys(), list(ma_dict["obs_ratio"].values()), marker="o" + ) + axs[3].bar(ma_dict["vza"].keys(), list(ma_dict["num"].values())) + + axs[0].grid(color="black", alpha=0.5, axis="y") + axs[1].grid(color="black", alpha=0.5, axis="y") + axs[2].grid(color="black", alpha=0.5, axis="y") + + axs[2].set_ylim(0.9, 1.1) + + axs[0].set_ylabel("VZA") + axs[1].set_ylabel("VAA") + axs[2].set_ylabel("Ratio") + axs[3].set_ylabel("Number of \nMeasurements") + fig.tight_layout() + fig.savefig(os.path.join(results_path, f"{site}_retrieval_plot.png")) + + return ma_dict + + +def wav_together_misalignment_calculator(dataset, wavelength, site): + ma_dict = { + "vza": [], + "unc_vza": [], + "vaa": [], + "ratio": {}, + "unc_ratio": {}, + "unc_vaa": {}, + "num": [], + "obs_ratio": {}, + "corr_grid": {}, + "offset": {}, + "ratio_grid": {}, + "obs_grid": {}, + "time": {}, + "unc_grid": {}, + } + + residuals = [] + + ds = dataset.set_coords("sza").where(dataset.sza > 0) + ds = ds.dropna(dim="date") + + # mean, unc = chi_square_minimiser_MCMC_10offsets(ds.sza.values, ds.saa.values, + # ds.dir_diff_ratio.values, + # np.array(list(ds.ratio.values)), + # plot_name=site) + + mean, unc, chi = chi_square_minimiser_MCMC( + ds.sza.values, + ds.saa.values, + ds.dir_diff_ratio.values, + np.array(list(ds.ratio.values)), + plot_name=site, + ) + + print("VZA", mean[0], unc[0]) + print("VAA", mean[1], unc[1]) + mean = list(mean) + unc = list(unc) + + if mean[0] < 0: + mean[0] = -mean[0] + mean[1] += -180 + + ma_dict["vza"] = mean[0] + ma_dict["unc_vza"] = unc[0] + ma_dict["vaa"] = mean[1] + ma_dict["unc_vaa"] = unc[1] + ma_dict["num"] = len(ds.sza.values) + + for iwav, wav in enumerate(wavelength): + ds = dataset.loc[dict(wv=wav)].set_coords("sza").where(dataset.sza > 0) + ds = ds.dropna(dim="date") + # try: + # ratio_grid = ratio_calculator_offset(mean[0], mean[1], mean[iwav+2], + # ds.sza.values, ds.saa.values, + # ds.dir_diff_ratio.values) + ratio_grid = ratio_calculator( + mean[0], mean[1], ds.sza.values, ds.saa.values, ds.dir_diff_ratio.values + ) + ratio_unc = ( + ratio_uncertainty_calculator( + mean[0], mean[1], ds.sza.values, ds.saa.values, unc[0], unc[1] + ) + * ratio_grid + / 100 + ) + correction_grid = ratio_calculator( + mean[0], mean[1], ds.sza.values, ds.saa.values, ds.dir_diff_ratio.values + ) + + ma_dict["ratio"][f"{wav}"] = np.mean(ratio_grid) + ma_dict["unc_ratio"][f"{wav}"] = np.sqrt( + (np.std(ratio_grid) ** 2 + np.sum(ratio_unc**2) / len(ratio_unc) ** 2) + ) + ma_dict["obs_ratio"][f"{wav}"] = np.mean(np.array(list(ds.ratio.values))) + # ma_dict['offset'][f'{wav}'] = mean[iwav + 2] + ma_dict["obs_grid"][f"{wav}"] = np.array(list(ds.ratio.values)) + ma_dict["corr_grid"][f"{wav}"] = np.array( + list(ds.ratio.values / correction_grid) + ) + ma_dict["ratio_grid"][f"{wav}"] = ratio_grid + ma_dict["time"][f"{wav}"] = np.array(list(ds.time.values)) + # + # except: + # ma_dict[f'{wav}'] = np.nan + + fig = plt.figure() + ax = fig.add_subplot(projection="polar") + ax.set_theta_zero_location("N") + ax.set_theta_direction(-1) + ax.errorbar( + np.radians(ma_dict["vaa"]), + ma_dict["vza"], + xerr=np.radians(ma_dict["unc_vaa"]), + yerr=ma_dict["unc_vza"], + capsize=5, + ) + ax.set_ylim([0, 5]) + fig.savefig(os.path.join(results_path, f"{site}_retrieval_plot_combined.png")) + + fig, ax = plt.subplots(1, 1, sharex=True) + ax.errorbar( + ma_dict["ratio"].keys(), + [ma_dict["ratio"][k] for k in ma_dict["ratio"].keys()], + yerr=list(ma_dict["unc_ratio"].values()), + marker="x", + linestyle="", + capsize=5, + ) + + ax.grid(color="black", alpha=0.5, axis="y") + ax.set_ylabel("Ratio") + ax.set_xlabel("Wavelength") + fig.tight_layout() + fig.savefig(os.path.join(results_path, f"{site}_ratio_plot_combined.png")) + + return ma_dict + + + +def wav_together_misalignment_calculator_offset(dataset, wavelength, site, normalise = False, norm_width = None): + ma_dict = {'vza': [], 'unc_vza': [], 'vaa': [], 'ratio': {}, 'unc_ratio': {}, 'unc_vaa': {}, 'num': [], + 'obs_ratio': {}, 'corr_grid': {}, 'offset': {}, 'ratio_grid': {}, 'obs_grid': {}, 'time': {}, + 'unc_grid': {}, 'offset_grid': {}, 'all_grid': {}, 'norm_corr_grid': {}, 'norm_width': norm_width, + 'norm_corr_grid_all': {}} + + residuals = [] + + ds = dataset.set_coords("sza").where(dataset.sza > 0) + ds = ds.dropna(dim="date") + + mean, unc, chi = chi_square_minimiser_MCMC_10offsets( + ds.sza.values, + ds.saa.values, + ds.dir_diff_ratio.values, + np.array(list(ds.ratio.values)), + plot_name=site, + ) + + # mean, unc, chi = chi_square_minimiser_MCMC(ds.sza.values, ds.saa.values, + # ds.dir_diff_ratio.values, + # np.array(list(ds.ratio.values)), + # plot_name=site) + + mean_only, unc_only, chi_only = chi_square_minimiser_MCMC_only_offsets( + ds.sza.values, + ds.saa.values, + ds.dir_diff_ratio.values, + np.array(list(ds.ratio.values)), + ) + print(chi, chi_only) + print("VZA", mean[0], unc[0]) + print("VAA", mean[1], unc[1]) + mean = list(mean) + unc = list(unc) + + # mean = [mean[0], mean[1], *mean_only] + if mean[0] < 0: + mean[0] = -mean[0] + mean[1] += -180 + + ma_dict["vza"] = mean[0] + ma_dict["unc_vza"] = unc[0] + ma_dict["vaa"] = mean[1] + ma_dict["unc_vaa"] = unc[1] + ma_dict["num"] = len(ds.sza.values) + + for iwav, wav in enumerate(wavelength): + # mean[iwav+2] = 0 + ds = dataset.loc[dict(wv=wav)].set_coords("sza").where(dataset.sza > 0) + ds = ds.dropna(dim="date") + # try: + ratio_grid = ratio_calculator_offset(mean[0], mean[1], mean[iwav+2], + ds.sza.values, ds.saa.values, + ds.dir_diff_ratio.values) + ratio_unc = ratio_uncertainty_calculator(mean[0], mean[1], + ds.sza.values, ds.saa.values, + unc[0], unc[1]) * ratio_grid / 100 + correction_grid = ratio_calculator(mean[0], mean[1], + ds.sza.values, ds.saa.values, + ds.dir_diff_ratio.values) + offset_grid = ratio_calculator_only_offsets(*mean_only, + ds.sza.values, ds.saa.values, + ds.dir_diff_ratio.values) + + + + ma_dict['ratio'][f'{wav}'] = np.mean(ratio_grid) + ma_dict['unc_ratio'][f'{wav}'] = np.sqrt( + (np.std(ratio_grid) ** 2 + np.sum(ratio_unc ** 2) / len(ratio_unc) ** 2)) + ma_dict['obs_ratio'][f'{wav}'] = np.mean(np.array(list(ds.ratio.values))) + ma_dict['offset'][f'{wav}'] = mean[iwav + 2] + ma_dict['obs_grid'][f'{wav}'] = np.array(list(ds.ratio.values)) + ma_dict['offset_grid'][f'{wav}'] = ds.ratio.values / offset_grid[iwav,:] + ma_dict['corr_grid'][f'{wav}'] = np.array(list(ds.ratio.values / correction_grid)) + ma_dict['ratio_grid'][f'{wav}'] = ratio_grid + ma_dict['all_grid'][f'{wav}'] = np.array(list(ds.ratio.values / ratio_grid)) + ma_dict['time'][f'{wav}'] = np.array(list(ds.time.values)) + ma_dict['unc_grid'][f'{wav}'] = ratio_unc + residuals += [float(x) for x in list((np.array(list(ds.ratio.values)) - ratio_grid) / ratio_unc)] + + if normalise: + if 'GHNA' in site: + noon_saa = 0 + else: + noon_saa = 180 + + if norm_width is None: + pass + else: + ds['correction_factor'] = ('date', correction_grid) + mean_corr_factor = np.mean( + ds.where( + (ds.saa > noon_saa - norm_width) | (ds.saa < -(noon_saa - norm_width)), drop = True + ).correction_factor.values) + ma_dict['norm_corr_grid'][f'{wav}'] = ma_dict['corr_grid'][f'{wav}'] * mean_corr_factor + ma_dict['norm_corr_grid_all'][f'{wav}'] = ma_dict['corr_grid'][f'{wav}'] * np.mean(ds.correction_factor.values) + + + fig, ax = plt.subplots(3, 1, sharex=True) + ax[0].scatter( + ds.date.values, np.array(list(ds.ratio.values)), marker="x", alpha=0.7 + ) + ax[1].scatter( + ds.date.values, + np.array(list(ds.ratio.values / correction_grid)), + marker="x", + alpha=0.7, + ) + ax[2].scatter( + ds.date.values, + np.array(list(ds.ratio.values)) + - np.array(list(ds.ratio.values / correction_grid)), + marker="x", + alpha=0.7, + ) + ax[0].grid(color="black", alpha=0.5, axis="y") + ax[1].grid(color="black", alpha=0.5, axis="y") + ax[2].grid(color="black", alpha=0.5, axis="y") + ax[0].set_ylabel("Observed Ratio") + ax[1].set_ylabel("Corrected Ratio") + ax[2].set_ylabel("Observed - Corrected Ratio") + ax[2].set_xlabel("Date") + fig.tight_layout() + fig.savefig(os.path.join(results_path, f"{site}_temporal_ratio_{wav}.png")) + + # except: + # ma_dict[f'{wav}'] = np.nan + print('Mean Correction Factor', np.mean(ds.correction_factor.values), np.std(ds.correction_factor.values)) + + fig = plt.figure() + ax = fig.add_subplot() + ax.hist(residuals, 50, density=True, edgecolor="black", align="mid") + fig.savefig(os.path.join(results_path, f"{site}_residuals_hist.png")) + + fig = plt.figure() + ax = fig.add_subplot(projection="polar") + ax.set_theta_zero_location("N") + ax.set_theta_direction(-1) + ax.errorbar( + np.radians(ma_dict["vaa"]), + ma_dict["vza"], + xerr=np.radians(ma_dict["unc_vaa"]), + yerr=ma_dict["unc_vza"], + capsize=5, + ) + ax.set_ylim([0, 5]) + fig.savefig(os.path.join(results_path, f"{site}_retrieval_plot_combined.png")) + + fig, ax = plt.subplots(2, 1, sharex=True, sharey=True) + ax[0].errorbar( + [int(x) for x in wavelength], + [ma_dict["obs_ratio"][k] for k in ma_dict["ratio"].keys()], + yerr=np.std(list(ma_dict["obs_grid"].values())), + marker="x", + linestyle="", + capsize=5, + ) + ax[1].errorbar( + [int(x) for x in wavelength], + [np.mean(ma_dict["corr_grid"][k]) for k in ma_dict["ratio"].keys()], + yerr=np.std(list(ma_dict["corr_grid"].values())), + marker="x", + linestyle="", + capsize=5, + ) + ax[0].grid(color="black", alpha=0.5, axis="y") + ax[1].grid(color="black", alpha=0.5, axis="y") + ax[0].set_ylabel("Observed Ratio") + ax[1].set_ylabel("Corrected Ratio") + ax[1].set_xlabel("Wavelength") + fig.tight_layout() + fig.savefig(os.path.join(results_path, f"{site}_spectral_ratio.png")) + + return ma_dict + + +# wav_together_misalignment_calculator_offset(data_v1, wavelengths, 'JSIT') + +""" +ma_dict = {'mean': {}, 'unc': {}} +for wv in ratio_dict.keys(): + vza, u_vza = chi_square_minimiser_MCMC(sza_measured, saa_measured,dir_diff_dict[wv], ratio_dict[wv]) + ma_dict['mean'][wv] = vza[0] + ma_dict['unc'][wv] = u_vza[0] + +plt.errorbar(ma_dict['mean'].keys(), list(ma_dict['mean'].values()), yerr = list(ma_dict['unc'].values()), + marker = '', linestyle = '', capsize = 5) +plt.show() +""" +# for wv in wavelengths: +# plot_misalign_sza(data, wv) +# plot_misalign_sza_all_wavs(data_before_may24) + +# find_misalignment_angles(data_after_may24, 0.02) +# plot_misalign_sza(data, '1640') +# wav_separated_misalignment_calculator(data, wavelengths) +bins = [ + 630, + 715, + 745, + 815, + 845, + 915, + 945, + 1015, + 1045, + 1115, + 1145, + 1315, + 1345, + 1415, + 1445, + 1515, + 1545, +] +time = [ + "0700", + "0730", + "0800", + "0830", + "0900", + "0930", + "1000", + "1030", + "1100", + "1130", + "1300", + "1330", + "1400", + "1430", + "1500", + "1530", +] +# wavelengths = ["550"] + +""" +times=np.array([int(x) for x in data_before_may24.time.values]) +i_time_val=np.where((times>800))[0] +data_before_may24=data_before_may24.isel(date=i_time_val) + +times=np.array([int(x) for x in data_after_may24.time.values]) +i_time_val=np.where((times>800))[0] +data_after_may24=data_after_may24.isel(date=i_time_val) + +times=np.array([int(x) for x in data_v1.time.values]) +i_time_val=np.where((times>800))[0] +data_v1=data_v1.isel(date=i_time_val) + +# before_dict = wav_separated_misalignment_calculator(data_before_may24, wavelengths) +# after_dict = wav_separated_misalignment_calculator(data_after_may24, wavelengths) +# all_dict = wav_separated_misalignment_calculator(data, wavelengths) +""" + +#before_dict = wav_together_misalignment_calculator_offset(data_before_may24, wavelengths, 'GHNAv3_before') +#after_dict = wav_together_misalignment_calculator_offset(data_after_may24, wavelengths, 'GHNAv3_after') +v1_dict = wav_together_misalignment_calculator_offset(data_v1, wavelengths, site, normalise=True, norm_width=5) + +for wav in wavelengths: + + fig, axs = plt.subplots(5, 1, sharex=True, figsize=(6, 10)) + + # before_dict['time'][f'{wav}'] = [int(x) for x in before_dict['time'][f'{wav}']] + # after_dict['time'][f'{wav}'] = [int(x) for x in after_dict['time'][f'{wav}']] + v1_dict["time"][f"{wav}"] = [int(x) for x in v1_dict["time"][f"{wav}"]] + + # before_means, be, bn = binned_statistic(list(before_dict['time'][f'{wav}']), list(before_dict['ratio_grid'][f'{wav}']), statistic = 'mean', bins = bins) + # before_std, be, bn = binned_statistic(list(before_dict['time'][f'{wav}']), list(before_dict['ratio_grid'][f'{wav}']), statistic = 'std', bins = bins) + + # after_means, bin_edges, binnumber = binned_statistic(list(after_dict['time'][f'{wav}']), list(after_dict['ratio_grid'][f'{wav}']), statistic = 'mean', bins = bins) + # after_std, bin_edge, binnumbe = binned_statistic(list(after_dict['time'][f'{wav}']), list(after_dict['ratio_grid'][f'{wav}']), statistic = 'std', bins = bins) + + v1_means, bin_edges, binnumber = binned_statistic( + list(v1_dict["time"][f"{wav}"]), + list(v1_dict["ratio_grid"][f"{wav}"]), + statistic="mean", + bins=bins, + ) + v1_std, bin_edge, binnumbe = binned_statistic( + list(v1_dict["time"][f"{wav}"]), + list(v1_dict["ratio_grid"][f"{wav}"]), + statistic="std", + bins=bins, + ) + + # before_obs_means, be, bn = binned_statistic(list(before_dict['time'][f'{wav}']), + # list(before_dict['obs_grid'][f'{wav}']), statistic='mean', bins=bins) + # before_obs_std, be, bn = binned_statistic(list(before_dict['time'][f'{wav}']), + # list(before_dict['obs_grid'][f'{wav}']), statistic='std', bins=bins) + + # after_obs_means, bin_edges, binnumber = binned_statistic(list(after_dict['time'][f'{wav}']), + # list(after_dict['obs_grid'][f'{wav}']), statistic='mean', + # bins=bins) + # after_obs_std, bin_edge, binnumbe = binned_statistic(list(after_dict['time'][f'{wav}']), + # list(after_dict['obs_grid'][f'{wav}']), statistic='std', + # bins=bins) + + v1_obs_means, bin_edges, binnumber = binned_statistic( + list(v1_dict["time"][f"{wav}"]), + list(v1_dict["obs_grid"][f"{wav}"]), + statistic="mean", + bins=bins, + ) + v1_obs_std, bin_edge, binnumbe = binned_statistic( + list(v1_dict["time"][f"{wav}"]), + list(v1_dict["obs_grid"][f"{wav}"]), + statistic="std", + bins=bins, + ) + + # before_corr_means, be, bn = binned_statistic(list(before_dict['time'][f'{wav}']), + # list(before_dict['corr_grid'][f'{wav}']), statistic='mean', bins=bins) + # before_corr_std, be, bn = binned_statistic(list(before_dict['time'][f'{wav}']), + # list(before_dict['corr_grid'][f'{wav}']), statistic='std', bins=bins) + + # after_corr_means, bin_edges, binnumber = binned_statistic(list(after_dict['time'][f'{wav}']), + # list(after_dict['corr_grid'][f'{wav}']), statistic='mean', + # bins=bins) + # after_corr_std, bin_edge, binnumbe = binned_statistic(list(after_dict['time'][f'{wav}']), + # list(after_dict['corr_grid'][f'{wav}']), statistic='std', + # bins=bins) + + v1_corr_means, bin_edges, binnumber = binned_statistic( + list(v1_dict["time"][f"{wav}"]), + list(v1_dict["corr_grid"][f"{wav}"]), + statistic="mean", + bins=bins, + ) + v1_corr_std, bin_edge, binnumbe = binned_statistic( + list(v1_dict["time"][f"{wav}"]), + list(v1_dict["corr_grid"][f"{wav}"]), + statistic="std", + bins=bins, + ) + + # before_sza, be, bn = binned_statistic(list(before_dict['time'][f'{wav}']), + # data_before_may24.sza.values, statistic='median', bins=bins) + # before_sza_std, be, bn = binned_statistic(list(before_dict['time'][f'{wav}']), + # data_before_may24.sza.values, statistic='std', bins=bins) + + v1_corr_means, bin_edges, binnumber = binned_statistic(list(v1_dict['time'][f'{wav}']), + list(v1_dict['norm_corr_grid'][f'{wav}']), statistic='mean', + bins=bins) + v1_corr_std, bin_edge, binnumbe = binned_statistic(list(v1_dict['time'][f'{wav}']), + list(v1_dict['norm_corr_grid'][f'{wav}']), statistic='std', bins=bins) + + v1_sza, bin_edges, binnumber = binned_statistic( + list(v1_dict["time"][f"{wav}"]), + data_v1.sza.values, + statistic="median", + bins=bins, + ) + v1_sza_std, bin_edge, binnumbe = binned_statistic( + list(v1_dict["time"][f"{wav}"]), data_v1.sza.values, statistic="std", bins=bins + ) + + # axs[1].errorbar(time, before_means, yerr=before_std, linestyle='', marker='x', capsize=5, color='red', + # label='2023-10-26 - 2024-05-23') + # axs[1].errorbar(time, after_means, yerr=after_std, linestyle='', marker='x', capsize=5, color='green', + # label='2024-05-24 - present') + axs[1].errorbar( + time, v1_means, yerr=v1_std, linestyle="", marker="x", capsize=5, color="blue" + ) + + # axs[0].errorbar(time, before_obs_means, yerr=before_obs_std, linestyle='', marker='x', capsize=5, color='red', + # label='2023-10-26 - 2024-05-23') + # axs[0].errorbar(time, after_obs_means, yerr=after_obs_std, linestyle='', marker='x', capsize=5, color='green', + # label='2024-05-24 - present') + axs[0].errorbar( + time, + v1_obs_means, + yerr=v1_obs_std, + linestyle="", + marker="x", + capsize=5, + color="blue", + ) + + # axs[2].scatter(time, before_obs_means-before_means, linestyle = '', marker = 'x', color = 'red') + # axs[2].scatter(time, after_obs_means-after_means, linestyle='', marker='x', color='green') + axs[2].scatter( + time, v1_obs_means - v1_means, linestyle="", marker="x", color="blue" + ) + + # axs[3].errorbar(time, before_corr_means, yerr=before_corr_std, linestyle='', marker='x', capsize=5, color='red', + # label='before') + # axs[3].errorbar(time, after_corr_means, yerr=after_corr_std, linestyle='', marker='x', capsize=5, color='green', + # label='after') + axs[3].errorbar( + time, + v1_corr_means, + yerr=v1_corr_std, + linestyle="", + marker="x", + capsize=5, + color="blue", + label="v1", + ) + + # axs[4].errorbar(time, before_sza, yerr=before_sza_std, linestyle='', marker='x', capsize=5, color='red', + # label='before') + # axs[4].errorbar(time, after_sza, yerr=after_sza_std, linestyle='', marker='x', capsize=5, color='green', + # label='after') + axs[4].errorbar( + time, + v1_sza, + yerr=v1_sza_std, + linestyle="", + marker="x", + capsize=5, + color="blue", + label="v1", + ) + + axs[1].set_ylabel("Modelled Ratio") + axs[0].set_ylabel("Observed Ratio") + axs[2].set_ylabel("Observed-modelled") + axs[3].set_ylabel("Corrected Observed Ratio") + axs[4].set_ylabel("SZA") + + #axs[3].errorbar(time, before_corr_means, yerr=before_corr_std, linestyle='', marker='x', capsize=5, color='red', + # label='before') + #axs[3].errorbar(time, after_corr_means, yerr=after_corr_std, linestyle='', marker='x', capsize=5, color='green', + # label='after') + axs[3].errorbar(time, v1_corr_means, yerr=v1_corr_std, linestyle='', marker='x', capsize=5, color='blue', + label='v1') + + #axs[4].errorbar(time, before_sza, yerr=before_sza_std, linestyle='', marker='x', capsize=5, color='red', + # label='before') + #axs[4].errorbar(time, after_sza, yerr=after_sza_std, linestyle='', marker='x', capsize=5, color='green', + # label='after') + axs[4].errorbar(time, v1_sza, yerr=v1_sza_std, linestyle='', marker='x', capsize=5, color='blue', + label='v1') + + axs[1].set_ylabel('Modelled Ratio') + axs[0].set_ylabel('Observed Ratio') + axs[2].set_ylabel('Observed-modelled') + axs[3].set_ylabel('Normalised Corrected Ratio') + axs[4].set_ylabel('SZA') + + axs[0].set_ylim(0.9,1.1) + axs[1].set_ylim(0.9,1.1) + axs[2].set_ylim(-0.05,0.05) + axs[3].set_ylim(0.9,1.1) + + axs[0].legend() + axs[0].grid(color="black", alpha=0.5, axis="y") + axs[1].grid(color="black", alpha=0.5, axis="y") + axs[2].grid(color="black", alpha=0.5, axis="y") + axs[3].grid(color="black", alpha=0.5, axis="y") + + fig.suptitle(f"{wav}") + + fig.tight_layout() + fig.savefig(os.path.join(results_path, f"{site}_ratio_tod_plot_{wav}.png")) + plt.close() + + binned_data_obs = binned_statistic_2d( + np.radians(data_v1.saa.values), + data_v1.sza.values, + list(v1_dict["obs_grid"][f"{wav}"]), + bins=10, + ) + binned_data_corr = binned_statistic_2d( + np.radians(data_v1.saa.values), + data_v1.sza.values, + list(v1_dict["corr_grid"][f"{wav}"]), + bins=10, + ) + binned_data_offset = binned_statistic_2d( + np.radians(data_v1.saa.values), + data_v1.sza.values, + list(v1_dict["offset_grid"][f"{wav}"]), + bins=10, + ) + binned_data_all = binned_statistic_2d( + np.radians(data_v1.saa.values), + data_v1.sza.values, + list(v1_dict["all_grid"][f"{wav}"]), + bins=10, + ) + + sza_mesh, saa_mesh = np.meshgrid(binned_data_corr.y_edge, binned_data_corr.x_edge) + saa_grid = binned_data_corr.x_edge + sza_grid = binned_data_corr.y_edge + + refl_corr = np.zeros((len(saa_grid), len(sza_grid))) + refl_obs = np.zeros((len(saa_grid), len(sza_grid))) + refl_offset = np.zeros((len(saa_grid), len(sza_grid))) + refl_all = np.zeros((len(saa_grid), len(sza_grid))) + refl_norm = np.zeros((len(saa_grid), len(sza_grid))) + refl_norm_all = np.zeros((len(saa_grid), len(sza_grid))) + for i in range(len(saa_grid) - 1): + for j in range(len(sza_grid) - 1): + id_series = np.where( + (data_v1.sza.values > sza_grid[j]) + & (data_v1.sza.values < sza_grid[j + 1]) + & (np.radians(data_v1.saa.values) > saa_grid[i]) + & (np.radians(data_v1.saa.values) < saa_grid[i + 1]) + )[0] + if len(id_series) == 1: + refl_corr[i, j] = list(v1_dict['corr_grid'][f'{wav}'])[id_series[0]] + refl_obs[i, j] = list(v1_dict['obs_grid'][f'{wav}'])[id_series[0]] + refl_offset[i, j] = list(v1_dict['offset_grid'][f'{wav}'])[id_series[0]] + refl_all[i, j] = list(v1_dict['all_grid'][f'{wav}'])[id_series[0]] + refl_norm[i, j] = list(v1_dict['norm_corr_grid'][f'{wav}'])[id_series[0]] + refl_norm_all[i, j] = list(v1_dict['norm_corr_grid_all'][f'{wav}'])[id_series[0]] + + elif len(id_series) > 1: + refl_corr[i, j] = np.mean([list(v1_dict['corr_grid'][f'{wav}'])[k] for k in id_series]) + refl_obs[i, j] = np.mean([list(v1_dict['obs_grid'][f'{wav}'])[k] for k in id_series]) + refl_offset[i, j] = np.mean([list(v1_dict['offset_grid'][f'{wav}'])[k] for k in id_series]) + refl_all[i, j] = np.mean([list(v1_dict['all_grid'][f'{wav}'])[k] for k in id_series]) + refl_norm[i, j] = np.mean([list(v1_dict['norm_corr_grid'][f'{wav}'])[k] for k in id_series]) + refl_norm_all[i, j] = np.mean([list(v1_dict['norm_corr_grid_all'][f'{wav}'])[k] for k in id_series]) + + refl_corr[refl_corr == 0] = np.nan + refl_obs[refl_obs == 0] = np.nan + refl_offset[refl_offset == 0] = np.nan + refl_all[refl_all == 0] = np.nan + refl_norm[refl_norm == 0] = np.nan + refl_norm_all[refl_norm_all == 0] = np.nan + + fig, axs = plt.subplots(2, 3, figsize = (20,16), subplot_kw={'projection': 'polar'}) + norm = matplotlib.colors.CenteredNorm(vcenter = 1, halfrange = 0.1) + for ax in axs[0]: + ax.set_theta_direction(-1) + ax.set_theta_offset(np.pi / 2.0) + for ax in axs[1]: + ax.set_theta_direction(-1) + ax.set_theta_offset(np.pi / 2.0) + im0 = axs[0, 0].pcolormesh( + saa_mesh, sza_mesh, refl_obs, shading="auto", cmap="coolwarm", norm=norm + ) + + im1 = axs[0, 1].pcolormesh( + saa_mesh, sza_mesh, refl_corr, shading="auto", cmap="coolwarm", norm=norm + ) + + im2 = axs[1, 0].pcolormesh( + saa_mesh, sza_mesh, refl_offset, shading="auto", cmap="coolwarm", norm=norm + ) + + im3 = axs[1,1].pcolormesh( + saa_mesh, + sza_mesh, + refl_all, + shading="auto", + cmap='coolwarm', + norm=norm) + + im4 = axs[0,2].pcolormesh( + saa_mesh, + sza_mesh, + refl_norm, + shading="auto", + cmap='coolwarm', + norm=norm) + + im5 = axs[1,2].pcolormesh( + saa_mesh, + sza_mesh, + refl_norm_all, + shading="auto", + cmap='coolwarm', + norm=norm) + + cbar0 = fig.colorbar(im0, ax = axs[0,0], fraction = 0.1, shrink = 0.8, pad = 0.2) + cbar1 = fig.colorbar(im1, ax = axs[0,1], fraction = 0.1, shrink = 0.8, pad = 0.2) + cbar2 = fig.colorbar(im2, ax = axs[1,0], fraction = 0.1, shrink = 0.8, pad = 0.2) + cbar3 = fig.colorbar(im3, ax=axs[1,1], fraction=0.1, shrink=0.8, pad=0.2) + cbar4 = fig.colorbar(im4, ax=axs[0,2], fraction=0.1, shrink=0.8, pad=0.2) + cbar5 = fig.colorbar(im5, ax=axs[1,2], fraction=0.1, shrink=0.8, pad=0.2) + + axs[0,0].set_title('Observed Ratio') + axs[0,1].set_title('Corrected Ratio') + axs[1,0].set_title('Only Offsets Ratio') + axs[1,1].set_title('Corrected Ratio \nIncluding Offsets') + axs[0,2].set_title('Normalised Corrected Ratio \nUsing Mid-Day {} Degree Norm'.format(v1_dict['norm_width'])) + axs[1,2].set_title('Normalised Corrected Ratio \nUsing All Data') + + fig.tight_layout() + fig.savefig(os.path.join(results_path, f"{site}_ratio_polar_plot_{wav}.png")) + plt.close() + + +""" +corrected_measurements = measurements * ratio_calculator(sza_measured, saa_measured, 1.7, 95) +corrected_perc = (clear_sky_model - corrected_measurements) / corrected_measurements * 100 +perc = (clear_sky_model - measurements) / measurements * 100 + +#corr_data = np.vstack((corrected_ratio, sza_measured, saa_measured, data[:,5])).T +#np.savetxt(r'T:\ECO\EOServer\data\insitu\hypernets\post_processing_qc\joe\irradiance_GHNA_v3_corrected.csv', corr_data, delimiter = ',') + +bins = [630, 715, 745, 815, 845, 915, 945, 1015, 1045, 1115, 1145, 1315, 1345, 1415, 1445, 1515, 1545] +time = ['0700', '0730', '0800', '0830', '0900', '0930', '1000', '1030', '1100', '1130', '1300', '1330', '1400', '1430', + '1500', '1530'] + +tod = data[:, 5] +timeWWUK = ['0900', '0930', '1000', '1030', '1100', '1130', '1300', '1330', '1400', '1430', '1500', '1530'] +binsWWUK = [845, 915, 945, 1015, 1045, 1115, 1145, 1315, 1345, 1415, 1445, 1515, 1545] + +corr_medians, bin_edges, binnumber = binned_statistic(tod, corrected_perc, statistic='median', bins=bins) +corr_std, bin_edge, binnumbe = binned_statistic(tod, corrected_perc, statistic='std', bins=bins) + +medians, bin_edges, binnumber = binned_statistic(tod, perc, statistic='median', bins=bins) +std, bin_edge, binnumbe = binned_statistic(tod, perc, statistic='std', bins=bins) + +plt.errorbar(time, medians, yerr=std, linestyle='', marker='x', capsize=5, color = 'red', label = 'measurements') +plt.errorbar(time, corr_medians, yerr=corr_std, linestyle='', marker='x', capsize=5, color = 'green', label = 'corrected') +plt.hlines(y=0, xmin=-0.1, xmax=15.5, alpha=0.5, color='black') +plt.xlabel('Time of Day') +plt.ylabel('Percentage Difference between \nModel and Measurements') +plt.legend() +plt.tight_layout() +plt.savefig(r'T:\ECO\EOServer\joe\hypernets_plots\misalignment\GHNAv3_corrected_tod_perc.png') +plt.show() + +bins = [-5, 5, 15, 25, 35, 45, 55, 65, 75] +szas_binned = [0, 10, 20, 30, 40, 50, 60, 70] + +corr_medians, bin_edges, binnumber = binned_statistic(sza_measured, corrected_perc, statistic='median', bins=bins) +corr_std, bin_edge, binnumbe = binned_statistic(sza_measured, corrected_perc, statistic='std', bins=bins) + +medians, bin_edges, binnumber = binned_statistic(sza_measured, perc, statistic='median', bins=bins) +std, bin_edge, binnumbe = binned_statistic(sza_measured, perc, statistic='std', bins=bins) + +plt.errorbar(szas_binned, medians, yerr=std, linestyle='', marker='x', capsize=5, color = 'red', label = 'measurements') +plt.errorbar(szas_binned, corr_medians, yerr=corr_std, linestyle='', marker='x', capsize=5, color = 'green', label = 'corrected') +plt.hlines(y=0, xmin=-0.1, xmax=70.5, alpha=0.5, color='black') +plt.xlabel('SZA') +plt.ylabel('Percentage Difference between \nModel and Measurements') +plt.tight_layout() +plt.savefig(r'T:\ECO\EOServer\joe\hypernets_plots\misalignment\GHNAv3_corrected_SZA_perc.png') +plt.show() + +""" diff --git a/hypernets_processor/post_processing/plane_fitter.py b/hypernets_processor/post_processing/plane_fitter.py new file mode 100644 index 00000000..7e903d70 --- /dev/null +++ b/hypernets_processor/post_processing/plane_fitter.py @@ -0,0 +1,227 @@ +import numpy as np +import pandas as pd +import matplotlib.pyplot as plt +from scipy.optimize import curve_fit +from mpl_toolkits.mplot3d import Axes3D +import seaborn as sns +from scipy.stats import chisquare, goodness_of_fit + +results_path = r"T:/ECO/EOServer/joe/hypernets_plots/plane_models/" + +# read and bin data +data_19 = pd.read_csv( + r"T:/ECO/EOServer/data/insitu/hypernets/post_processing_qc/joe/GHNA_2022_plane_definition.csv" +) +data_0875 = pd.read_csv( + r"T:/ECO/EOServer/data/insitu/hypernets/post_processing_qc/joe/prelim_cc0875.csv" +) +data_unmasked = pd.read_csv( + r"T:\ECO\EOServer\data\insitu\hypernets\post_processing_qc\joe\GHNA_2022_prelim_and_raa10.csv" +) + + +def raa_binning(data, raa_limits): + + if raa_limits[1] > raa_limits[0]: + new_data = data[(data["raa"] > raa_limits[0]) & (data["raa"] < raa_limits[1])] + new_data = new_data.reset_index(drop=True) + return new_data + + if raa_limits[1] < raa_limits[0]: + new_data = data[(data["raa"] > raa_limits[0]) | (data["raa"] < raa_limits[1])] + new_data = new_data.reset_index(drop=True) + return new_data + + +data_19_30_90 = raa_binning(data_19, (30, 90)) +data_19_90_150 = raa_binning(data_19, (90, 150)) +data_19_150_210 = raa_binning(data_19, (150, 210)) +data_19_210_270 = raa_binning(data_19, (210, 270)) +data_19_270_330 = raa_binning(data_19, (270, 330)) +data_19_330_30 = raa_binning(data_19, (330, 30)) + +data_19_list = [ + data_19_30_90, + data_19_90_150, + data_19_150_210, + data_19_210_270, + data_19_270_330, + data_19_330_30, +] + +data_0875_30_90 = raa_binning(data_0875, (30, 90)) +data_0875_90_150 = raa_binning(data_0875, (90, 150)) +data_0875_150_210 = raa_binning(data_0875, (150, 210)) +data_0875_210_270 = raa_binning(data_0875, (210, 270)) +data_0875_270_330 = raa_binning(data_0875, (270, 330)) +data_0875_330_30 = raa_binning(data_0875, (330, 30)) + +data_0875_list = [ + data_0875_30_90, + data_0875_90_150, + data_0875_150_210, + data_0875_210_270, + data_0875_270_330, + data_0875_330_30, +] + +data_unmasked_30_90 = raa_binning(data_unmasked, (30, 90)) +data_unmasked_90_150 = raa_binning(data_unmasked, (90, 150)) +data_unmasked_150_210 = raa_binning(data_unmasked, (150, 210)) +data_unmasked_210_270 = raa_binning(data_unmasked, (210, 270)) +data_unmasked_270_330 = raa_binning(data_unmasked, (270, 330)) +data_unmasked_330_30 = raa_binning(data_unmasked, (330, 30)) + +data_unmasked_list = [ + data_unmasked_30_90, + data_unmasked_90_150, + data_unmasked_150_210, + data_unmasked_210_270, + data_unmasked_270_330, + data_unmasked_330_30, +] + +# define plane surface model + + +def quad_plane(xy, a, b, c, d, e, f): + x, y = xy + return a + b * x + c * y + d * x**2 + e * y**2 + f * x * y + + +def lin_plane(xy, a, b, c): + x, y = xy + return a + b * x + c * y + + +# curve fit +def plane_fit_and_plot(data, data_setting, title, stds, plot=False): + x = data[" sza"].values + y = data[" vza"].values + z = data[" refl_550nm"].values + + popt, pcov = curve_fit(quad_plane, (x, y), z) + below = popt - stds * np.sqrt(np.diag(pcov)) + above = popt + stds * np.sqrt(np.diag(pcov)) + + z_below = ( + quad_plane((data_setting[" sza"].values, data_setting[" vza"].values), *below) + - data_setting[" refl_550nm"].values * 0.05 + ) + z_above = ( + quad_plane((data_setting[" sza"].values, data_setting[" vza"].values), *above) + + data_setting[" refl_550nm"].values * 0.05 + ) + + if plot is True: + fig = plt.figure() + ax = fig.add_subplot(111, projection="3d") + + ax.scatter( + data_setting[" sza"].values, + data_setting[" vza"].values, + data_setting[" refl_550nm"].values, + ) + + x_range = np.linspace(0, 90, 1000) + y_range = np.linspace(0, 90, 1000) + + X, Y = np.meshgrid(x_range, y_range) + Z = quad_plane((X, Y), *popt) + Z_below = quad_plane((X, Y), *below) - 0.05 * Z + Z_above = quad_plane((X, Y), *above) + 0.05 * Z + + ax.plot_surface(X, Y, Z, alpha=0.5) + ax.plot_surface(X, Y, Z_above, alpha=0.5) + ax.plot_surface(X, Y, Z_below, alpha=0.5) + + ax.set_xlabel("SZA") + ax.set_ylabel("VZA") + ax.set_zlabel("Reflectance") + # fig.savefig(results_path + title) + plt.show() + + fig, axs = plt.subplots(1, 3, figsize=(6, 4), sharey=True) + for i in range(3): + axs[i].plot(Y, quad_plane(((i + 1) * 20, Y), *popt), color="blue") + axs[i].plot( + Y, quad_plane(((i + 1) * 20, Y), *below) - 0.1 * Z, color="green" + ) + axs[i].plot( + Y, quad_plane(((i + 1) * 20, Y), *above) + 0.1 * Z, color="orange" + ) + axs[i].set_title(f"SZA = {(i+1)*20}") + axs[i].scatter( + data_setting[ + data_setting[" sza"].between((i + 1) * 20 - 1, (i + 1) * 20 + 1) + ][" vza"].values, + data_setting[ + data_setting[" sza"].between((i + 1) * 20 - 1, (i + 1) * 20 + 1) + ][" refl_550nm"].values, + color="black", + s=5, + ) + axs[i].set_xlabel("VZA") + axs[0].set_ylabel("Reflectance") + fig.suptitle("RAA (270,330)") + # fig.savefig(results_path + 'sza_slices_tol01_270_330.png') + plt.show() + + outliers = data_setting[ + (data_setting[" refl_550nm"] > z_above) + | (data_setting[" refl_550nm"] < z_below) + ] + good_data = data_setting[ + (data_setting[" refl_550nm"] < z_above) + & (data_setting[" refl_550nm"] > z_below) + ] + good_data.reset_index(drop=True, inplace=True) + outliers.reset_index(drop=True, inplace=True) + + return outliers, good_data + + +# redefine outliers using planes +outs_19, good_19 = plane_fit_and_plot( + data_19_list[0], data_unmasked_list[0], "", 3, plot=False +) +for i in range(len(data_19_list) - 1): + outs_19 = pd.concat( + [ + outs_19, + plane_fit_and_plot( + data_19_list[i + 1], data_unmasked_list[i + 1], "", 3, plot=False + )[0], + ], + ignore_index=True, + ) + good_19 = pd.concat( + [ + good_19, + plane_fit_and_plot( + data_19_list[i + 1], data_unmasked_list[i + 1], "", 3, plot=False + )[1], + ], + ignore_index=True, + ) + +""" +outs_0875, good_0875 = plane_fit_and_plot(data_0875_list[0], data_unmasked_list[0], '', 3, plot = False) +for i in range(len(data_0875_list) - 1): + outs_0875 = pd.concat([outs_0875, plane_fit_and_plot(data_0875_list[i+1], data_unmasked_list[i+1], '', 3, plot = False)[0]], ignore_index = True) + good_0875 = pd.concat([good_0875, plane_fit_and_plot(data_0875_list[i+1], data_unmasked_list[i+1], '', 3, plot = False)[1]], ignore_index = True) +""" + +# outs_19.to_csv(r'T:/ECO/EOServer/data/insitu/hypernets/post_processing_qc/joe/GHNA_2022_outliers.csv') +# good_19.to_csv(r'T:/ECO/EOServer/data/insitu/hypernets/post_processing_qc/joe/GHNA_2022_good.csv') + + +# outs_0875.to_csv(r'T:/ECO/EOServer/data/insitu/hypernets/post_processing_qc/joe/outliers_unmasked_0875.csv') +# good_0875.to_csv(r'T:/ECO/EOServer/data/insitu/hypernets/post_processing_qc/joe/good_unmasked_0875.csv') + + +print(len(outs_19), len(good_19)) +print("....") +# print(len(outs_0875), len(good_0875)) + +plane_fit_and_plot(data_19_330_30, data_unmasked_330_30, "plane_330_30", 3, True) diff --git a/hypernets_processor/post_processing/polar_plotter.py b/hypernets_processor/post_processing/polar_plotter.py new file mode 100644 index 00000000..0961bd47 --- /dev/null +++ b/hypernets_processor/post_processing/polar_plotter.py @@ -0,0 +1,241 @@ +import numpy as np +import hypernets_brdf_data_io as data_io +import os +import glob +import matplotlib + +matplotlib.use("Agg") + +import matplotlib.pyplot as plt +import xarray as xr +import pandas as pd + +results_path = r"T:/ECO/EOServer/joe/hypernets_plots/polar_plots_JSIT/" + +data_19 = pd.read_csv( + r"T:/ECO/EOServer/data/insitu/hypernets/post_processing_qc/joe/prelim_cc19.csv" +) +data_0875 = pd.read_csv( + r"T:/ECO/EOServer/data/insitu/hypernets/post_processing_qc/joe/prelim_cc0875.csv" +) +data_JSIT = pd.read_csv( + r"T:/ECO/EOServer/data/insitu/hypernets/post_processing_qc/joe/JSIT_dates_times_raa.csv" +) + +data_WWUKv3 = pd.read_csv( + r"T:/ECO/EOServer/data/insitu/hypernets/post_processing_qc/WWUKv3_QC_good.csv" +) + + +def get_month(data): + months = [] + for i in range(len(data)): + month = str(data["date"][i])[4:6] + months.append(month) + new_data = data.assign(month=months) + return new_data + + +data_19 = get_month(data_19) +data_0875 = get_month(data_0875) +data_JSIT = get_month(data_JSIT) +data_WWUKv3 = get_month(data_WWUKv3) + + +def plot_polar_reflectance( + plotpath, vza, vaa, sza, saa, refl, wavelength, vmin=None, vmax=None, label=None +): + raa_tol = 2.5 + vza_tol = 5 + + vaa = vaa % 360 + saa = saa % 360 + raa = (saa - vaa + 360) % 360 + + raa_grid = np.arange(0, 360, 5) + vza_grid = np.array([0, 10, 20, 30, 40, 50, 60]) + + vaa_mesh, vza_mesh = np.meshgrid(np.radians(raa_grid), vza_grid) + + refl_2d = np.zeros((len(raa_grid), len(vza_grid))) + sza_list = [] + saa_list = [] + for i in range(len(raa_grid)): + for j in range(len(vza_grid)): + id_series = np.where( + (np.abs(raa - raa_grid[i]) < raa_tol) + & (np.abs(vza - vza_grid[j]) < vza_tol) + )[0] + if len(id_series) == 1: + refl_2d[i, j] = refl[id_series] + saa_list.append(saa[id_series][0]) + sza_list.append(sza[id_series][0]) + + elif len(id_series) > 1: + refl_2d[i, j] = np.mean(refl[id_series]) + saa_list.append(float(np.mean(saa[id_series]))) + sza_list.append(float(np.mean(sza[id_series]))) + + saa_list = np.radians(saa_list) + + refl_2d[refl_2d == 0] = np.nan + + fig = plt.figure() + ax = plt.subplot(1, 1, 1, projection="polar") + ax.set_theta_direction(-1) + ax.set_theta_offset(np.pi / 2.0) + im = ax.pcolormesh( + vaa_mesh, + vza_mesh, + refl_2d.T, + shading="auto", + cmap=plt.get_cmap("jet"), + vmin=vmin, + vmax=vmax, + ) + # ax.scatter(saa_list, sza_list, color="black", marker="o") + + cbar = fig.colorbar(im) + + if label is None: + cbar.set_label("reflectance at %s nm" % wavelength, rotation=270, labelpad=15) + else: + cbar.set_label(label, rotation=270, labelpad=15) + + fig.savefig(plotpath) + plt.close(fig) + + +def plot_polar_reflectance_std( + plotpath, vza, vaa, refl, wavelength, vmin=None, vmax=None, label=None +): + vaa_tol = 2 + vza_tol = 2 + + vaa = vaa % 360 + + vaa_grid = np.arange(8, 368, 15) + vza_grid = np.array([0, 5, 10, 20, 30, 40, 50, 60]) + + vaa_mesh, vza_mesh = np.meshgrid(np.radians(vaa_grid), vza_grid) + + refl_2d = np.zeros((len(vaa_grid), len(vza_grid))) + for i in range(len(vaa_grid)): + for j in range(len(vza_grid)): + id_series = np.where( + (np.abs(vaa - vaa_grid[i]) < vaa_tol) + & (np.abs(vza - vza_grid[j]) < vza_tol) + )[0] + if len(id_series) == 1: + refl_2d[i, j] = refl[id_series] + + elif len(id_series) > 1: + refl_2d[i, j] = np.std(refl[id_series]) + + refl_2d[refl_2d == 0] = np.nan + + fig = plt.figure() + ax = plt.subplot(1, 1, 1, projection="polar") + ax.set_theta_direction(-1) + ax.set_theta_offset(np.pi / 2.0) + im = ax.pcolormesh( + vaa_mesh, + vza_mesh, + refl_2d.T, + shading="auto", + cmap=plt.get_cmap("jet"), + vmin=vmin, + vmax=vmax, + ) + + cbar = fig.colorbar(im) + + if label is None: + cbar.set_label("reflectance at %s nm" % wavelength, rotation=270, labelpad=15) + else: + cbar.set_label(label, rotation=270, labelpad=15) + + fig.savefig(plotpath) + plt.close(fig) + + +def monthly_plotter(data, name): + mon = ["01", "02", "03", "04", "05", "06", "07", "08", "09", "10", "11", "12"] + tim = [ + 530, + 600, + 630, + 700, + 730, + 800, + 830, + 900, + 930, + 1000, + 1030, + 1100, + 1130, + 1200, + 1230, + 1300, + 1330, + 1400, + 1430, + 1500, + 1530, + 1600, + 1630, + 1700, + 1730, + 1800, + ] + for i in range(len(mon)): + data_monthly = data[data["month"] == mon[i]] + + if len(data_monthly) != 0: + for j in range(len(tim) - 1): + data_mon_time = data_monthly[ + (data_monthly["time"] > tim[j]) + & (data_monthly["time"] < tim[j + 1]) + ] + + if len(data_mon_time) != 0: + plot_polar_reflectance( + results_path + name + "_{}_{}_mean".format(mon[i], tim[j]), + data_mon_time[" vza"].values, + data_mon_time[" vaa"].values, + data_mon_time[" sza"].values, + data_mon_time[" saa"].values, + data_mon_time[" refl_550nm"].values, + 550, + ) + plot_polar_reflectance_std( + results_path + name + "_{}_{}_std".format(mon[i], tim[j]), + data_mon_time[" vza"].values, + data_mon_time[" vaa"].values, + data_mon_time[" refl_550nm"].values, + 550, + ) + + +# plot_polar_reflectance_std(results_path + 'test.png', data_subset[' vza'].values, data_subset[' vaa'].values, data_subset[' refl_550nm'].values, 550) +# monthly_plotter(data_19, '1_9') +# monthly_plotter(data_WWUKv3, 'WWUKv3') +aug = data_WWUKv3[data_WWUKv3["month"] == "08"] +sep = data_WWUKv3[data_WWUKv3["month"] == "09"] + + +def plot_month(data, plotname): + plot_polar_reflectance( + results_path + plotname, + data[" vza"].values, + data[" vaa"].values, + data[" sza"].values, + data[" saa"].values, + data[" refl_550nm"].values, + 550, + ) + + +plot_month(aug, "WWUKv3_aug.png") +plot_month(sep, "WWUKv3_sep.png") diff --git a/hypernets_processor/post_processing/post_processing_find_outliers.py b/hypernets_processor/post_processing/post_processing_find_outliers.py index 932a0f39..5762df99 100644 --- a/hypernets_processor/post_processing/post_processing_find_outliers.py +++ b/hypernets_processor/post_processing/post_processing_find_outliers.py @@ -1,6 +1,7 @@ """ post processing """ + import numpy as np import warnings import os @@ -10,7 +11,8 @@ mpl.use("Agg") import matplotlib.pyplot as plt import matplotlib.dates as mdates -myFmt = mdates.DateFormatter('%y-%m-%d') + +myFmt = mdates.DateFormatter("%y-%m-%d") import pysolar import datetime import glob @@ -33,13 +35,35 @@ plot_path = r"/home/data/insitu/hypernets/archive_qc_Mar2024/qc_plots" plotter = Plotting("", plot_path, ".png") -bad_flags=["pt_ref_invalid", "half_of_scans_masked", "not_enough_dark_scans", "not_enough_rad_scans", - "not_enough_irr_scans", "no_clear_sky_irradiance", "variable_irradiance", - "half_of_uncertainties_too_big", "discontinuity_VNIR_SWIR", "single_irradiance_used"] +bad_flags = [ + "pt_ref_invalid", + "half_of_scans_masked", + "not_enough_dark_scans", + "not_enough_rad_scans", + "not_enough_irr_scans", + "no_clear_sky_irradiance", + "variable_irradiance", + "half_of_uncertainties_too_big", + "discontinuity_VNIR_SWIR", + "single_irradiance_used", +] check_flags = ["single_irradiance_used"] -colors = ["black", "yellow", "cyan", "red", "green", "blue", "magenta", "orange", "navy", "gray", "brown", - "greenyellow", "purple"] +colors = [ + "black", + "yellow", + "cyan", + "red", + "green", + "blue", + "magenta", + "orange", + "navy", + "gray", + "brown", + "greenyellow", + "purple", +] wavs = [500, 900, 1100, 1600] hour_bins = [0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24] @@ -55,20 +79,23 @@ "PEAN", "DEGE", "LOBE", - "JAES" + "JAES", ] # sites = ["WWUK"] sites_thresh = [2, 2, 2, 2, 2, 2, 2, 2, 3, 3] -plot_against = ["acquisition_time",] +plot_against = [ + "acquisition_time", +] + +vza_min = 0 +vza_max = 5 -vza_min=0 -vza_max=5 +start_datetime = datetime.datetime.strptime("20220501T000000", "%Y%m%dT%H%M%S") +end_datetime = datetime.datetime.strptime("20221031T000000", "%Y%m%dT%H%M%S") -start_datetime=datetime.datetime.strptime("20220501T000000",'%Y%m%dT%H%M%S') -end_datetime=datetime.datetime.strptime("20221031T000000",'%Y%m%dT%H%M%S') +plot_min = 0 +plot_max = 0.5 -plot_min=0 -plot_max=0.5 def make_time_series_plot( wavs, @@ -209,7 +236,7 @@ def make_time_series_plot( ) valids = measurand_wav[np.where(mask == 0)[0]] if plot_min is not None and plot_max is not None: - plt.ylim([plot_min,plot_max]) + plt.ylim([plot_min, plot_max]) else: plt.ylim( [ @@ -222,7 +249,11 @@ def make_time_series_plot( plt.xlabel("datetime") plt.gca().xaxis.set_major_formatter(myFmt) plt.xticks(rotation=45) - plt.savefig(os.path.join(plot_path, "qc_%s_%s.png" % (tag, wavs[i])), dpi=300, bbox_inches="tight") + plt.savefig( + os.path.join(plot_path, "qc_%s_%s.png" % (tag, wavs[i])), + dpi=300, + bbox_inches="tight", + ) plt.clf() print("plot done ", os.path.join(plot_path, "qc_%s_%s.png" % (tag, wavs[i]))) return mask @@ -257,10 +288,10 @@ def extract_reflectances(files, wavs, vza, vaa, site): ds = read_hypernets_file( files[i], vza=vza, vaa=vaa, filter_flags=False, max_angle_tolerance=2 ) - if ds is None or len(ds.series)==0: + if ds is None or len(ds.series) == 0: mask[i] = 1 - times[i] = times[i-1] - #print("bad angle for file:", vza, vaa, files[i]) + times[i] = times[i - 1] + # print("bad angle for file:", vza, vaa, files[i]) continue flagged = DatasetUtil.get_flags_mask_or(ds["quality_flag"], bad_flags) @@ -273,7 +304,7 @@ def extract_reflectances(files, wavs, vza, vaa, site): ds.acquisition_time.values[0], ) - if (times[i]>start_datetime) and (times[i] start_datetime) and (times[i] < end_datetime): valid[i] = 1 if not flagged: @@ -282,15 +313,28 @@ def extract_reflectances(files, wavs, vza, vaa, site): # print(site,times[i],ds.quality_flag.values,[DatasetUtil.get_set_flags(flag) for flag in ds["quality_flag"]],files[i]) if flagged_check: - print("check",site,times[i],ds.quality_flag.values,[DatasetUtil.get_set_flags(flag) for flag in ds["quality_flag"]],files[i]) - + print( + "check", + site, + times[i], + ds.quality_flag.values, + [DatasetUtil.get_set_flags(flag) for flag in ds["quality_flag"]], + files[i], + ) else: if not any(flagged): mask[i] = 0 if any(flagged_check): - print("check",site,times[i],ds.quality_flag.values,[DatasetUtil.get_set_flags(flag) for flag in ds["quality_flag"]],files[i]) + print( + "check", + site, + times[i], + ds.quality_flag.values, + [DatasetUtil.get_set_flags(flag) for flag in ds["quality_flag"]], + files[i], + ) ids = [np.argmin(np.abs(ds.wavelength.values - wav)) for wav in wavs] refl[i] = np.mean(ds.reflectance.values[ids, :]) @@ -299,11 +343,12 @@ def extract_reflectances(files, wavs, vza, vaa, site): np.mean(ds.acquisition_time.values) ) - times=times[np.where(valid)] - refl=refl[np.where(valid)] - mask=mask[np.where(valid)] + times = times[np.where(valid)] + refl = refl[np.where(valid)] + mask = mask[np.where(valid)] return times, refl, mask + def read_hypernets_file( filepath, vza=None, @@ -372,7 +417,7 @@ def sigma_clip( # Remove NaNs from input values values = np.array(values) - mask[np.where(np.isnan(values))]=2 + mask[np.where(np.isnan(values))] = 2 # Continue loop until result converges diff = 10e10 @@ -450,7 +495,12 @@ def fit_2weekbins(xvals, yvals, mask): def vegetation_checks(ds, iseries): - print(ds["reflectance"].values.shape, ds.wavelength.values.shape, np.argmin(np.abs(ds.wavelength.values - 490)), iseries) + print( + ds["reflectance"].values.shape, + ds.wavelength.values.shape, + np.argmin(np.abs(ds.wavelength.values - 490)), + iseries, + ) b2 = ds["reflectance"].values[ np.argmin(np.abs(ds.wavelength.values - 490)), iseries ] # 490 nm @@ -504,16 +554,16 @@ def vegetation_checks(ds, iseries): for iseries in range(len(site_ds[0].viewing_zenith_angle.values)): vza = round(site_ds[0].viewing_zenith_angle.values[iseries]) vaa = round(site_ds[0].viewing_azimuth_angle.values[iseries]) - if (vzavza_max): + if (vza < vza_min) or (vza > vza_max): continue times, refl, mask = extract_reflectances(files, wavs, vza, vaa, site) if site == "WWUK" or site == "BASP": for ifile in range(len(site_ds)): - iseries_file=np.argmin(( - site_ds[ifile]["viewing_zenith_angle"].values - vza - ) ** 2 + ( - np.abs(site_ds[ifile]["viewing_azimuth_angle"].values - vaa) - ) ** 2) + iseries_file = np.argmin( + (site_ds[ifile]["viewing_zenith_angle"].values - vza) ** 2 + + (np.abs(site_ds[ifile]["viewing_azimuth_angle"].values - vaa)) + ** 2 + ) if not vegetation_checks(site_ds[ifile], iseries_file): mask[ifile] = 3 if True: @@ -524,6 +574,6 @@ def vegetation_checks(ds, iseries): mask, hour_bins, "%s_%s_%s" % (site, vza, vaa), - #n_max_points=sites_points[isite], + # n_max_points=sites_points[isite], sigma_thresh=sites_thresh[isite], - ) \ No newline at end of file + ) diff --git a/hypernets_processor/post_processing/post_processing_generate_csv.py b/hypernets_processor/post_processing/post_processing_generate_csv.py new file mode 100644 index 00000000..61d1864c --- /dev/null +++ b/hypernets_processor/post_processing/post_processing_generate_csv.py @@ -0,0 +1,248 @@ +import datetime + +import numpy as np +import hypernets_brdf_data_io as data_io +import os +import glob +import matplotlib.pyplot as plt + +# set appropriate folders (different for linux or windows) and settings + +# for windows: +# data_path = r"T:\ECO\EOServer\data\insitu\hypernets\archive" +# results_path = r"T:\ECO\EOServer\data\insitu\hypernets\post_processing_qc" + +# for eoserver: +os.environ['HDF5_USE_FILE_LOCKING'] = 'FALSE' +data_path = r"T:/ECO/EOServer/data/insitu/hypernets/archive" +results_path = r"T:/ECO/EOServer/data/insitu/hypernets/post_processing_qc" + +# results_path = os.path.join(results_path,brdf_model) +if not os.path.exists(results_path): + os.mkdir(results_path) + +# tags = ["JSIT","GHNA_v1", "GHNA_v3", 'WWUK_2024'] +# sites = ["JSIT","GHNA","GHNA", 'WWUK'] + +# start_times = ["20240409T0000","20220517T0000","20231026T0000", '20240101T0000'] +# stop_times = ["20250101T0000","20231018T0000","20250101T0000", '20250101T0000'] + +tags = ["GHNA_v3", "WWUK_2024"] +sites = ["GHNA", "WWUK"] + +start_times = ["20231026T0000", "20240101T0000"] +stop_times = ["20250101T0000", "20250101T0000"] + +wavelength = [415, 490, 550, 665, 675, 705, 740, 765, 842, 870, 1020, 1640] +plot_wavelength = 550 + + +# start_tod = ["0900","0930","1000","1030","1100"] +# stop_tod = ["0930","1000","1030","1100","1130"] +# +# start_tod = ["0900","1000","1100"] +# stop_tod = ["1000","1100","1200"] + +start_tod = [None] +stop_tod = [None] + +# #set minimum and maximum reflectance for colourbar plots (Set to None to let it work it out automatically) +vmin = None # 0.18 +vmax = None # 0.28 + +vzas = [None] # [20,30,40] +vaas = [None] # [83,98,113,263,278,298] + +SITE_PERIODS = { + "GHNA": [ + { + "start_date": "2022-05-17", + "stop_date": "2023-10-17", + "HYPSTAR_SN": 220261, + "comments": "v1 instrument", + }, + { + "start_date": "2023-10-26", + "stop_date": "present", + "HYPSTAR_SN": 222316, + "comments": "v3 instrument", + }, + ], + # "IFAR": [{"start_date": "2022-06-17", "stop_date": "2022-10-12" ,"HYPSTAR_SN": 221211, "comments": ""},{"start_date": "2023-08-04" , "stop_date": "2024-08-03" ,"HYPSTAR_SN": 221211, "comments": "Calibration E: 2023-01-25, L: 2023-01-27"}], + "JAES": [ + { + "start_date": "2023-04-14", + "stop_date": "2024-05-09", + "HYPSTAR_SN": 221461, + "comments": "2023-05-20 leaves on (subjective judgement); 2023-09-19 cleaned optics; 2023-10-18 first snow, leaves off", + }, + { + "start_date": "2024-06-05", + "stop_date": "2024-11-13", + "HYPSTAR_SN": 221461, + "comments": "", + }, + { + "start_date": "2024-11-13", + "stop_date": "present", + "HYPSTAR_SN": 222317, + "comments": "", + }, + ], + "JSIT": [ + { + "start_date": "2024-04-09", + "stop_date": "2025-07-01", + "HYPSTAR_SN": 222315, + "comments": "2024 soybean (sowing date 08/04/2023); 2024/2025 durum wheat", + }, + { + "start_date": "2025-07-01", + "stop_date": "present", + "HYPSTAR_SN": 222315, + "comments": "2024 soybean (sowing date 08/04/2023); 2024/2025 durum wheat", + } + ], + "LOBE": [ + { + "start_date": "2023-05-31", + "stop_date": "2023-08-11", + "lat": 50.55149, + "lon": 4.74591, + "HYPSTAR_SN": 222312, + "comments": "Potato crop, furrowed ground; Non-standard sequence used, all data has quality flag `vza_irradiance' raised", + }, + { + "start_date": "2024-05-17", + "stop_date": "2024-07-17", + "lat": 50.55151, + "lon": 4.74601, + "HYPSTAR_SN": 222312, + "comments": "Winter Wheat crop, Flat soil surface", + }, + { + "start_date": "2024-10-01", + "stop_date": "2024-11-13", + "lat": 50.55151, + "lon": 4.74601, + "HYPSTAR_SN": 222312, + "comments": "Cover crop, Flat soil surface", + }, + { + "start_date": "2025-04-15", + "stop_date": "present", + "lat": 50.55151, + "lon": 4.74601, + "HYPSTAR_SN": 222312, + "comments": "Sugar beet roots, Flat soil surface", + }, + ], + "PEAN": [ + { + "start_date": "2022-01-26", + "stop_date": "2022-02-08", + "HYPSTAR_SN": 221233, + "comments": "v1 instrument", + }, + { + "start_date": "2022-12-25", + "stop_date": "2023-02-02", + "HYPSTAR_SN": 221233, + "comments": "v1 instrument", + }, + { + "start_date": "2023-12-30", + "stop_date": "2024-01-23", + "HYPSTAR_SN": 222313, + "comments": "v1 instrument", + }, + ], + "WWUK": [ + { + "start_date": "2021-11-18", + "stop_date": "2024-01-16", + "HYPSTAR_SN": 220251, + "comments": "v1 instrument", + }, + { + "start_date": "2024-06-26", + "stop_date": "2024-12-31", + "HYPSTAR_SN": 222314, + "comments": "v3 instrument", + }, + { + "start_date": "2025-01-01", + "stop_date": "present", + "HYPSTAR_SN": 222314, + "comments": "v3 instrument", + }, + + ], +} + +for site in SITE_PERIODS.keys(): + for period in SITE_PERIODS[site]: + if site != 'JSIT': + continue + #if period["stop_date"] != "present": + # continue + print(site, period) + tag = "%s_%s_%s" % (site, period["start_date"], period["stop_date"]) + start_time = period["start_date"] + stop_time = period["stop_date"] + if stop_time == "present": + stop_time = datetime.datetime.now() + + files = glob.glob(os.path.join(data_path, site, "*", "*", "*", "*", "*L2A*.nc")) + print(files) + for ii in range(len(start_tod)): + files = data_io.filter_files_quick( + files, + start_time, + stop_time, + tod_start=start_tod[ii], + tod_stop=stop_tod[ii], + ) + print("selected", files) + for vza in vzas: + for vaa in vaas: + # read in hypernets data (which returns object of brdf_model.BRDFMeasurements) + HCRFmeas = data_io.read_data_hypernets( + files, i=None, vza=vza, vaa=vaa, mask=False + ) + + HCRFmeas.export_csv( + os.path.join( + results_path, + "%s_%s_%s_%s_%s.csv" + % (tag, vza, vaa, start_tod[ii], stop_tod[ii]), + ), + wavelength, + ) + + fig1, ax1 = plt.subplots() + ax1.plot( + HCRFmeas.get_datetimes(), + HCRFmeas.get_reflectance(plot_wavelength), + "o", + label="before", + ) + ax1.set_xlabel("datetime") + ax1.set_ylabel("reflectance") + ax1.legend(ncol=2) + ax1.set_ylim([vmin, vmax]) + fig1.savefig( + os.path.join( + results_path, + "refl_calibration_diff_%s_%s_%s_%s_%s_%s.png" + % ( + tag, + vza, + vaa, + start_tod[ii], + stop_tod[ii], + plot_wavelength, + ), + ) + ) + plt.clf() diff --git a/hypernets_processor/post_processing/post_processing_quality_checks.py b/hypernets_processor/post_processing/post_processing_quality_checks.py index fa034e71..c10f3d71 100644 --- a/hypernets_processor/post_processing/post_processing_quality_checks.py +++ b/hypernets_processor/post_processing/post_processing_quality_checks.py @@ -1,6 +1,7 @@ """ post processing """ + import numpy as np import warnings import os @@ -10,7 +11,8 @@ mpl.use("Agg") import matplotlib.pyplot as plt import matplotlib.dates as mdates -myFmt = mdates.DateFormatter('%y-%m-%d') + +myFmt = mdates.DateFormatter("%y-%m-%d") import pysolar import datetime import glob @@ -33,13 +35,35 @@ plot_path = r"/home/data/insitu/hypernets/archive_qc_Jan2024/qc_plots" plotter = Plotting("", plot_path, ".png") -bad_flags=["pt_ref_invalid", "half_of_scans_masked", "not_enough_dark_scans", "not_enough_rad_scans", - "not_enough_irr_scans", "no_clear_sky_irradiance", "variable_irradiance", - "half_of_uncertainties_too_big", "discontinuity_VNIR_SWIR", "single_irradiance_used"] +bad_flags = [ + "pt_ref_invalid", + "half_of_scans_masked", + "not_enough_dark_scans", + "not_enough_rad_scans", + "not_enough_irr_scans", + "no_clear_sky_irradiance", + "variable_irradiance", + "half_of_uncertainties_too_big", + "discontinuity_VNIR_SWIR", + "single_irradiance_used", +] check_flags = ["single_irradiance_used"] -colors = ["black", "yellow", "cyan", "red", "green", "blue", "magenta", "orange", "navy", "gray", "brown", - "greenyellow", "purple"] +colors = [ + "black", + "yellow", + "cyan", + "red", + "green", + "blue", + "magenta", + "orange", + "navy", + "gray", + "brown", + "greenyellow", + "purple", +] def make_time_series_plot( @@ -191,7 +215,11 @@ def make_time_series_plot( plt.xlabel("datetime") plt.gca().xaxis.set_major_formatter(myFmt) plt.xticks(rotation=45) - plt.savefig(os.path.join(plot_path, "qc_%s_%s.png" % (tag, wavs[i])), dpi=300, bbox_inches="tight") + plt.savefig( + os.path.join(plot_path, "qc_%s_%s.png" % (tag, wavs[i])), + dpi=300, + bbox_inches="tight", + ) plt.clf() print("plot done ", os.path.join(plot_path, "qc_%s_%s.png" % (tag, wavs[i]))) return mask @@ -226,10 +254,10 @@ def extract_reflectances(files, wavs, vza, vaa, site): ds = read_hypernets_file( files[i], vza=vza, vaa=vaa, filter_flags=False, max_angle_tolerance=2 ) - if ds is None or len(ds.series)==0: + if ds is None or len(ds.series) == 0: mask[i] = 1 - times[i] = times[i-1] - #print("bad angle for file:", vza, vaa, files[i]) + times[i] = times[i - 1] + # print("bad angle for file:", vza, vaa, files[i]) continue flagged = DatasetUtil.get_flags_mask_or(ds["quality_flag"], bad_flags) @@ -248,15 +276,28 @@ def extract_reflectances(files, wavs, vza, vaa, site): # print(site,times[i],ds.quality_flag.values,[DatasetUtil.get_set_flags(flag) for flag in ds["quality_flag"]],files[i]) if flagged_check: - print("check",site,times[i],ds.quality_flag.values,[DatasetUtil.get_set_flags(flag) for flag in ds["quality_flag"]],files[i]) - + print( + "check", + site, + times[i], + ds.quality_flag.values, + [DatasetUtil.get_set_flags(flag) for flag in ds["quality_flag"]], + files[i], + ) else: if not any(flagged): mask[i] = 0 if any(flagged_check): - print("check",site,times[i],ds.quality_flag.values,[DatasetUtil.get_set_flags(flag) for flag in ds["quality_flag"]],files[i]) + print( + "check", + site, + times[i], + ds.quality_flag.values, + [DatasetUtil.get_set_flags(flag) for flag in ds["quality_flag"]], + files[i], + ) ids = [np.argmin(np.abs(ds.wavelength.values - wav)) for wav in wavs] refl[i] = np.mean(ds.reflectance.values[ids, :]) @@ -266,6 +307,7 @@ def extract_reflectances(files, wavs, vza, vaa, site): ) return times, refl, mask + def read_hypernets_file( filepath, vza=None, @@ -334,7 +376,7 @@ def sigma_clip( # Remove NaNs from input values values = np.array(values) - mask[np.where(np.isnan(values))]=2 + mask[np.where(np.isnan(values))] = 2 # Continue loop until result converges diff = 10e10 @@ -412,7 +454,12 @@ def fit_2weekbins(xvals, yvals, mask): def vegetation_checks(ds, iseries): - print(ds["reflectance"].values.shape, ds.wavelength.values.shape, np.argmin(np.abs(ds.wavelength.values - 490)), iseries) + print( + ds["reflectance"].values.shape, + ds.wavelength.values.shape, + np.argmin(np.abs(ds.wavelength.values - 490)), + iseries, + ) b2 = ds["reflectance"].values[ np.argmin(np.abs(ds.wavelength.values - 490)), iseries ] # 490 nm @@ -459,7 +506,7 @@ def vegetation_checks(ds, iseries): "PEAN", "DEGE", "LOBE", - "JAES" + "JAES", ] # sites = ["WWUK"] sites_thresh = [2, 2, 2, 2, 2, 2, 2, 2, 3, 3] @@ -483,20 +530,20 @@ def vegetation_checks(ds, iseries): (site_ds[ifile].wavelength > 380) & (site_ds[ifile].wavelength < 1700) )[0] site_ds[ifile] = site_ds[ifile].isel(wavelength=ids_wav) - #if site == "BASP": - #ids_series = np.where((site_ds[ifile]["viewing_zenith_angle"] < 35))[0] - #site_ds[ifile] = site_ds[ifile].isel(series=ids_series) - # for vza in [5, 10]: - # for vaa in [263, 273, 293]: - # angledif_series = ( - # site_ds[ifile]["viewing_zenith_angle"].values - vza - # ) ** 2 + ( - # np.abs(site_ds[ifile]["viewing_azimuth_angle"].values - vaa) - # ) ** 2 - # ids_series = np.where( - # angledif_series != np.min(angledif_series) - # )[0] - # site_ds[ifile] = site_ds[ifile].isel(series=ids_series) + # if site == "BASP": + # ids_series = np.where((site_ds[ifile]["viewing_zenith_angle"] < 35))[0] + # site_ds[ifile] = site_ds[ifile].isel(series=ids_series) + # for vza in [5, 10]: + # for vaa in [263, 273, 293]: + # angledif_series = ( + # site_ds[ifile]["viewing_zenith_angle"].values - vza + # ) ** 2 + ( + # np.abs(site_ds[ifile]["viewing_azimuth_angle"].values - vaa) + # ) ** 2 + # ids_series = np.where( + # angledif_series != np.min(angledif_series) + # )[0] + # site_ds[ifile] = site_ds[ifile].isel(series=ids_series) for iseries in range(len(site_ds[0].viewing_zenith_angle.values)): vza = round(site_ds[0].viewing_zenith_angle.values[iseries]) @@ -504,11 +551,11 @@ def vegetation_checks(ds, iseries): times, refl, mask = extract_reflectances(files, wavs, vza, vaa, site) if site == "WWUK" or site == "BASP": for ifile in range(len(site_ds)): - iseries_file=np.argmin(( - site_ds[ifile]["viewing_zenith_angle"].values - vza - ) ** 2 + ( - np.abs(site_ds[ifile]["viewing_azimuth_angle"].values - vaa) - ) ** 2) + iseries_file = np.argmin( + (site_ds[ifile]["viewing_zenith_angle"].values - vza) ** 2 + + (np.abs(site_ds[ifile]["viewing_azimuth_angle"].values - vaa)) + ** 2 + ) if not vegetation_checks(site_ds[ifile], iseries_file): mask[ifile] = 3 if True: @@ -519,28 +566,28 @@ def vegetation_checks(ds, iseries): mask, hour_bins, "%s_%s_%s" % (site, vza, vaa), - #n_max_points=sites_points[isite], + # n_max_points=sites_points[isite], sigma_thresh=sites_thresh[isite], ) for ifile in range(len(site_ds)): ds_curr = site_ds[ifile] - ds_curr.quality_flag.attrs[ - "flag_meanings" - ] = ds_curr.quality_flag.attrs["flag_meanings"].replace( - "placeholder1", "postprocessing_outliers" + ds_curr.quality_flag.attrs["flag_meanings"] = ( + ds_curr.quality_flag.attrs["flag_meanings"].replace( + "placeholder1", "postprocessing_outliers" + ) ) if site == "WWUK": - ds_curr.quality_flag.attrs[ - "flag_meanings" - ] = ds_curr.quality_flag.attrs["flag_meanings"].replace( - "placeholder1", "postprocessing_outliers" + ds_curr.quality_flag.attrs["flag_meanings"] = ( + ds_curr.quality_flag.attrs["flag_meanings"].replace( + "placeholder1", "postprocessing_outliers" + ) ) - iseries_file = np.argmin(( - site_ds[ifile]["viewing_zenith_angle"].values - vza - ) ** 2 + ( - np.abs(site_ds[ifile]["viewing_azimuth_angle"].values - vaa) - ) ** 2) + iseries_file = np.argmin( + (site_ds[ifile]["viewing_zenith_angle"].values - vza) ** 2 + + (np.abs(site_ds[ifile]["viewing_azimuth_angle"].values - vaa)) + ** 2 + ) if mask2[ifile] > 0: ds_curr.reflectance[:, iseries_file] *= np.nan ds_curr.u_rel_random_reflectance[:, iseries_file] *= np.nan @@ -560,7 +607,11 @@ def vegetation_checks(ds, iseries): if files_nmaskedseries[ifile] < len(site_ds[ifile].series) / 2: print(files_nmaskedseries[ifile], os.path.basename(files[ifile])) site_ds[ifile].to_netcdf( - os.path.join(out_path, site, os.path.basename(files[ifile]).replace("L2A","L2B")) + os.path.join( + out_path, + site, + os.path.basename(files[ifile]).replace("L2A", "L2B"), + ) ) f.write( "%s,%s,%s,%s \n" diff --git a/hypernets_processor/post_processing/produce_clear_sky_models.py b/hypernets_processor/post_processing/produce_clear_sky_models.py new file mode 100644 index 00000000..f9013969 --- /dev/null +++ b/hypernets_processor/post_processing/produce_clear_sky_models.py @@ -0,0 +1,407 @@ +import numpy as np +from brdf_model import BRDFModelFactory +import hypernets_brdf_data_io as data_io +from brdf_model import Comparisons +from curepy import MCMCRetrieval, plot_trace, plot_corner +import os +import glob +import matplotlib.pyplot as plt +from pysolar.solar import * +from forts.utils.produce_example_1d import ProduceExample1D +import pytz +import xarray as xr +import pandas as pd +from scrappi.interface import ( + set_credentials, + perform_query, + make_query_with_tolerance, + download_product, + list_satellite_products_api, + make_fs, + generate_bounding_lat_lon, +) + +from scrappi.utils.download_utils import download_and_read_ERA5, download_and_read_CAMS + +# for windows: +data_path = r"T:\ECO\EOServer\data\insitu\hypernets\archive" +results_path = r"T:\ECO\EOServer\data\insitu\hypernets\post_processing_qc\irradiance" + +# for eoserver: +data_path = r"/mnt/t/data/insitu/hypernets/archive" +results_path = r"/mnt/t/data/insitu/hypernets/post_processing_qc/irradiance" + +if not os.path.exists(results_path): + os.mkdir(results_path) + +# SITE_LOCATIONS = { +# "ATGE": [52.466778, 12.959778], +# # "BASP": [39.049139, -2.075917], +# "DEGE": [53.868278, 13.268556], +# "GHNA": [-23.60153, 15.12589], +# "IFAR": [-34.592322, -58.479017], +# "JAES": [58.281975, 27.312959], +# "JSIT": [44.874305, 11.979201], +# "LOBE": [50.551493, 4.745911], +# "PEAN": [-71.940128, 23.305260], +# "WWUK": [51.777206,-1.338494], +# } + +SITE_LOCATIONS = { + # "ATGE": [52.466778, 12.959778], + # # "BASP": [39.049139, -2.075917], + # "DEGE": [53.868278, 13.268556], + "GHNA": [-23.60153, 15.12589], + # "IFAR": [-34.592322, -58.479017], + "JAES": [58.281975, 27.312959], + "JSIT": [44.874305, 11.979201], + "LOBE": [50.551493, 4.745911], + "PEAN": [-71.940128, 23.305260], + "WWUK": [51.777206, -1.338494], +} + +SITE_ALTITUDE = { + "ATGE": 0.045, + "DEGE": 0.022, + "GHNA": 0.510, + "JSIT": 0, + "JAES": 0.045, + "IFAR": 0.021, + "LOBE": 0.168, + "PEAN": 1.297, + "WWUK": 0.125, +} + +atmosphere_type = { + "ATGE": "midlatitude_summer", + "BASP": "midlatitude_summer", + "DEGE": "midlatitude_summer", + "GHNA": "midlatitude_summer", + "IFAR": "midlatitude_summer", + "JAES": "midlatitude_summer", + "JSIT": "midlatitude_summer", + "LOBE": "midlatitude_summer", + "PEAN": "midlatitude_summer", + "WWUK": "midlatitude_summer", +} + +aerosol_type = { + "ATGE": "continental_average", + "BASP": "continental_average", + "DEGE": "continental_average", + "GHNA": "desert", + "JSIT": "continental_average", + "JAES": "continental_average", + "IFAR": "continental_average", + "LOBE": "continental_average", + "PEAN": "antarctic", + "WWUK": "continental_average", +} + +file_paths = { + # "ATGE": [52.466778, 12.959778], + # "BASP": [39.049139, -2.075917], + # "DEGE": [53.868278, 13.268556], + "GHNA": os.path.join( + data_path, + "GHNA", + "2023", + "10", + "28", + "SEQ20231028T090128", + "HYPERNETS_L_GHNA_L2A_REF_20231028T0901_20240124T1403_v2.0.nc", + ), + # "IFAR": [-34.592322, -58.479017], + "JAES": os.path.join( + data_path, + "JAES", + "2024", + "08", + "08", + "SEQ20240808T090049", + "HYPERNETS_L_JAES_L2A_REF_20240808T0900_20240906T0650_v2.1.nc", + ), + "JSIT": os.path.join( + data_path, + "JSIT", + "2024", + "08", + "07", + "SEQ20240807T090056", + "HYPERNETS_L_JSIT_L2A_REF_20240807T0900_20240807T1054_v2.0.nc", + ), + "LOBE": os.path.join( + data_path, + "LOBE", + "2024", + "07", + "11", + "SEQ20240711T100125", + "HYPERNETS_L_LOBE_L2A_REF_20240711T1001_20240711T1423_v2.0.nc", + ), + "PEAN": os.path.join( + data_path, + "PEAN", + "2022", + "01", + "29", + "SEQ20220129T070017", + "HYPERNETS_L_PEAN_L2A_REF_20220129T0700_20240112T0725_v2.0.nc", + ), + "WWUK": os.path.join( + data_path, + "WWUK", + "2022", + "04", + "26", + "SEQ20220426T113243", + "HYPERNETS_L_WWUK_L2A_REF_20220426T1132_20240101T1641_v2.0.nc", + ), +} + +file_paths_irr = { + "GHNA": os.path.join( + data_path, + "GHNA", + "2023", + "10", + "28", + "SEQ20231028T090128", + "HYPERNETS_L_GHNA_L1B_IRR_20231028T0901_20240124T1403_v2.0.nc", + ), + # "IFAR": [-34.592322, -58.479017], + "JAES": os.path.join( + data_path, + "JAES", + "2024", + "08", + "08", + "SEQ20240808T090049", + "HYPERNETS_L_JAES_L1B_IRR_20240808T0900_20240906T0650_v2.1.nc", + ), + "JSIT": os.path.join( + data_path, + "JSIT", + "2024", + "08", + "07", + "SEQ20240807T090056", + "HYPERNETS_L_JSIT_L1B_IRR_20240807T0900_20240807T1054_v2.0.nc", + ), + "LOBE": os.path.join( + data_path, + "LOBE", + "2024", + "07", + "11", + "SEQ20240711T100125", + "HYPERNETS_L_LOBE_L1B_IRR_20240711T1001_20240711T1423_v2.0.nc", + ), + "PEAN": os.path.join( + data_path, + "PEAN", + "2022", + "01", + "29", + "SEQ20220129T070017", + "HYPERNETS_L_PEAN_L1B_IRR_20220129T0700_20240112T0725_v2.0.nc", + ), + "WWUK": os.path.join( + data_path, + "WWUK", + "2022", + "04", + "26", + "SEQ20220426T113243", + "HYPERNETS_L_WWUK_L1B_IRR_20220426T1132_20240101T1641_v2.0.nc", + ), +} +start_time = "20220101T0000" +stop_time = "20230101T0000" + +run_RT = True + +tag = "irrwav" +tag = "_" + tag + + +def combine_direct_to_diffuse_ratio_sza( + site, + aod, + irr_files_path=None, + median_aod=False, + tag="", +): + files = glob.glob( + os.path.join(irr_files_path, "irr_clear_sky_%s*%s.nc" % (site, tag)) + ) + files.sort() + direct_to_diffuse_all = [xr.open_dataset(file) for file in files] + direct_to_diffuse_subset = [ + direct_to_diffuse + for direct_to_diffuse in direct_to_diffuse_all + if direct_to_diffuse.attrs["aod"] == aod + ] + szas = [ + direct_to_diffuse.attrs["sza"] for direct_to_diffuse in direct_to_diffuse_subset + ] + comb_ds = xr.concat(direct_to_diffuse_subset, pd.Index(szas, name="sza")) + comb_ds.attrs["aod"] = aod + if median_aod: + comb_ds.to_netcdf( + os.path.join(irr_files_path, "%s_clear_sky_medianaod%s.nc" % (site, tag)) + ) + else: + comb_ds.to_netcdf( + os.path.join(irr_files_path, "%s_clear_sky_aod%s%s.nc" % (site, aod, tag)) + ) + return comb_ds + + +def interpolate_irradiance_sza(sza, ds_irr): + ds_irr_temp = ds_irr.copy() + ds_irr_temp["solar_irradiance_BOA"].values = ( + ds_irr_temp["solar_irradiance_BOA"].values + / np.cos(ds_irr_temp["sza"].values / 180 * np.pi)[:, None] + ) + ds_irr_temp = ds_irr_temp.interp(sza=sza, method="linear") + ds_irr_temp["solar_irradiance_BOA"].values = ds_irr_temp[ + "solar_irradiance_BOA" + ].values * np.cos(ds_irr_temp["sza"].values / 180 * np.pi) + return ds_irr_temp["solar_irradiance_BOA"].values + + +for site in SITE_LOCATIONS.keys(): + lat, lon = SITE_LOCATIONS[site] + + # query = { + # "collections": ["COP_DEM_GLO30_DTED"], + # "geom": [lat - 0.25, lon - 0.25, lat + 0.25, lon + 0.25], + # # making sure there are at least 2 grid points in query + # } + # + # products = perform_query( + # "eodag", + # query, + # config_dict={"config_file_path": os.path.join(results_path, "eodag.yml")}, + # ) + # + # products.set_fs("t-drive") + + # path = download_product( + # "eodag", + # products, + # config_dict={"config_file_path": os.path.join(results_path, "eodag.yml")}, + # ) + # print(path) + + path_era5 = os.path.join( + results_path, "ds_era5_%s_%s_%s.nc" % (site, start_time, stop_time) + ) + if os.path.exists(path_era5): + ds_era5 = xr.open_dataset(path_era5) + else: + ds_era5 = download_and_read_ERA5( + start_time, + stop_time, + lat, + lon, + config_path=os.path.join(results_path, "eodag.yml"), + ) + + ds_era5.to_netcdf(path_era5) + + path_cams = os.path.join( + results_path, "ds_cams_%s_%s_%s.nc" % (site, start_time, stop_time) + ) + if os.path.exists(path_cams): + ds_cams = xr.open_dataset(path_cams) + else: + ds_cams = download_and_read_CAMS( + start_time, + stop_time, + lat, + lon, + config_path=os.path.join(results_path, "eodag.yml"), + ) + ds_cams.to_netcdf(path_cams) + + if run_RT: + + ds_HYP = data_io.read_hypernets_file( + file_paths[site], + vza=20, + ) + ds_HYP = ds_HYP.mean(dim="series") + + ds_HYP_irr = data_io.read_hypernets_file( + file_paths_irr[site], + ) + ds_HYP_irr = ds_HYP_irr.mean(dim="series") + + atmosphere_ex = {} + atmosphere_ex["shape"] = None + atmosphere_ex["atmosphere_type"] = atmosphere_type[site] + atmosphere_ex["aerosol_type"] = aerosol_type[ + site + ] # by default aerosol type is urban + atmosphere_ex["AOD"] = -99 # by default aerosol type is urban + atmosphere_ex["H2O"] = -99 # by default aerosol type is urban + atmosphere_ex["O3"] = -99 # by default aerosol type is urban + atmosphere_ex["CH4"] = -99 # by default aerosol type is urban + atmosphere_ex["CO2"] = -99 # by default aerosol type is urban + atmosphere_ex["P"] = -99 # by default aerosol type is urban + + alt = SITE_ALTITUDE[site] + + median_aod = np.median(ds_cams.aod550.values) + + for aod in [0.0, 0.1, 0.2, 0.3, median_aod]: + for szai in np.arange(0, 90, 10): + if os.path.exists( + os.path.join( + results_path, + "irr_clear_sky_%s_%s_%s%s.nc" % (site, aod, szai, tag), + ) + ): + continue + else: + forts_prod = ProduceExample1D( + resolution="coarse", atmosphere=atmosphere_ex + ) + print(ds_HYP_irr.wavelength.values) + ds_irr = forts_prod.produce_solar_irradiance_hypernets( + ds_HYP, + szai, + aod=aod, + h2o=ds_era5.tcwv.values.mean(), + O3=ds_era5.tco3.values.mean() / (2.1415 * 10 ** (-5)), + P=ds_era5.sp.values.mean() / 100, + altitude=alt, + band_centres=ds_HYP_irr.wavelength.values, + ) + ds_irr.attrs["h2o"] = ds_era5.tcwv.values.mean() + ds_irr.attrs["O3"] = ds_era5.tco3.values.mean() / ( + 2.1415 * 10 ** (-5) + ) + ds_irr.attrs["P"] = ds_era5.sp.values.mean() / 100 + ds_irr.attrs["aod"] = aod + ds_irr.attrs["sza"] = szai + ds_irr.attrs["altitude"] = alt + + ds_irr.to_netcdf( + os.path.join( + results_path, + "irr_clear_sky_%s_%s_%s%s.nc" % (site, aod, szai, tag), + ) + ) + + if aod == median_aod: + comb_irr_ds = combine_direct_to_diffuse_ratio_sza( + site, aod, results_path, median_aod=True, tag=tag + ) + else: + comb_irr_ds = combine_direct_to_diffuse_ratio_sza( + site, aod, results_path, tag=tag + ) + interpolate_irradiance_sza(13.5, comb_irr_ds) diff --git a/hypernets_processor/post_processing/zenodo_generate_csv.py b/hypernets_processor/post_processing/zenodo_generate_csv.py new file mode 100644 index 00000000..1dc65349 --- /dev/null +++ b/hypernets_processor/post_processing/zenodo_generate_csv.py @@ -0,0 +1,92 @@ +import datetime + +import numpy as np +import os +import glob +import matplotlib.pyplot as plt +import xarray as xr +from hypernets_processor.utils.utils import convert_datetime + +# set appropriate folders (different for linux or windows) and settings + +local_fs=True + +if local_fs: + # for local windows machine + data_path = r"C:\Users\pdv\data\insitu\hypernets\archive_L2B" + results_path = r"C:\Users\pdv\data\insitu\hypernets\postprocessing\zenodo" + +# for windows: +# data_path = r"\\eoserver\home\data\insitu\hypernets\archive" +# results_path = r"\\eoserver\home\data\insitu\hypernets\archive" + +# for eoserver: +# data_path = r"/home/data/insitu/hypernets/archive" +# results_path = r"/mnt/t/data/insitu/hypernets/archive" + +# results_path = os.path.join(results_path,brdf_model) +if not os.path.exists(results_path): + os.mkdir(results_path) + +sites = ["LOBE", "JAES","JSIT", "GHNA", "WWUK"] + +start_times = ["20220101T0000", "20220101T0000","20220101T0000","20220101T0000","20220101T0000"] +stop_times = ["20251001T0000", "20251001T0000","20251001T0000","20251001T0000","20251001T0000"] + +SITE_PERIODS = { + "ATGE": [{"start_date": "2022-10-16" , "stop_date": "2024-05-02" ,"HYPSTAR_SN": 221231, "valid": False, "comments": ""}], + "BASP": [{"start_date": "2022-07-18" , "stop_date": "2022-07-22" ,"HYPSTAR_SN": 221233, "valid": False, "comments": "v2 instrument"}], + "DEGE": [{"start_date": "2021-07-29" , "stop_date": "2023-10-24" ,"HYPSTAR_SN": 220241, "valid": False, "comments": ""}], + "GHNA": [{"start_date": "2022-05-17" , "stop_date": "2023-10-17" ,"HYPSTAR_SN": 220261, "valid": True, "comments": "v1 instrument"},{"start_date": "2023-10-20" , "stop_date": "2024-05-23" ,"HYPSTAR_SN": 222316, "valid": True, "comments": "v3 instrument"},{"start_date": "2024-05-24" , "stop_date": "2025-03-27" ,"HYPSTAR_SN": 222316, "valid": True, "comments": "instrument cleaned and realigned but not replaced/calibrated"},{"start_date": "2025-03-27" , "stop_date": "2025-09-27" ,"HYPSTAR_SN": 222316, "valid": False, "comments": "Grassy period following rain event (also affected by some pan angle offset error)"}, {"start_date": "2025-10-15" , "stop_date": "present" ,"HYPSTAR_SN": 220251, "valid": False, "comments": "instrument replaced, not enough data yet for valid L2B QC"}], + "IFAR": [{"start_date": "2022-06-17", "stop_date": "2022-10-12" ,"HYPSTAR_SN": 221211, "valid": False, "comments": ""},{"start_date": "2023-08-04" , "stop_date": "2024-08-03" ,"HYPSTAR_SN": 221211, "valid": False, "comments": "Calibration E: 2023-01-25, L: 2023-01-27"}], + "JAES": [{"start_date": "2023-04-14", "stop_date": "2023-05-11" ,"HYPSTAR_SN": 221461, "valid": False, "comments": "invalid period for L2B due to site heterogeneity, 2023-05-20 leaves on (subjective judgement); 2023-09-19 cleaned optics; 2023-10-18 first snow, leaves off"},{"start_date": "2023-05-11", "stop_date": "2023-09-07" ,"HYPSTAR_SN": 221461, "valid": True, "comments": "2023-05-20 leaves on (subjective judgement); 2023-09-19 cleaned optics; 2023-10-18 first snow, leaves off"},{"start_date": "2023-09-07", "stop_date": "2024-05-09" ,"HYPSTAR_SN": 221461, "valid": False, "comments": "invalid period for L2B due to site heterogeneity, 2023-05-20 leaves on (subjective judgement); 2023-09-19 cleaned optics; 2023-10-18 first snow, leaves off"},{"start_date": "2024-06-05", "stop_date": "2024-09-04" ,"HYPSTAR_SN": 221461, "valid": True, "comments": "homogeneous period"}, {"start_date": "2024-09-04", "stop_date": "2024-11-13" ,"HYPSTAR_SN": 221461, "valid": False, "comments": "invalid period for cal/val due to site heterogeneity"}, {"start_date": "2024-11-13", "stop_date": "2025-05-01" ,"HYPSTAR_SN": 222317, "valid": False, "comments": "invalid period for cal/val due to site heterogeneity"},{"start_date": "2025-05-01", "stop_date": "2025-06-16" ,"HYPSTAR_SN": 222317, "valid": True, "comments": "homogeneous period"}, {"start_date": "2025-06-16", "stop_date": "2025-07-04" ,"HYPSTAR_SN": 221461, "valid": False, "comments": "invalid due to site heterogeneity, swapped radiometer (was 222317, now 221461); LED source (was 422361 (but reported 3230503), now 323051); rain sensor and monitor PD assembly"},{"start_date": "2025-07-04", "stop_date": "2025-09-03" ,"HYPSTAR_SN": 221461, "valid": True, "comments": "swapped radiometer (was 222317, now 221461); LED source (was 422361 (but reported 3230503), now 323051); rain sensor and monitor PD assembly"}, {"start_date": "2025-09-03", "stop_date": "present" ,"HYPSTAR_SN": 221461, "valid": False, "comments": "invalid due to site heterogeneity, swapped radiometer (was 222317, now 221461); LED source (was 422361 (but reported 3230503), now 323051); rain sensor and monitor PD assembly"}], + "JSIT": [{"start_date": "2024-04-09", "stop_date": "2024-11-16" ,"HYPSTAR_SN": 222315, "valid": False, "comments": "soybean (sowing date 08/04/2023); lot of spatial variability (likely due to field disturbance during deployment) resulting in this data not being suitable for satellite validation."},{"start_date": "2024-11-27", "stop_date": "2025-03-10" ,"HYPSTAR_SN": 222315, "valid": False, "comments": "growing period, unsuitable for cal/val due to site heterogeneity, durum wheat"},{"start_date": "2025-03-10", "stop_date": "2025-07-01" ,"HYPSTAR_SN": 222315, "valid": True, "comments": "durum wheat; Anti-fungal treatments on April 2nd, 2025 between 15:00 and 15:30; Field has been affected by lodging starting from May 8th-9th, 2025"}, {"start_date": "2025-07-02", "stop_date": "2025-08-01" ,"HYPSTAR_SN": 222315, "valid": True, "comments": "fallow field with residues/weeds."}, {"start_date": "2025-08-01", "stop_date": "present" ,"HYPSTAR_SN": 222315, "valid": False, "comments": "fallow field with residues/weeds. Period too short for QC and site heterogeneous."}], + "LOBE": [{"start_date": "2023-05-31", "stop_date": "2023-08-11", "lat": 50.55149, "lon": 4.74591, "HYPSTAR_SN": 222312, "valid": False, "comments": "Potato crop, furrowed ground; Non-standard sequence used, all data has quality flag `vza_irradiance' raised, not enough data for L2B processing"}, {"start_date": "2024-05-17", "stop_date": "2024-07-17", "lat": 50.55151 , "lon": 4.74601,"HYPSTAR_SN": 222312, "valid": True, "comments": "Winter Wheat crop, Flat soil surface"}, {"start_date": "2024-10-05", "stop_date": "2024-11-13", "lat": 50.55151 , "lon": 4.74601,"HYPSTAR_SN": 222312, "valid": True, "comments": "Cover crop, Flat soil surface"}, {"start_date": "2025-04-15", "stop_date": "2025-05-03", "lat": 50.55151 , "lon": 4.74601,"HYPSTAR_SN": 222312, "valid": True, "comments": "Bare soil surface"}, {"start_date": "2025-05-03", "stop_date": "2025-06-03", "lat": 50.55151 , "lon": 4.74601,"HYPSTAR_SN": 222312, "valid": False, "comments": "Growing season, site heterogeneity too high for L2B processing"}, {"start_date": "2025-06-03", "stop_date": "present", "lat": 50.55151 , "lon": 4.74601,"HYPSTAR_SN": 222312, "valid": True, "comments": "Sugar beet roots, Flat soil surface"}], + "PEAN": [{"start_date": "2022-01-26" , "stop_date": "2022-02-08" ,"HYPSTAR_SN": 221233, "valid": False, "comments": "Disturbed surface due to deployment. A crossbar for additional stability was installed on 2022-01-31. Improvement of north orientation was attempted 2022-02-01. Powered by 10 ft container equipped with solar and backup generator, located south of the instrument (visible in S2 imagery)."},{"start_date": "2022-12-25" , "stop_date": "2023-02-02" ,"HYPSTAR_SN": 221233, "valid": False, "comments": "Natural snow surface, with small scale relief changing with wind and snow. Powered by 10 ft container equipped with solar and backup generator, located south of the instrument (visible in S2 imagery)."},{"start_date": "2023-12-30" , "stop_date": "2024-01-23" ,"HYPSTAR_SN": 222313, "valid": False, "comments": "Natural snow surface, with small scale relief changing with wind and snow. Power to the instrument was flaky in the first few days, with several measurements/days missing. More reliable measurements from 2024-01-09."}], + "WWUK": [{"start_date": "2021-11-18" , "stop_date": "2022-05-05" ,"HYPSTAR_SN": 220251, "valid": False, "comments": "v1 instrument, winter data too heterogenious"} ,{"start_date": "2022-05-05" , "stop_date": "2022-10-11" ,"HYPSTAR_SN": 220251, "valid": True, "comments": "v1 instrument, summer"} ,{"start_date": "2022-10-11" , "stop_date": "2023-04-29" ,"HYPSTAR_SN": 220251, "valid": False, "comments": "v1 instrument, winter data too heterogenious"} ,{"start_date": "2023-04-29" , "stop_date": "2023-11-01" ,"HYPSTAR_SN": 220251, "valid": True, "comments": "v1 instrument, summer"}, {"start_date": "2023-11-01" , "stop_date": "2024-06-26" ,"HYPSTAR_SN": 220251, "valid": False, "comments": "v1 instrument, winter data too heterogenious"},{"start_date": "2024-06-26" , "stop_date": "present" ,"HYPSTAR_SN": 222314, "valid": False, "comments": "v3 instrument, too few clear-sky measurements for reliable QC"}], +} + +for site in SITE_PERIODS.keys(): + with open(os.path.join(results_path, "Zenodo_L2B_sequences_%s.csv" % site), "w") as f: + f.write( + "#Sequence_ID,filename,start_acquisition_time,stop_acquisition_time,latitude,longitude,period comment\n" + ) + for period in SITE_PERIODS[site]: + tag = "%s_%s_%s" % (site, period["start_date"], period["stop_date"]) + start_time = convert_datetime(period["start_date"]) + stop_time = convert_datetime(period["stop_date"]) + + if local_fs: + files = glob.glob( + os.path.join(data_path, site, "*L2B*.nc") + ) + else: + files = glob.glob( + os.path.join(data_path, site, "*", "*", "*", "*", "*L2B*.nc") + ) + for file in files: + ds_HYP = xr.open_dataset(file) + dt_hyp_min = convert_datetime(ds_HYP.acquisition_time.values.min()) + dt_hyp_max = convert_datetime(ds_HYP.acquisition_time.values.max()) + if (dt_hyp_min > start_time) and (dt_hyp_max < stop_time): + print( + ds_HYP.attrs["sequence_id"], + os.path.basename(file), + dt_hyp_min, + dt_hyp_max, + ds_HYP.attrs["site_latitude"], + ds_HYP.attrs["site_longitude"], + period["comments"], + ) + f.write( + "%s,%s,%s,%s,%s,%s,%s\n" + % ( + ds_HYP.attrs["sequence_id"], + os.path.basename(file), + dt_hyp_min.isoformat(), + dt_hyp_max.isoformat(), + ds_HYP.attrs["site_latitude"], + ds_HYP.attrs["site_longitude"], + period["comments"], + ) + ) diff --git a/hypernets_processor/rhymer/rhymer/ancillary/rhymer_ancillary.py b/hypernets_processor/rhymer/rhymer/ancillary/rhymer_ancillary.py index 57f168f8..c4b51890 100755 --- a/hypernets_processor/rhymer/rhymer/ancillary/rhymer_ancillary.py +++ b/hypernets_processor/rhymer/rhymer/ancillary/rhymer_ancillary.py @@ -309,6 +309,13 @@ def gdas_extract(self, isodate, lon, lat, process="GDASFNL", method="linear"): ## hours until archived nowcasts are available time_archive = 30 + ## Bug fix for West longitudes 2024-06-08 + ## change from hypstar lon formaat (-180° to +180°) to GDAS (0° to 360°) + if lon > 0: + lon360 = lon + else: + lon360 = lon + 360 + ## date time now = datetime.datetime.utcnow() if type(isodate) is str: @@ -391,7 +398,7 @@ def gdas_extract(self, isodate, lon, lat, process="GDASFNL", method="linear"): yy = np.arange(len(lats)) ## interpolate indices - loni = np.interp(lon, lons, xx) + loni = np.interp(lon360, lons, xx) if lats[0] > lats[-1]: lati = np.interp(lat * -1, lats * -1, yy) ## invert lats else: diff --git a/hypernets_processor/rhymer/rhymer/hypstar/rhymer_hypstar.py b/hypernets_processor/rhymer/rhymer/hypstar/rhymer_hypstar.py index 5cc384a3..46247e29 100755 --- a/hypernets_processor/rhymer/rhymer/hypstar/rhymer_hypstar.py +++ b/hypernets_processor/rhymer/rhymer/hypstar/rhymer_hypstar.py @@ -301,9 +301,9 @@ def get_wind(self, l1b): if anc_wind is not None: wind.append(anc_wind["w"]) now = datetime.now() - l1b.attrs[ - "rhof_wind_source" - ] = "NCEP/GDAS FNL 0.25 ds083.3 | DOI: 10.5065/D65Q4T4Z" + l1b.attrs["rhof_wind_source"] = ( + "NCEP/GDAS FNL 0.25 ds083.3 | DOI: 10.5065/D65Q4T4Z" + ) l1b["rhof_wind"].values = wind return l1b @@ -327,7 +327,6 @@ def get_rhof(self, l1b): rhof_raa[i] = ((ra - 180) % 360) - 180 - ## get air_water_int reflectance if self.context.get_config_value("rhof_option") == "Mobley1999": if ( diff --git a/hypernets_processor/rhymer/rhymer/processing/rhymer_processing.py b/hypernets_processor/rhymer/rhymer/processing/rhymer_processing.py index 7df5156a..a7ae826a 100755 --- a/hypernets_processor/rhymer/rhymer/processing/rhymer_processing.py +++ b/hypernets_processor/rhymer/rhymer/processing/rhymer_processing.py @@ -3,6 +3,7 @@ import numpy as np import os + ## interpolate Mobley sky reflectance LUT ## ths is sun zenith angle ## thv is viewing angle diff --git a/hypernets_processor/sequence_processor.py b/hypernets_processor/sequence_processor.py index a9340e4c..2725d9e7 100755 --- a/hypernets_processor/sequence_processor.py +++ b/hypernets_processor/sequence_processor.py @@ -2,6 +2,9 @@ Contains main class for processing sequence data """ +import glob +import xarray as xr + from hypernets_processor.version import __version__ from hypernets_processor.calibration.calibrate import Calibrate from hypernets_processor.surface_reflectance.surface_reflectance import ( @@ -17,10 +20,13 @@ from hypernets_processor.calibration.calibration_converter import CalibrationConverter from obsarray.templater.dataset_util import DatasetUtil as du from hypernets_processor.data_utils.quality_checks import QualityChecks +from hypernets_processor.data_utils.site_specific_quality_checks import ( + SiteSpecificQualityChecks, +) import warnings import os -from datetime import datetime +from datetime import datetime, timezone """___Authorship___""" __author__ = "Pieter De Vis" @@ -61,65 +67,107 @@ def process_sequence(self, sequence_path): calcon = CalibrationConverter(self.context) cal = Calibrate(self.context) surf = SurfaceReflectance(self.context) - qc = QualityChecks(self.context) - avg = Average( - self.context, - ) + ssqc = SiteSpecificQualityChecks(self.context) rhymer = RhymerHypstar(self.context) writer = HypernetsWriter(self.context) + # initialise all data products to None + L0a_irr = None + L0a_rad = None + L0a_bla = None + L0a_swir_irr = None + L0a_swir_rad = None + L0a_swir_bla = None + + L1a_rad, L0a_rad_masked, L0a_rad_bla_masked = None, None, None + L1a_irr, L0a_irr_masked, L0a_irr_bla_masked = None, None, None + + L1a_swir_rad, L0a_swir_rad_masked, L0a_swir_rad_bla_masked = ( + None, + None, + None, + ) + + L1a_swir_irr, L0a_swir_irr_masked, L0a_swir_irr_bla_masked = ( + None, + None, + None, + ) + + L1b_rad = None + L1b_irr = None + + L1c = None + L2a = None + + # when reprocessing from specific level, read in and set that data + if self.context.get_config_value("reprocess_from"): + directory = writer.return_directory() + if self.context.get_config_value("reprocess_from").upper() == "L2A": + L1b_rad = self.find_preexisting_file(directory, "L1B_RAD") + L1b_irr = self.find_preexisting_file(directory, "L1B_IRR") + L2a = self.find_preexisting_file(directory, "L2A") + elif self.context.get_config_value("reprocess_from").upper() == "L1B": + L1b_rad = self.find_preexisting_file(directory, "L1B_RAD") + L1b_irr = self.find_preexisting_file(directory, "L1B_IRR") + else: + raise ValueError( + "It is only possible to reprocess from L2A or L1B files. Please change the `reprocess_from' config value" + ) + with warnings.catch_warnings(): if not self.context.get_config_value("verbose"): warnings.simplefilter("ignore") - tstart = datetime.now() + tstart = datetime.now(timezone.utc) self.context.set_config_value("start_time_processing_sequence", tstart) if self.context.get_config_value("network") == "w": calibration_data_rad, calibration_data_irr = calcon.read_calib_files( sequence_path ) # Read L0 - self.context.logger.info("Reading raw data...") - l0a_irr, l0a_rad, l0a_bla = reader.read_sequence( - sequence_path, calibration_data_rad, calibration_data_irr - ) - self.context.logger.info("Done") + if not self.context.get_config_value("reprocess_from"): + self.context.logger.info("Reading raw data...") + L0a_irr, L0a_rad, L0a_bla = reader.read_sequence( + sequence_path, calibration_data_rad, calibration_data_irr + ) + self.context.logger.info("Done") # Calibrate to L1a - if self.context.get_config_value("max_level") in [ + if self.context.get_config_value("max_level").upper() in [ "L1A", "L1B", "L1C", "L2A", + "L2B", ]: - self.context.logger.info("Processing to L1a...") - if l0a_rad: - L1a_rad, l0a_rad_masked, l0a_rad_bla_masked = cal.calibrate_l1a( - "radiance", l0a_rad, l0a_bla, calibration_data_rad + if L0a_rad: + self.context.logger.info("Processing to L1a radiance...") + L1a_rad, L0a_rad_masked, L0a_rad_bla_masked = cal.calibrate_l1a( + "radiance", L0a_rad, L0a_bla, calibration_data_rad ) - else: - L1a_rad, l0a_rad_masked, l0a_rad_bla_masked = None, None, None + self.context.logger.info("Done") - if l0a_irr: - L1a_irr, l0a_irr_masked, l0a_irr_bla_masked = cal.calibrate_l1a( - "irradiance", l0a_irr, l0a_bla, calibration_data_irr + if L0a_irr: + self.context.logger.info("Processing to L1a irradiance...") + L1a_irr, L0a_irr_masked, L0a_irr_bla_masked = cal.calibrate_l1a( + "irradiance", L0a_irr, L0a_bla, calibration_data_irr ) - else: - L1a_irr, l0a_irr_masked, l0a_irr_bla_masked = None, None, None - self.context.logger.info("Done") + self.context.logger.info("Done") - if l0a_rad and l0a_irr: - if self.context.get_config_value("max_level") in [ + if L0a_rad_masked and L0a_irr_masked: + if self.context.get_config_value("max_level").upper() in [ "L1B", "L1C", "L2A", + "L2B", ]: self.context.logger.info("Processing to L1b radiance...") L1b_rad = cal.calibrate_l1b( "radiance", - l0a_rad_masked, - l0a_rad_bla_masked, + L0a_rad_masked, + L0a_rad_bla_masked, calibration_data_rad, ) # print(L1b_rad) @@ -129,20 +177,21 @@ def process_sequence(self, sequence_path): self.context.logger.info("Processing to L1b irradiance...") L1b_irr = cal.calibrate_l1b( "irradiance", - l0a_irr_masked, - l0a_irr_bla_masked, + L0a_irr_masked, + L0a_irr_bla_masked, calibration_data_irr, ) - else: - L1b_rad = None - L1b_irr = None - if L1b_rad and L1b_irr: - if self.context.get_config_value("max_level") in ["L1C", "L2A"]: + azis = rhymer.checkazimuths(L1a_rad) + + if L1b_rad and L1b_irr and len(azis) > 0: + if self.context.get_config_value("max_level").upper() in [ + "L1C", + "L2A", + "L2B", + ]: self.context.logger.info("Processing to L1c...") # check if different azimuth angles within single sequence - azis = rhymer.checkazimuths(L1a_rad) - for a in azis: print("Processing for azimuth:{}".format(a)) rad_, irr_, ra = rhymer.selectazimuths(L1a_rad, L1a_irr, a) @@ -162,7 +211,10 @@ def process_sequence(self, sequence_path): L1c = surf.reflectance_w(L1c_int, L1b_irr, razangle=ra) self.context.logger.info("Done") - if self.context.get_config_value("max_level") == "L2A": + if self.context.get_config_value("max_level").upper() in [ + "L2A", + "L2B", + ]: self.context.logger.info("Processing to L2a...") # add relative azimuth angle for the filename L2a = surf.process_l2(L1c, razangle=ra) @@ -173,147 +225,159 @@ def process_sequence(self, sequence_path): else: self.context.logger.info("Not a standard sequence") - self.context.anomaly_handler.add_anomaly("ms") + self.context.anomaly_handler.add_anomaly("in") elif self.context.get_config_value("network") == "l": comb = CombineSWIR(self.context) intp = Interpolate(self.context) # Read L0 - self.context.logger.info("Reading raw data...") - ( - calibration_data_rad, - calibration_data_irr, - calibration_data_swir_rad, - calibration_data_swir_irr, - ) = calcon.read_calib_files(sequence_path) - - ( - l0a_irr, - l0a_rad, - l0a_bla, - l0a_swir_irr, - l0a_swir_rad, - l0a_swir_bla, - ) = reader.read_sequence( - sequence_path, - calibration_data_rad, - calibration_data_irr, - calibration_data_swir_rad, - calibration_data_swir_irr, - ) - self.context.logger.info("Done") + if not self.context.get_config_value("reprocess_from"): + self.context.logger.info("Reading raw data...") + ( + calibration_data_rad, + calibration_data_irr, + calibration_data_swir_rad, + calibration_data_swir_irr, + ) = calcon.read_calib_files(sequence_path) + + ( + L0a_irr, + L0a_rad, + L0a_bla, + L0a_swir_irr, + L0a_swir_rad, + L0a_swir_bla, + ) = reader.read_sequence( + sequence_path, + calibration_data_rad, + calibration_data_irr, + calibration_data_swir_rad, + calibration_data_swir_irr, + ) + self.context.logger.info("Done") - if self.context.get_config_value("max_level") in [ + if self.context.get_config_value("max_level").upper() in [ "L1A", "L1B", "L1C", "L2A", + "L2B", ]: - self.context.logger.info("Processing to L1a...") - if l0a_rad and l0a_bla: - L1a_rad, l0a_rad_masked, l0a_rad_bla_masked = cal.calibrate_l1a( - "radiance", l0a_rad, l0a_bla, calibration_data_rad + if L0a_rad and L0a_bla: + self.context.logger.info("Processing to L1a radiance...") + L1a_rad, L0a_rad_masked, L0a_rad_bla_masked = cal.calibrate_l1a( + "radiance", L0a_rad, L0a_bla, calibration_data_rad ) - else: - L1a_rad, l0a_rad_masked, l0a_rad_bla_masked = None, None, None + self.context.logger.info("Done") - if l0a_irr and l0a_bla: - L1a_irr, l0a_irr_masked, l0a_irr_bla_masked = cal.calibrate_l1a( - "irradiance", l0a_irr, l0a_bla, calibration_data_irr + if L0a_irr and L0a_bla: + self.context.logger.info("Processing to L1a irradiance...") + L1a_irr, L0a_irr_masked, L0a_irr_bla_masked = cal.calibrate_l1a( + "irradiance", L0a_irr, L0a_bla, calibration_data_irr ) - else: - L1a_irr, l0a_irr_masked, l0a_irr_bla_masked = None, None, None + self.context.logger.info("Done") - if l0a_swir_rad and l0a_swir_bla: + if L0a_swir_rad and L0a_swir_bla: + self.context.logger.info("Processing to L1a SWIR radiance...") ( L1a_swir_rad, - l0a_swir_rad_masked, - l0a_swir_rad_bla_masked, + L0a_swir_rad_masked, + L0a_swir_rad_bla_masked, ) = cal.calibrate_l1a( "radiance", - l0a_swir_rad, - l0a_swir_bla, + L0a_swir_rad, + L0a_swir_bla, calibration_data_swir_rad, swir=True, ) - else: - L1a_swir_rad, l0a_swir_rad_masked, l0a_swir_rad_bla_masked = ( - None, - None, - None, - ) + self.context.logger.info("Done") - if l0a_swir_irr and l0a_swir_bla: + if L0a_swir_irr and L0a_swir_bla: + self.context.logger.info("Processing to L1a SWIR irradiance...") ( L1a_swir_irr, - l0a_swir_irr_masked, - l0a_swir_irr_bla_masked, + L0a_swir_irr_masked, + L0a_swir_irr_bla_masked, ) = cal.calibrate_l1a( "irradiance", - l0a_swir_irr, - l0a_swir_bla, + L0a_swir_irr, + L0a_swir_bla, calibration_data_swir_irr, swir=True, ) - else: - L1a_swir_irr, l0a_swir_irr_masked, l0a_swir_irr_bla_masked = ( - None, - None, - None, - ) - self.context.logger.info("Done") + self.context.logger.info("Done") - if self.context.get_config_value("max_level") in ["L1B", "L1C", "L2A"]: - if l0a_rad_masked and l0a_swir_rad_masked: + if self.context.get_config_value("max_level").upper() in [ + "L1B", + "L1C", + "L2A", + "L2B", + ]: + if L0a_rad_masked and L0a_swir_rad_masked: self.context.logger.info("Processing to L1b radiance...") L1b_rad = comb.combine( "radiance", - l0a_rad_masked, - l0a_rad_bla_masked, - l0a_swir_rad_masked, - l0a_swir_rad_bla_masked, + L0a_rad_masked, + L0a_rad_bla_masked, + L0a_swir_rad_masked, + L0a_swir_rad_bla_masked, calibration_data_rad, calibration_data_swir_rad, ) self.context.logger.info("Done") - else: - L1b_rad = None - if l0a_irr_masked and l0a_swir_irr_masked: + if L0a_irr_masked and L0a_swir_irr_masked: self.context.logger.info("Processing to L1b irradiance...") L1b_irr = comb.combine( "irradiance", - l0a_irr_masked, - l0a_irr_bla_masked, - l0a_swir_irr_masked, - l0a_swir_irr_bla_masked, + L0a_irr_masked, + L0a_irr_bla_masked, + L0a_swir_irr_masked, + L0a_swir_irr_bla_masked, calibration_data_irr, calibration_data_swir_irr, ) self.context.logger.info("Done") - else: - L1b_irr = None - if L1b_rad and L1b_irr: - if self.context.get_config_value("max_level") in ["L1C", "L2A"]: + if ( + L1b_rad + and L1b_irr + and ( + not self.context.get_config_value("reprocess_from") + or not self.context.get_config_value("reprocess_from").upper() + == "L2A" + ) + ): + if self.context.get_config_value("max_level") in [ + "L1C", + "L2A", + "L2B", + ]: self.context.logger.info("Processing to L1c...") L1c = intp.interpolate_l1c(L1b_rad, L1b_irr) self.context.logger.info("Done") - if self.context.get_config_value("max_level") == "L2A": + if self.context.get_config_value("max_level") in ["L2A", "L2B"]: self.context.logger.info("Processing to L2a...") L2a = surf.process_l2(L1c) self.context.logger.info("Done") - else: + elif not self.context.get_config_value("reprocess_from"): self.context.logger.info("Not a standard sequence") - self.context.anomaly_handler.add_anomaly("ms") + self.context.anomaly_handler.add_anomaly("in") + if L2a: + if self.context.get_config_value("max_level").upper() in [ + "L2B", + ]: + self.context.logger.info("Processing to L2B...") + L2b = ssqc.apply_site_specific_QC(L2a, L1b_rad, L1b_irr) + self.context.logger.info("Done") else: raise NameError( "Invalid network: " + self.context.get_config_value("network") ) - tend = datetime.now() + tend = datetime.now(timezone.utc) print( "time for computation of one seq (min, sec):{}".format( divmod((tend - tstart).total_seconds(), 60) @@ -321,6 +385,30 @@ def process_sequence(self, sequence_path): ) return None + def find_preexisting_file(self, directory, level): + """ + function to find preexisting product from previous run + + :param directory: product directory + :param level: level of product (including both level and type e.g. L1B_IRR) + :return: product xarray object + """ + + files = glob.glob(os.path.join(directory, "*%s*.nc" % level)) + if len(files) > 1: + self.context.logger.info( + "multiple %s files found, using the most recent one" % level + ) + + elif len(files) == 0: + self.context.logger.info("no %s file found for this sequence in folder %s" % (level, directory)) + self.context.anomaly_handler.add_anomaly("npr") + return None + + file = files[-1] + self.context.logger.info("starting from file: %s" % (file)) + return xr.open_dataset(file) + if __name__ == "__main__": pass diff --git a/hypernets_processor/surface_reflectance/measurement_functions/water_network_protocol.py b/hypernets_processor/surface_reflectance/measurement_functions/water_network_protocol.py index 6a191efe..3b8009b0 100755 --- a/hypernets_processor/surface_reflectance/measurement_functions/water_network_protocol.py +++ b/hypernets_processor/surface_reflectance/measurement_functions/water_network_protocol.py @@ -26,8 +26,12 @@ def meas_function(self, water_leaving_radiance, irradiance, wavelength): # retrieve variables for NIR SIMIL correction w1 = self.context.get_config_value("similarity_w1") w2 = self.context.get_config_value("similarity_w2") - alpha = 1/self.context.get_config_value("similarity_alpha") # take ^-1 as in Ruddick et al., 2005 - print("NSIMIL COR applied with lambda {}, {} and alpha {}".format(w1, w2, alpha)) + alpha = 1 / self.context.get_config_value( + "similarity_alpha" + ) # take ^-1 as in Ruddick et al., 2005 + print( + "NSIMIL COR applied with lambda {}, {} and alpha {}".format(w1, w2, alpha) + ) iref1, wref1 = self.rhs.closest_idx(wavelength, w1) iref2, wref2 = self.rhs.closest_idx(wavelength, w2) diff --git a/hypernets_processor/surface_reflectance/surface_reflectance.py b/hypernets_processor/surface_reflectance/surface_reflectance.py index f4cc45df..ed945b38 100755 --- a/hypernets_processor/surface_reflectance/surface_reflectance.py +++ b/hypernets_processor/surface_reflectance/surface_reflectance.py @@ -81,18 +81,9 @@ def reflectance_w(self, dataset, l1birr, razangle): store_unc_percent=True, ) else: - measurandstring="water_leaving_radiance" - measurand = water_protocol_function.run(L1c) + measurandstring = "water_leaving_radiance" + measurand = water_protocol_function.run_meas_function(L1c) L1c[measurandstring].values = measurand - L1c = L1c.drop( - [ - "u_rel_random_" + measurandstring, - "u_rel_systematic_indep_" + measurandstring, - "u_rel_systematic_corr_rad_irr_" + measurandstring, - "err_corr_systematic_indep_" + measurandstring, - "err_corr_systematic_corr_rad_irr_" + measurandstring, - ] - ) measurement_function_protocol = self.context.get_config_value( "measurement_function_surface_reflectance" @@ -119,20 +110,12 @@ def reflectance_w(self, dataset, l1birr, razangle): ) else: measurandstring = ["reflectance_nosc", "reflectance", "epsilon"] - reflectance_nosc, reflectance, epsilon = water_protocol_function.run(L1c) + reflectance_nosc, reflectance, epsilon = ( + water_protocol_function.run_meas_function(L1c) + ) L1c[measurandstring[0]].values = reflectance_nosc - L1c[measurandstring[1]].values =reflectance - L1c[measurandstring[2]].values =epsilon - # for varsurf in measurandstring: - # L1c = L1c.drop( - # [ - # "u_rel_random_" + varsurf, - # "u_rel_systematic_indep_" + varsurf, - # "u_rel_systematic_corr_rad_irr_" + varsurf, - # "err_corr_systematic_indep_" + varsurf, - # "err_corr_systematic_corr_rad_irr_" + varsurf, - # ] - # ) + L1c[measurandstring[1]].values = reflectance + L1c[measurandstring[2]].values = epsilon failSimil = self.qual.qc_similarity(L1c) L1c["quality_flag"][np.where(failSimil == 1)] = DatasetUtil.set_flag( @@ -268,6 +251,8 @@ def process_l2(self, dataset, razangle=None): self.plot.plot_polar_reflectance( dataset_l2a, self.context.get_config_value("plot_polar_wav") ) + if self.context.get_config_value("plot_polar_ndvi") is not None: + self.plot.plot_polar_reflectance(dataset_l2a, "ndvi") if self.context.get_config_value("plot_uncertainty"): self.plot.plot_relative_uncertainty( "reflectance", dataset_l2a, refl=True diff --git a/hypernets_processor/test/debug_sequence_processor.py b/hypernets_processor/test/debug_sequence_processor.py index 7ff0110f..7130b740 100755 --- a/hypernets_processor/test/debug_sequence_processor.py +++ b/hypernets_processor/test/debug_sequence_processor.py @@ -30,7 +30,7 @@ def process_sequence(): Command line interface to sequence_processor_main for ad-hoc job processing """ - #network = "water" + # network = "water" network = "land" job_config_path = JOB_CONFIG_TEMPLATE_PATH @@ -48,16 +48,20 @@ def process_sequence(): # Read config file job_config = read_config_file([job_config_path, processor_defaults]) - #job_config["Input"]["raw_data_directory"] = r"C:\Users\pdv\data\insitu\hypernets\raw_data\water\SEQ20230707T120104" - job_config["Input"]["raw_data_directory"] = r"C:\Users\pdv\data\insitu\hypernets\raw_data\GHNA\DATA\SEQ20231102T090040" + # job_config["Input"]["raw_data_directory"] = r"C:\Users\pdv\data\insitu\hypernets\raw_data\water\SEQ20231011T144127" + # job_config["Input"][ + # "raw_data_directory" + # ] = r"C:\Users\pdv\data\insitu\hypernets\raw_data\JAES\SEQ20250227T090042" + job_config["Input"]["raw_data_directory"] = r"C:\Users\pdv\data\insitu\hypernets\raw_data\GHNA\DATA\SEQ20240228T093039" + #job_config["Input"]["raw_data_directory"] = r"C:\Users\pdv\data\insitu\hypernets\raw_data\WWUK\DATA\SEQ20230707T090040" # job_config["Input"][ # "raw_data_directory" # ] = r"C:\Users\pdv\data\insitu\hypernets\raw_data\ATGE\SEQ20220823T153206" - job_config["Output"]["archive_directory"] = r"C:\Users\pdv\data\insitu\hypernets\archive_test" - - job_config["Processor"]["max_level"] = "L2A" + job_config["Output"][ + "archive_directory" + ] = r"C:\Users\pdv\data\insitu\hypernets\archive_test" no_unc = False diff --git a/hypernets_processor/test/test.log b/hypernets_processor/test/test.log index fdb3b81b..e69de29b 100755 --- a/hypernets_processor/test/test.log +++ b/hypernets_processor/test/test.log @@ -1,4075 +0,0 @@ -2023-05-08 10:55:48,997 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220909T133739 -2023-05-08 10:55:49,004 : INFO : Processing Anomalies: ['x'] -2023-05-08 10:55:49,004 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220909T133739Failed: OSError('/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist') -2023-05-08 10:55:49,006 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 86, in process_sequence - calibration_data_rad,calibration_data_irr = calcon.read_calib_files(sequence_path) - File "/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_converter.py", line 82, in read_calib_files - raise IOError( -OSError: /home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist - -2023-05-08 10:55:49,006 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220920T155313 -2023-05-08 10:55:49,008 : INFO : Processing Anomalies: ['x', 'x'] -2023-05-08 10:55:49,008 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220920T155313Failed: KeyError('Metadata') -2023-05-08 10:55:49,010 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 86, in process_sequence - calibration_data_rad,calibration_data_irr = calcon.read_calib_files(sequence_path) - File "/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_converter.py", line 57, in read_calib_files - globalattr = dict(metadata["Metadata"]) - File "/usr/lib/python3.8/configparser.py", line 960, in __getitem__ - raise KeyError(key) -KeyError: 'Metadata' - -2023-05-08 10:55:49,010 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220920T160035 -2023-05-08 10:55:49,014 : INFO : Processing Anomalies: ['x', 'x', 'x'] -2023-05-08 10:55:49,014 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220920T160035Failed: OSError('/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist') -2023-05-08 10:55:49,014 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 86, in process_sequence - calibration_data_rad,calibration_data_irr = calcon.read_calib_files(sequence_path) - File "/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_converter.py", line 82, in read_calib_files - raise IOError( -OSError: /home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist - -2023-05-08 10:55:49,014 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220920T160544 -2023-05-08 10:55:49,021 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x'] -2023-05-08 10:55:49,021 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220920T160544Failed: OSError('/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist') -2023-05-08 10:55:49,022 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 86, in process_sequence - calibration_data_rad,calibration_data_irr = calcon.read_calib_files(sequence_path) - File "/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_converter.py", line 82, in read_calib_files - raise IOError( -OSError: /home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist - -2023-05-08 10:55:49,022 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220920T163034 -2023-05-08 10:55:49,028 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:49,028 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220920T163034Failed: OSError('/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist') -2023-05-08 10:55:49,028 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 86, in process_sequence - calibration_data_rad,calibration_data_irr = calcon.read_calib_files(sequence_path) - File "/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_converter.py", line 82, in read_calib_files - raise IOError( -OSError: /home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist - -2023-05-08 10:55:49,029 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220920T170027 -2023-05-08 10:55:49,033 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:49,033 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220920T170027Failed: OSError('/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist') -2023-05-08 10:55:49,033 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 86, in process_sequence - calibration_data_rad,calibration_data_irr = calcon.read_calib_files(sequence_path) - File "/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_converter.py", line 82, in read_calib_files - raise IOError( -OSError: /home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist - -2023-05-08 10:55:49,033 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220920T173027 -2023-05-08 10:55:49,037 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:49,038 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220920T173027Failed: OSError('/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist') -2023-05-08 10:55:49,038 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 86, in process_sequence - calibration_data_rad,calibration_data_irr = calcon.read_calib_files(sequence_path) - File "/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_converter.py", line 82, in read_calib_files - raise IOError( -OSError: /home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist - -2023-05-08 10:55:49,038 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220920T180035 -2023-05-08 10:55:49,042 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:49,043 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220920T180035Failed: OSError('/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist') -2023-05-08 10:55:49,043 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 86, in process_sequence - calibration_data_rad,calibration_data_irr = calcon.read_calib_files(sequence_path) - File "/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_converter.py", line 82, in read_calib_files - raise IOError( -OSError: /home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist - -2023-05-08 10:55:49,043 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220920T183028 -2023-05-08 10:55:49,048 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:49,048 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220920T183028Failed: OSError('/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist') -2023-05-08 10:55:49,048 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 86, in process_sequence - calibration_data_rad,calibration_data_irr = calcon.read_calib_files(sequence_path) - File "/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_converter.py", line 82, in read_calib_files - raise IOError( -OSError: /home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist - -2023-05-08 10:55:49,048 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220920T190028 -2023-05-08 10:55:49,053 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:49,053 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220920T190028Failed: OSError('/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist') -2023-05-08 10:55:49,054 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 86, in process_sequence - calibration_data_rad,calibration_data_irr = calcon.read_calib_files(sequence_path) - File "/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_converter.py", line 82, in read_calib_files - raise IOError( -OSError: /home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist - -2023-05-08 10:55:49,054 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220920T193027 -2023-05-08 10:55:49,058 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:49,058 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220920T193027Failed: OSError('/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist') -2023-05-08 10:55:49,058 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 86, in process_sequence - calibration_data_rad,calibration_data_irr = calcon.read_calib_files(sequence_path) - File "/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_converter.py", line 82, in read_calib_files - raise IOError( -OSError: /home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist - -2023-05-08 10:55:49,059 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220920T200035 -2023-05-08 10:55:49,064 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:49,064 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220920T200035Failed: OSError('/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist') -2023-05-08 10:55:49,064 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 86, in process_sequence - calibration_data_rad,calibration_data_irr = calcon.read_calib_files(sequence_path) - File "/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_converter.py", line 82, in read_calib_files - raise IOError( -OSError: /home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist - -2023-05-08 10:55:49,065 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220920T203028 -2023-05-08 10:55:49,070 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:49,070 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220920T203028Failed: OSError('/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist') -2023-05-08 10:55:49,071 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 86, in process_sequence - calibration_data_rad,calibration_data_irr = calcon.read_calib_files(sequence_path) - File "/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_converter.py", line 82, in read_calib_files - raise IOError( -OSError: /home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist - -2023-05-08 10:55:49,071 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220920T210035 -2023-05-08 10:55:49,075 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:49,075 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220920T210035Failed: OSError('/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist') -2023-05-08 10:55:49,076 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 86, in process_sequence - calibration_data_rad,calibration_data_irr = calcon.read_calib_files(sequence_path) - File "/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_converter.py", line 82, in read_calib_files - raise IOError( -OSError: /home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist - -2023-05-08 10:55:49,076 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220920T213029 -2023-05-08 10:55:49,082 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:49,083 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220920T213029Failed: OSError('/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist') -2023-05-08 10:55:49,084 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 86, in process_sequence - calibration_data_rad,calibration_data_irr = calcon.read_calib_files(sequence_path) - File "/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_converter.py", line 82, in read_calib_files - raise IOError( -OSError: /home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist - -2023-05-08 10:55:49,084 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220920T220028 -2023-05-08 10:55:49,093 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:49,093 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220920T220028Failed: OSError('/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist') -2023-05-08 10:55:49,094 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 86, in process_sequence - calibration_data_rad,calibration_data_irr = calcon.read_calib_files(sequence_path) - File "/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_converter.py", line 82, in read_calib_files - raise IOError( -OSError: /home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist - -2023-05-08 10:55:49,094 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220920T223029 -2023-05-08 10:55:49,099 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:49,100 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220920T223029Failed: OSError('/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist') -2023-05-08 10:55:49,100 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 86, in process_sequence - calibration_data_rad,calibration_data_irr = calcon.read_calib_files(sequence_path) - File "/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_converter.py", line 82, in read_calib_files - raise IOError( -OSError: /home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist - -2023-05-08 10:55:49,100 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220920T230111 -2023-05-08 10:55:49,107 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:49,107 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220920T230111Failed: OSError('/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist') -2023-05-08 10:55:49,108 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 86, in process_sequence - calibration_data_rad,calibration_data_irr = calcon.read_calib_files(sequence_path) - File "/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_converter.py", line 82, in read_calib_files - raise IOError( -OSError: /home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist - -2023-05-08 10:55:49,108 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220920T233029 -2023-05-08 10:55:49,113 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:49,113 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220920T233029Failed: OSError('/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist') -2023-05-08 10:55:49,113 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 86, in process_sequence - calibration_data_rad,calibration_data_irr = calcon.read_calib_files(sequence_path) - File "/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_converter.py", line 82, in read_calib_files - raise IOError( -OSError: /home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist - -2023-05-08 10:55:49,114 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220921T000029 -2023-05-08 10:55:49,119 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:49,119 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220921T000029Failed: OSError('/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist') -2023-05-08 10:55:49,120 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 86, in process_sequence - calibration_data_rad,calibration_data_irr = calcon.read_calib_files(sequence_path) - File "/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_converter.py", line 82, in read_calib_files - raise IOError( -OSError: /home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist - -2023-05-08 10:55:49,120 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220921T003029 -2023-05-08 10:55:49,127 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:49,127 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220921T003029Failed: OSError('/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist') -2023-05-08 10:55:49,128 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 86, in process_sequence - calibration_data_rad,calibration_data_irr = calcon.read_calib_files(sequence_path) - File "/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_converter.py", line 82, in read_calib_files - raise IOError( -OSError: /home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist - -2023-05-08 10:55:49,128 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220921T010029 -2023-05-08 10:55:49,134 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:49,134 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220921T010029Failed: OSError('/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist') -2023-05-08 10:55:49,135 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 86, in process_sequence - calibration_data_rad,calibration_data_irr = calcon.read_calib_files(sequence_path) - File "/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_converter.py", line 82, in read_calib_files - raise IOError( -OSError: /home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist - -2023-05-08 10:55:49,135 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220921T013028 -2023-05-08 10:55:49,141 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:49,142 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220921T013028Failed: OSError('/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist') -2023-05-08 10:55:49,143 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 86, in process_sequence - calibration_data_rad,calibration_data_irr = calcon.read_calib_files(sequence_path) - File "/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_converter.py", line 82, in read_calib_files - raise IOError( -OSError: /home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist - -2023-05-08 10:55:49,143 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220921T020036 -2023-05-08 10:55:49,150 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:49,150 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220921T020036Failed: OSError('/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist') -2023-05-08 10:55:49,150 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 86, in process_sequence - calibration_data_rad,calibration_data_irr = calcon.read_calib_files(sequence_path) - File "/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_converter.py", line 82, in read_calib_files - raise IOError( -OSError: /home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist - -2023-05-08 10:55:49,151 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220921T023029 -2023-05-08 10:55:49,157 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:49,157 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220921T023029Failed: OSError('/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist') -2023-05-08 10:55:49,158 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 86, in process_sequence - calibration_data_rad,calibration_data_irr = calcon.read_calib_files(sequence_path) - File "/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_converter.py", line 82, in read_calib_files - raise IOError( -OSError: /home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist - -2023-05-08 10:55:49,158 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220921T030029 -2023-05-08 10:55:49,163 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:49,163 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220921T030029Failed: OSError('/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist') -2023-05-08 10:55:49,163 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 86, in process_sequence - calibration_data_rad,calibration_data_irr = calcon.read_calib_files(sequence_path) - File "/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_converter.py", line 82, in read_calib_files - raise IOError( -OSError: /home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist - -2023-05-08 10:55:49,164 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220921T033036 -2023-05-08 10:55:49,168 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:49,169 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220921T033036Failed: OSError('/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist') -2023-05-08 10:55:49,169 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 86, in process_sequence - calibration_data_rad,calibration_data_irr = calcon.read_calib_files(sequence_path) - File "/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_converter.py", line 82, in read_calib_files - raise IOError( -OSError: /home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist - -2023-05-08 10:55:49,169 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220921T040035 -2023-05-08 10:55:49,174 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:49,174 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220921T040035Failed: OSError('/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist') -2023-05-08 10:55:49,175 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 86, in process_sequence - calibration_data_rad,calibration_data_irr = calcon.read_calib_files(sequence_path) - File "/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_converter.py", line 82, in read_calib_files - raise IOError( -OSError: /home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist - -2023-05-08 10:55:49,175 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220921T043028 -2023-05-08 10:55:49,180 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:49,180 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220921T043028Failed: OSError('/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist') -2023-05-08 10:55:49,181 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 86, in process_sequence - calibration_data_rad,calibration_data_irr = calcon.read_calib_files(sequence_path) - File "/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_converter.py", line 82, in read_calib_files - raise IOError( -OSError: /home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist - -2023-05-08 10:55:49,181 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220921T050028 -2023-05-08 10:55:49,185 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:49,185 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220921T050028Failed: OSError('/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist') -2023-05-08 10:55:49,186 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 86, in process_sequence - calibration_data_rad,calibration_data_irr = calcon.read_calib_files(sequence_path) - File "/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_converter.py", line 82, in read_calib_files - raise IOError( -OSError: /home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist - -2023-05-08 10:55:49,186 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220921T053036 -2023-05-08 10:55:49,191 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:49,191 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220921T053036Failed: OSError('/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist') -2023-05-08 10:55:49,192 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 86, in process_sequence - calibration_data_rad,calibration_data_irr = calcon.read_calib_files(sequence_path) - File "/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_converter.py", line 82, in read_calib_files - raise IOError( -OSError: /home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist - -2023-05-08 10:55:49,192 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220921T060026 -2023-05-08 10:55:49,197 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:49,197 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220921T060026Failed: OSError('/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist') -2023-05-08 10:55:49,198 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 86, in process_sequence - calibration_data_rad,calibration_data_irr = calcon.read_calib_files(sequence_path) - File "/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_converter.py", line 82, in read_calib_files - raise IOError( -OSError: /home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist - -2023-05-08 10:55:49,198 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220921T063032 -2023-05-08 10:55:49,204 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:49,204 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220921T063032Failed: OSError('/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist') -2023-05-08 10:55:49,204 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 86, in process_sequence - calibration_data_rad,calibration_data_irr = calcon.read_calib_files(sequence_path) - File "/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_converter.py", line 82, in read_calib_files - raise IOError( -OSError: /home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist - -2023-05-08 10:55:49,204 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220921T070025 -2023-05-08 10:55:49,210 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:49,210 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220921T070025Failed: OSError('/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist') -2023-05-08 10:55:49,211 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 86, in process_sequence - calibration_data_rad,calibration_data_irr = calcon.read_calib_files(sequence_path) - File "/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_converter.py", line 82, in read_calib_files - raise IOError( -OSError: /home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist - -2023-05-08 10:55:49,211 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220921T073033 -2023-05-08 10:55:49,215 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:49,216 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220921T073033Failed: OSError('/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist') -2023-05-08 10:55:49,216 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 86, in process_sequence - calibration_data_rad,calibration_data_irr = calcon.read_calib_files(sequence_path) - File "/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_converter.py", line 82, in read_calib_files - raise IOError( -OSError: /home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist - -2023-05-08 10:55:49,217 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220921T080026 -2023-05-08 10:55:49,221 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:49,222 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220921T080026Failed: OSError('/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist') -2023-05-08 10:55:49,222 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 86, in process_sequence - calibration_data_rad,calibration_data_irr = calcon.read_calib_files(sequence_path) - File "/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_converter.py", line 82, in read_calib_files - raise IOError( -OSError: /home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist - -2023-05-08 10:55:49,222 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220921T083026 -2023-05-08 10:55:49,227 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:49,227 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220921T083026Failed: OSError('/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist') -2023-05-08 10:55:49,228 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 86, in process_sequence - calibration_data_rad,calibration_data_irr = calcon.read_calib_files(sequence_path) - File "/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_converter.py", line 82, in read_calib_files - raise IOError( -OSError: /home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist - -2023-05-08 10:55:49,228 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220921T090032 -2023-05-08 10:55:49,233 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:49,233 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220921T090032Failed: OSError('/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist') -2023-05-08 10:55:49,233 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 86, in process_sequence - calibration_data_rad,calibration_data_irr = calcon.read_calib_files(sequence_path) - File "/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_converter.py", line 82, in read_calib_files - raise IOError( -OSError: /home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist - -2023-05-08 10:55:49,233 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220921T093033 -2023-05-08 10:55:49,238 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:49,238 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220921T093033Failed: OSError('/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist') -2023-05-08 10:55:49,239 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 86, in process_sequence - calibration_data_rad,calibration_data_irr = calcon.read_calib_files(sequence_path) - File "/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_converter.py", line 82, in read_calib_files - raise IOError( -OSError: /home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist - -2023-05-08 10:55:49,239 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220921T100033 -2023-05-08 10:55:49,244 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:49,244 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220921T100033Failed: OSError('/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist') -2023-05-08 10:55:49,245 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 86, in process_sequence - calibration_data_rad,calibration_data_irr = calcon.read_calib_files(sequence_path) - File "/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_converter.py", line 82, in read_calib_files - raise IOError( -OSError: /home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist - -2023-05-08 10:55:49,245 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220921T103033 -2023-05-08 10:55:49,251 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:49,251 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220921T103033Failed: OSError('/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist') -2023-05-08 10:55:49,251 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 86, in process_sequence - calibration_data_rad,calibration_data_irr = calcon.read_calib_files(sequence_path) - File "/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_converter.py", line 82, in read_calib_files - raise IOError( -OSError: /home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist - -2023-05-08 10:55:49,252 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220921T110033 -2023-05-08 10:55:49,257 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:49,257 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220921T110033Failed: OSError('/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist') -2023-05-08 10:55:49,257 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 86, in process_sequence - calibration_data_rad,calibration_data_irr = calcon.read_calib_files(sequence_path) - File "/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_converter.py", line 82, in read_calib_files - raise IOError( -OSError: /home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist - -2023-05-08 10:55:49,257 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220921T113033 -2023-05-08 10:55:49,263 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:49,263 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220921T113033Failed: OSError('/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist') -2023-05-08 10:55:49,263 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 86, in process_sequence - calibration_data_rad,calibration_data_irr = calcon.read_calib_files(sequence_path) - File "/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_converter.py", line 82, in read_calib_files - raise IOError( -OSError: /home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist - -2023-05-08 10:55:49,263 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220921T120033 -2023-05-08 10:55:49,268 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:49,268 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220921T120033Failed: OSError('/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist') -2023-05-08 10:55:49,269 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 86, in process_sequence - calibration_data_rad,calibration_data_irr = calcon.read_calib_files(sequence_path) - File "/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_converter.py", line 82, in read_calib_files - raise IOError( -OSError: /home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist - -2023-05-08 10:55:49,269 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220921T123034 -2023-05-08 10:55:49,274 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:49,274 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220921T123034Failed: OSError('/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist') -2023-05-08 10:55:49,275 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 86, in process_sequence - calibration_data_rad,calibration_data_irr = calcon.read_calib_files(sequence_path) - File "/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_converter.py", line 82, in read_calib_files - raise IOError( -OSError: /home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist - -2023-05-08 10:55:49,275 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220921T130033 -2023-05-08 10:55:49,280 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:49,280 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220921T130033Failed: OSError('/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist') -2023-05-08 10:55:49,281 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 86, in process_sequence - calibration_data_rad,calibration_data_irr = calcon.read_calib_files(sequence_path) - File "/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_converter.py", line 82, in read_calib_files - raise IOError( -OSError: /home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist - -2023-05-08 10:55:49,281 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220921T133025 -2023-05-08 10:55:49,286 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:49,286 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220921T133025Failed: OSError('/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist') -2023-05-08 10:55:49,286 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 86, in process_sequence - calibration_data_rad,calibration_data_irr = calcon.read_calib_files(sequence_path) - File "/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_converter.py", line 82, in read_calib_files - raise IOError( -OSError: /home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist - -2023-05-08 10:55:49,286 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220921T140032 -2023-05-08 10:55:49,291 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:49,292 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220921T140032Failed: OSError('/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist') -2023-05-08 10:55:49,292 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 86, in process_sequence - calibration_data_rad,calibration_data_irr = calcon.read_calib_files(sequence_path) - File "/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_converter.py", line 82, in read_calib_files - raise IOError( -OSError: /home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist - -2023-05-08 10:55:49,292 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220921T143027 -2023-05-08 10:55:49,301 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:49,301 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220921T143027Failed: OSError('/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist') -2023-05-08 10:55:49,302 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 86, in process_sequence - calibration_data_rad,calibration_data_irr = calcon.read_calib_files(sequence_path) - File "/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_converter.py", line 82, in read_calib_files - raise IOError( -OSError: /home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist - -2023-05-08 10:55:49,302 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220921T150033 -2023-05-08 10:55:49,311 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:49,311 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220921T150033Failed: OSError('/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist') -2023-05-08 10:55:49,312 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 86, in process_sequence - calibration_data_rad,calibration_data_irr = calcon.read_calib_files(sequence_path) - File "/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_converter.py", line 82, in read_calib_files - raise IOError( -OSError: /home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist - -2023-05-08 10:55:49,312 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220921T153026 -2023-05-08 10:55:49,320 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:49,321 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220921T153026Failed: OSError('/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist') -2023-05-08 10:55:49,321 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 86, in process_sequence - calibration_data_rad,calibration_data_irr = calcon.read_calib_files(sequence_path) - File "/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_converter.py", line 82, in read_calib_files - raise IOError( -OSError: /home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist - -2023-05-08 10:55:49,322 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220921T160027 -2023-05-08 10:55:49,332 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:49,333 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220921T160027Failed: OSError('/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist') -2023-05-08 10:55:49,333 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 86, in process_sequence - calibration_data_rad,calibration_data_irr = calcon.read_calib_files(sequence_path) - File "/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_converter.py", line 82, in read_calib_files - raise IOError( -OSError: /home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist - -2023-05-08 10:55:49,334 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220921T163033 -2023-05-08 10:55:49,344 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:49,344 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220921T163033Failed: OSError('/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist') -2023-05-08 10:55:49,345 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 86, in process_sequence - calibration_data_rad,calibration_data_irr = calcon.read_calib_files(sequence_path) - File "/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_converter.py", line 82, in read_calib_files - raise IOError( -OSError: /home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist - -2023-05-08 10:55:49,345 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220921T170027 -2023-05-08 10:55:49,352 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:49,352 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220921T170027Failed: OSError('/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist') -2023-05-08 10:55:49,353 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 86, in process_sequence - calibration_data_rad,calibration_data_irr = calcon.read_calib_files(sequence_path) - File "/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_converter.py", line 82, in read_calib_files - raise IOError( -OSError: /home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist - -2023-05-08 10:55:49,353 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220921T173026 -2023-05-08 10:55:49,363 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:49,363 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220921T173026Failed: OSError('/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist') -2023-05-08 10:55:49,364 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 86, in process_sequence - calibration_data_rad,calibration_data_irr = calcon.read_calib_files(sequence_path) - File "/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_converter.py", line 82, in read_calib_files - raise IOError( -OSError: /home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist - -2023-05-08 10:55:49,365 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220921T180026 -2023-05-08 10:55:49,373 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:49,373 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220921T180026Failed: OSError('/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist') -2023-05-08 10:55:49,373 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 86, in process_sequence - calibration_data_rad,calibration_data_irr = calcon.read_calib_files(sequence_path) - File "/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_converter.py", line 82, in read_calib_files - raise IOError( -OSError: /home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist - -2023-05-08 10:55:49,374 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220921T183027 -2023-05-08 10:55:49,382 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:49,382 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220921T183027Failed: OSError('/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist') -2023-05-08 10:55:49,383 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 86, in process_sequence - calibration_data_rad,calibration_data_irr = calcon.read_calib_files(sequence_path) - File "/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_converter.py", line 82, in read_calib_files - raise IOError( -OSError: /home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist - -2023-05-08 10:55:49,383 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220921T190034 -2023-05-08 10:55:49,393 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:49,394 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220921T190034Failed: OSError('/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist') -2023-05-08 10:55:49,394 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 86, in process_sequence - calibration_data_rad,calibration_data_irr = calcon.read_calib_files(sequence_path) - File "/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_converter.py", line 82, in read_calib_files - raise IOError( -OSError: /home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist - -2023-05-08 10:55:49,394 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220921T193033 -2023-05-08 10:55:49,404 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:49,404 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220921T193033Failed: OSError('/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist') -2023-05-08 10:55:49,405 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 86, in process_sequence - calibration_data_rad,calibration_data_irr = calcon.read_calib_files(sequence_path) - File "/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_converter.py", line 82, in read_calib_files - raise IOError( -OSError: /home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist - -2023-05-08 10:55:49,405 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220921T200033 -2023-05-08 10:55:49,415 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:49,416 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220921T200033Failed: OSError('/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist') -2023-05-08 10:55:49,416 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 86, in process_sequence - calibration_data_rad,calibration_data_irr = calcon.read_calib_files(sequence_path) - File "/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_converter.py", line 82, in read_calib_files - raise IOError( -OSError: /home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist - -2023-05-08 10:55:49,416 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220921T203027 -2023-05-08 10:55:49,426 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:49,427 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220921T203027Failed: OSError('/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist') -2023-05-08 10:55:49,427 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 86, in process_sequence - calibration_data_rad,calibration_data_irr = calcon.read_calib_files(sequence_path) - File "/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_converter.py", line 82, in read_calib_files - raise IOError( -OSError: /home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist - -2023-05-08 10:55:49,428 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220921T210027 -2023-05-08 10:55:49,437 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:49,437 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220921T210027Failed: OSError('/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist') -2023-05-08 10:55:49,438 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 86, in process_sequence - calibration_data_rad,calibration_data_irr = calcon.read_calib_files(sequence_path) - File "/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_converter.py", line 82, in read_calib_files - raise IOError( -OSError: /home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist - -2023-05-08 10:55:49,438 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220921T213028 -2023-05-08 10:55:49,448 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:49,448 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220921T213028Failed: OSError('/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist') -2023-05-08 10:55:49,449 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 86, in process_sequence - calibration_data_rad,calibration_data_irr = calcon.read_calib_files(sequence_path) - File "/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_converter.py", line 82, in read_calib_files - raise IOError( -OSError: /home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist - -2023-05-08 10:55:49,449 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220921T220034 -2023-05-08 10:55:49,455 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:49,456 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220921T220034Failed: OSError('/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist') -2023-05-08 10:55:49,456 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 86, in process_sequence - calibration_data_rad,calibration_data_irr = calcon.read_calib_files(sequence_path) - File "/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_converter.py", line 82, in read_calib_files - raise IOError( -OSError: /home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist - -2023-05-08 10:55:49,457 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220921T223034 -2023-05-08 10:55:49,464 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:49,464 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220921T223034Failed: OSError('/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist') -2023-05-08 10:55:49,465 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 86, in process_sequence - calibration_data_rad,calibration_data_irr = calcon.read_calib_files(sequence_path) - File "/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_converter.py", line 82, in read_calib_files - raise IOError( -OSError: /home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist - -2023-05-08 10:55:49,465 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220921T230034 -2023-05-08 10:55:49,472 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:49,472 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220921T230034Failed: OSError('/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist') -2023-05-08 10:55:49,472 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 86, in process_sequence - calibration_data_rad,calibration_data_irr = calcon.read_calib_files(sequence_path) - File "/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_converter.py", line 82, in read_calib_files - raise IOError( -OSError: /home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist - -2023-05-08 10:55:49,472 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220921T233027 -2023-05-08 10:55:49,478 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:49,478 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220921T233027Failed: OSError('/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist') -2023-05-08 10:55:49,478 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 86, in process_sequence - calibration_data_rad,calibration_data_irr = calcon.read_calib_files(sequence_path) - File "/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_converter.py", line 82, in read_calib_files - raise IOError( -OSError: /home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist - -2023-05-08 10:55:49,478 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220922T000028 -2023-05-08 10:55:49,485 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:49,486 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220922T000028Failed: OSError('/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist') -2023-05-08 10:55:49,486 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 86, in process_sequence - calibration_data_rad,calibration_data_irr = calcon.read_calib_files(sequence_path) - File "/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_converter.py", line 82, in read_calib_files - raise IOError( -OSError: /home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist - -2023-05-08 10:55:49,486 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220922T003035 -2023-05-08 10:55:49,492 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:49,492 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220922T003035Failed: OSError('/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist') -2023-05-08 10:55:49,493 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 86, in process_sequence - calibration_data_rad,calibration_data_irr = calcon.read_calib_files(sequence_path) - File "/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_converter.py", line 82, in read_calib_files - raise IOError( -OSError: /home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist - -2023-05-08 10:55:49,493 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220922T010035 -2023-05-08 10:55:49,499 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:49,499 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220922T010035Failed: OSError('/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist') -2023-05-08 10:55:49,500 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 86, in process_sequence - calibration_data_rad,calibration_data_irr = calcon.read_calib_files(sequence_path) - File "/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_converter.py", line 82, in read_calib_files - raise IOError( -OSError: /home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist - -2023-05-08 10:55:49,500 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220922T013028 -2023-05-08 10:55:49,509 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:49,509 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220922T013028Failed: OSError('/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist') -2023-05-08 10:55:49,510 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 86, in process_sequence - calibration_data_rad,calibration_data_irr = calcon.read_calib_files(sequence_path) - File "/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_converter.py", line 82, in read_calib_files - raise IOError( -OSError: /home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist - -2023-05-08 10:55:49,510 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220922T020028 -2023-05-08 10:55:49,517 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:49,517 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220922T020028Failed: OSError('/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist') -2023-05-08 10:55:49,518 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 86, in process_sequence - calibration_data_rad,calibration_data_irr = calcon.read_calib_files(sequence_path) - File "/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_converter.py", line 82, in read_calib_files - raise IOError( -OSError: /home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist - -2023-05-08 10:55:49,518 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220922T023035 -2023-05-08 10:55:49,526 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:49,527 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220922T023035Failed: OSError('/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist') -2023-05-08 10:55:49,527 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 86, in process_sequence - calibration_data_rad,calibration_data_irr = calcon.read_calib_files(sequence_path) - File "/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_converter.py", line 82, in read_calib_files - raise IOError( -OSError: /home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist - -2023-05-08 10:55:49,528 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220922T030034 -2023-05-08 10:55:49,535 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:49,536 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220922T030034Failed: OSError('/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist') -2023-05-08 10:55:49,536 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 86, in process_sequence - calibration_data_rad,calibration_data_irr = calcon.read_calib_files(sequence_path) - File "/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_converter.py", line 82, in read_calib_files - raise IOError( -OSError: /home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist - -2023-05-08 10:55:49,536 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220922T033035 -2023-05-08 10:55:49,543 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:49,543 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220922T033035Failed: OSError('/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist') -2023-05-08 10:55:49,544 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 86, in process_sequence - calibration_data_rad,calibration_data_irr = calcon.read_calib_files(sequence_path) - File "/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_converter.py", line 82, in read_calib_files - raise IOError( -OSError: /home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist - -2023-05-08 10:55:49,544 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220922T040028 -2023-05-08 10:55:49,551 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:49,551 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220922T040028Failed: OSError('/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist') -2023-05-08 10:55:49,552 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 86, in process_sequence - calibration_data_rad,calibration_data_irr = calcon.read_calib_files(sequence_path) - File "/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_converter.py", line 82, in read_calib_files - raise IOError( -OSError: /home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist - -2023-05-08 10:55:49,552 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220922T043034 -2023-05-08 10:55:49,560 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:49,560 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220922T043034Failed: OSError('/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist') -2023-05-08 10:55:49,561 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 86, in process_sequence - calibration_data_rad,calibration_data_irr = calcon.read_calib_files(sequence_path) - File "/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_converter.py", line 82, in read_calib_files - raise IOError( -OSError: /home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist - -2023-05-08 10:55:49,561 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220922T050140 -2023-05-08 10:55:49,568 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:49,569 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220922T050140Failed: OSError('/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist') -2023-05-08 10:55:49,569 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 86, in process_sequence - calibration_data_rad,calibration_data_irr = calcon.read_calib_files(sequence_path) - File "/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_converter.py", line 82, in read_calib_files - raise IOError( -OSError: /home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist - -2023-05-08 10:55:49,570 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220922T053111 -2023-05-08 10:55:49,578 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:49,578 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220922T053111Failed: OSError('/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist') -2023-05-08 10:55:49,579 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 86, in process_sequence - calibration_data_rad,calibration_data_irr = calcon.read_calib_files(sequence_path) - File "/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_converter.py", line 82, in read_calib_files - raise IOError( -OSError: /home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist - -2023-05-08 10:55:49,579 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220922T060050 -2023-05-08 10:55:49,594 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:49,595 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220922T060050Failed: OSError('/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist') -2023-05-08 10:55:49,595 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 86, in process_sequence - calibration_data_rad,calibration_data_irr = calcon.read_calib_files(sequence_path) - File "/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_converter.py", line 82, in read_calib_files - raise IOError( -OSError: /home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist - -2023-05-08 10:55:49,596 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220922T063026 -2023-05-08 10:55:49,606 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:49,606 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220922T063026Failed: OSError('/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist') -2023-05-08 10:55:49,607 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 86, in process_sequence - calibration_data_rad,calibration_data_irr = calcon.read_calib_files(sequence_path) - File "/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_converter.py", line 82, in read_calib_files - raise IOError( -OSError: /home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist - -2023-05-08 10:55:49,607 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220922T070142 -2023-05-08 10:55:49,614 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:49,615 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220922T070142Failed: OSError('/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist') -2023-05-08 10:55:49,615 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 86, in process_sequence - calibration_data_rad,calibration_data_irr = calcon.read_calib_files(sequence_path) - File "/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_converter.py", line 82, in read_calib_files - raise IOError( -OSError: /home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist - -2023-05-08 10:55:49,615 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220922T073027 -2023-05-08 10:55:49,623 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:49,623 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220922T073027Failed: OSError('/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist') -2023-05-08 10:55:49,623 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 86, in process_sequence - calibration_data_rad,calibration_data_irr = calcon.read_calib_files(sequence_path) - File "/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_converter.py", line 82, in read_calib_files - raise IOError( -OSError: /home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist - -2023-05-08 10:55:49,624 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220922T080034 -2023-05-08 10:55:49,629 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:49,630 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220922T080034Failed: OSError('/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist') -2023-05-08 10:55:49,630 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 86, in process_sequence - calibration_data_rad,calibration_data_irr = calcon.read_calib_files(sequence_path) - File "/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_converter.py", line 82, in read_calib_files - raise IOError( -OSError: /home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist - -2023-05-08 10:55:49,630 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220922T083529 -2023-05-08 10:55:49,636 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:49,637 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220922T083529Failed: OSError('/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist') -2023-05-08 10:55:49,637 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 86, in process_sequence - calibration_data_rad,calibration_data_irr = calcon.read_calib_files(sequence_path) - File "/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_converter.py", line 82, in read_calib_files - raise IOError( -OSError: /home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist - -2023-05-08 10:55:49,637 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220922T100412 -2023-05-08 10:55:49,643 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:49,643 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220922T100412Failed: OSError('/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist') -2023-05-08 10:55:49,643 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 86, in process_sequence - calibration_data_rad,calibration_data_irr = calcon.read_calib_files(sequence_path) - File "/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_converter.py", line 82, in read_calib_files - raise IOError( -OSError: /home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist - -2023-05-08 10:55:49,644 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220922T103034 -2023-05-08 10:55:49,649 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:49,650 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220922T103034Failed: OSError('/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist') -2023-05-08 10:55:49,650 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 86, in process_sequence - calibration_data_rad,calibration_data_irr = calcon.read_calib_files(sequence_path) - File "/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_converter.py", line 82, in read_calib_files - raise IOError( -OSError: /home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist - -2023-05-08 10:55:49,650 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220922T110025 -2023-05-08 10:55:49,657 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:49,657 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220922T110025Failed: OSError('/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist') -2023-05-08 10:55:49,657 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 86, in process_sequence - calibration_data_rad,calibration_data_irr = calcon.read_calib_files(sequence_path) - File "/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_converter.py", line 82, in read_calib_files - raise IOError( -OSError: /home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist - -2023-05-08 10:55:49,657 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220922T113035 -2023-05-08 10:55:49,665 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:49,665 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220922T113035Failed: OSError('/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist') -2023-05-08 10:55:49,666 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 86, in process_sequence - calibration_data_rad,calibration_data_irr = calcon.read_calib_files(sequence_path) - File "/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_converter.py", line 82, in read_calib_files - raise IOError( -OSError: /home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist - -2023-05-08 10:55:49,666 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220922T114524 -2023-05-08 10:55:49,672 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:49,672 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220922T114524Failed: OSError('/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist') -2023-05-08 10:55:49,673 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 86, in process_sequence - calibration_data_rad,calibration_data_irr = calcon.read_calib_files(sequence_path) - File "/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_converter.py", line 82, in read_calib_files - raise IOError( -OSError: /home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibration_files/HYPSTAR_cal/hypstar_123456/HYPERNETS_CAL_HYPSTAR_123456_RAD_v1.0.nc calibration file does not exist - -2023-05-08 10:55:49,673 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220922T120741 -2023-05-08 10:55:49,987 : INFO : Reading raw data... -2023-05-08 10:55:49,994 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 10:55:49,994 : DEBUG : {'Total Dataset Length': 4131, 'Spectrum Type Information': 136, 'acquisition_time': 1663848724759, 'integration_time': 8192, 'temperature': 31.407642364501953, 'Pixel Count': 2048, 'acceleration_x_mean': -16222, 'acceleration_x_std': 45, 'acceleration_y_mean': -1238, 'acceleration_y_std': 46, 'acceleration_z_mean': 27, 'acceleration_z_std': 127} -2023-05-08 10:55:49,998 : DEBUG : Wavelength range: 163.96437072753906 -1145.4518535429638 -2023-05-08 10:55:50,121 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 10:55:50,121 : DEBUG : Reading Data spectra ... -2023-05-08 10:55:50,122 : DEBUG : Reading CRC32 ... -2023-05-08 10:55:50,124 : ERROR : Lattitude is not found, using default values instead for lat, lon, sza and saa. -2023-05-08 10:55:50,128 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:50,128 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220922T120741Failed: UnboundLocalError("local variable 'vaa' referenced before assignment") -2023-05-08 10:55:50,130 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 89, in process_sequence - l0_irr,l0_rad,l0_bla = reader.read_sequence(sequence_path,calibration_data_rad,calibration_data_irr) - File "/home/cgoyens/hypernets_processor/hypernets_processor/data_io/hypernets_reader.py", line 1041, in read_sequence - l0_irr = self.read_series( - File "/home/cgoyens/hypernets_processor/hypernets_processor/data_io/hypernets_reader.py", line 378, in read_series - ds["viewing_azimuth_angle"][scan_number] = vaa -UnboundLocalError: local variable 'vaa' referenced before assignment - -2023-05-08 10:55:50,131 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220922T123146 -2023-05-08 10:55:50,367 : INFO : Reading raw data... -2023-05-08 10:55:50,374 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 10:55:50,374 : DEBUG : {'Total Dataset Length': 4131, 'Spectrum Type Information': 136, 'acquisition_time': 1663850176446, 'integration_time': 8192, 'temperature': 30.760961532592773, 'Pixel Count': 2048, 'acceleration_x_mean': -16222, 'acceleration_x_std': 39, 'acceleration_y_mean': -1230, 'acceleration_y_std': 47, 'acceleration_z_mean': 20, 'acceleration_z_std': 121} -2023-05-08 10:55:50,378 : DEBUG : Wavelength range: 163.96437072753906 -1145.4518535429638 -2023-05-08 10:55:50,483 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 10:55:50,483 : DEBUG : Reading Data spectra ... -2023-05-08 10:55:50,484 : DEBUG : Reading CRC32 ... -2023-05-08 10:55:50,486 : ERROR : Lattitude is not found, using default values instead for lat, lon, sza and saa. -2023-05-08 10:55:50,489 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:50,489 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220922T123146Failed: UnboundLocalError("local variable 'vaa' referenced before assignment") -2023-05-08 10:55:50,489 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 89, in process_sequence - l0_irr,l0_rad,l0_bla = reader.read_sequence(sequence_path,calibration_data_rad,calibration_data_irr) - File "/home/cgoyens/hypernets_processor/hypernets_processor/data_io/hypernets_reader.py", line 1041, in read_sequence - l0_irr = self.read_series( - File "/home/cgoyens/hypernets_processor/hypernets_processor/data_io/hypernets_reader.py", line 378, in read_series - ds["viewing_azimuth_angle"][scan_number] = vaa -UnboundLocalError: local variable 'vaa' referenced before assignment - -2023-05-08 10:55:50,492 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220922T130025 -2023-05-08 10:55:50,767 : INFO : Reading raw data... -2023-05-08 10:55:50,773 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 10:55:50,773 : DEBUG : {'Total Dataset Length': 4131, 'Spectrum Type Information': 136, 'acquisition_time': 1663851882658, 'integration_time': 4096, 'temperature': 29.233213424682617, 'Pixel Count': 2048, 'acceleration_x_mean': -16223, 'acceleration_x_std': 40, 'acceleration_y_mean': -1239, 'acceleration_y_std': 49, 'acceleration_z_mean': 24, 'acceleration_z_std': 134} -2023-05-08 10:55:50,777 : DEBUG : Wavelength range: 163.96437072753906 -1145.4518535429638 -2023-05-08 10:55:50,888 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 10:55:50,888 : DEBUG : Reading Data spectra ... -2023-05-08 10:55:50,890 : DEBUG : Reading CRC32 ... -2023-05-08 10:55:50,891 : ERROR : Lattitude is not found, using default values instead for lat, lon, sza and saa. -2023-05-08 10:55:50,895 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:50,895 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220922T130025Failed: UnboundLocalError("local variable 'vaa' referenced before assignment") -2023-05-08 10:55:50,895 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 89, in process_sequence - l0_irr,l0_rad,l0_bla = reader.read_sequence(sequence_path,calibration_data_rad,calibration_data_irr) - File "/home/cgoyens/hypernets_processor/hypernets_processor/data_io/hypernets_reader.py", line 1041, in read_sequence - l0_irr = self.read_series( - File "/home/cgoyens/hypernets_processor/hypernets_processor/data_io/hypernets_reader.py", line 378, in read_series - ds["viewing_azimuth_angle"][scan_number] = vaa -UnboundLocalError: local variable 'vaa' referenced before assignment - -2023-05-08 10:55:50,897 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220922T130644 -2023-05-08 10:55:51,147 : INFO : Reading raw data... -2023-05-08 10:55:51,154 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 10:55:51,154 : DEBUG : {'Total Dataset Length': 4131, 'Spectrum Type Information': 136, 'acquisition_time': 1663852234346, 'integration_time': 4096, 'temperature': 30.096433639526367, 'Pixel Count': 2048, 'acceleration_x_mean': -16218, 'acceleration_x_std': 38, 'acceleration_y_mean': -1212, 'acceleration_y_std': 44, 'acceleration_z_mean': 55, 'acceleration_z_std': 110} -2023-05-08 10:55:51,157 : DEBUG : Wavelength range: 163.96437072753906 -1145.4518535429638 -2023-05-08 10:55:51,273 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 10:55:51,273 : DEBUG : Reading Data spectra ... -2023-05-08 10:55:51,274 : DEBUG : Reading CRC32 ... -2023-05-08 10:55:51,276 : ERROR : Lattitude is not found, using default values instead for lat, lon, sza and saa. -2023-05-08 10:55:51,280 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:51,280 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220922T130644Failed: UnboundLocalError("local variable 'vaa' referenced before assignment") -2023-05-08 10:55:51,280 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 89, in process_sequence - l0_irr,l0_rad,l0_bla = reader.read_sequence(sequence_path,calibration_data_rad,calibration_data_irr) - File "/home/cgoyens/hypernets_processor/hypernets_processor/data_io/hypernets_reader.py", line 1041, in read_sequence - l0_irr = self.read_series( - File "/home/cgoyens/hypernets_processor/hypernets_processor/data_io/hypernets_reader.py", line 378, in read_series - ds["viewing_azimuth_angle"][scan_number] = vaa -UnboundLocalError: local variable 'vaa' referenced before assignment - -2023-05-08 10:55:51,282 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220922T131226 -2023-05-08 10:55:51,527 : INFO : Reading raw data... -2023-05-08 10:55:51,534 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 10:55:51,534 : DEBUG : {'Total Dataset Length': 4131, 'Spectrum Type Information': 136, 'acquisition_time': 1663852573706, 'integration_time': 4096, 'temperature': 30.760961532592773, 'Pixel Count': 2048, 'acceleration_x_mean': -16218, 'acceleration_x_std': 40, 'acceleration_y_mean': -1225, 'acceleration_y_std': 47, 'acceleration_z_mean': 34, 'acceleration_z_std': 125} -2023-05-08 10:55:51,540 : DEBUG : Wavelength range: 163.96437072753906 -1145.4518535429638 -2023-05-08 10:55:51,677 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 10:55:51,677 : DEBUG : Reading Data spectra ... -2023-05-08 10:55:51,678 : DEBUG : Reading CRC32 ... -2023-05-08 10:55:51,680 : ERROR : Lattitude is not found, using default values instead for lat, lon, sza and saa. -2023-05-08 10:55:51,683 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:51,684 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220922T131226Failed: UnboundLocalError("local variable 'vaa' referenced before assignment") -2023-05-08 10:55:51,684 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 89, in process_sequence - l0_irr,l0_rad,l0_bla = reader.read_sequence(sequence_path,calibration_data_rad,calibration_data_irr) - File "/home/cgoyens/hypernets_processor/hypernets_processor/data_io/hypernets_reader.py", line 1041, in read_sequence - l0_irr = self.read_series( - File "/home/cgoyens/hypernets_processor/hypernets_processor/data_io/hypernets_reader.py", line 378, in read_series - ds["viewing_azimuth_angle"][scan_number] = vaa -UnboundLocalError: local variable 'vaa' referenced before assignment - -2023-05-08 10:55:51,686 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220922T133141 -2023-05-08 10:55:51,933 : INFO : Reading raw data... -2023-05-08 10:55:51,941 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 10:55:51,941 : DEBUG : {'Total Dataset Length': 4131, 'Spectrum Type Information': 136, 'acquisition_time': 1663853764688, 'integration_time': 8192, 'temperature': 31.966148376464844, 'Pixel Count': 2048, 'acceleration_x_mean': -16192, 'acceleration_x_std': 42, 'acceleration_y_mean': -1205, 'acceleration_y_std': 52, 'acceleration_z_mean': 33, 'acceleration_z_std': 130} -2023-05-08 10:55:51,944 : DEBUG : Wavelength range: 163.96437072753906 -1145.4518535429638 -2023-05-08 10:55:52,060 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 10:55:52,060 : DEBUG : Reading Data spectra ... -2023-05-08 10:55:52,063 : DEBUG : Reading CRC32 ... -2023-05-08 10:55:52,064 : ERROR : Lattitude is not found, using default values instead for lat, lon, sza and saa. -2023-05-08 10:55:52,068 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:52,068 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220922T133141Failed: UnboundLocalError("local variable 'vaa' referenced before assignment") -2023-05-08 10:55:52,068 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 89, in process_sequence - l0_irr,l0_rad,l0_bla = reader.read_sequence(sequence_path,calibration_data_rad,calibration_data_irr) - File "/home/cgoyens/hypernets_processor/hypernets_processor/data_io/hypernets_reader.py", line 1041, in read_sequence - l0_irr = self.read_series( - File "/home/cgoyens/hypernets_processor/hypernets_processor/data_io/hypernets_reader.py", line 378, in read_series - ds["viewing_azimuth_angle"][scan_number] = vaa -UnboundLocalError: local variable 'vaa' referenced before assignment - -2023-05-08 10:55:52,070 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220922T134441 -2023-05-08 10:55:52,368 : INFO : Reading raw data... -2023-05-08 10:55:52,380 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 10:55:52,380 : DEBUG : {'Total Dataset Length': 4131, 'Spectrum Type Information': 136, 'acquisition_time': 1663854516653, 'integration_time': 8192, 'temperature': 30.876066207885742, 'Pixel Count': 2048, 'acceleration_x_mean': -16217, 'acceleration_x_std': 39, 'acceleration_y_mean': -1224, 'acceleration_y_std': 47, 'acceleration_z_mean': 32, 'acceleration_z_std': 114} -2023-05-08 10:55:52,387 : DEBUG : Wavelength range: 163.96437072753906 -1145.4518535429638 -2023-05-08 10:55:52,514 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 10:55:52,514 : DEBUG : Reading Data spectra ... -2023-05-08 10:55:52,515 : DEBUG : Reading CRC32 ... -2023-05-08 10:55:52,517 : ERROR : Lattitude is not found, using default values instead for lat, lon, sza and saa. -2023-05-08 10:55:52,521 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:52,521 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220922T134441Failed: UnboundLocalError("local variable 'vaa' referenced before assignment") -2023-05-08 10:55:52,521 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 89, in process_sequence - l0_irr,l0_rad,l0_bla = reader.read_sequence(sequence_path,calibration_data_rad,calibration_data_irr) - File "/home/cgoyens/hypernets_processor/hypernets_processor/data_io/hypernets_reader.py", line 1041, in read_sequence - l0_irr = self.read_series( - File "/home/cgoyens/hypernets_processor/hypernets_processor/data_io/hypernets_reader.py", line 378, in read_series - ds["viewing_azimuth_angle"][scan_number] = vaa -UnboundLocalError: local variable 'vaa' referenced before assignment - -2023-05-08 10:55:52,523 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220922T142545 -2023-05-08 10:55:52,788 : INFO : Reading raw data... -2023-05-08 10:55:52,795 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 10:55:52,796 : DEBUG : {'Total Dataset Length': 4131, 'Spectrum Type Information': 136, 'acquisition_time': 1663857003568, 'integration_time': 8192, 'temperature': 29.32369613647461, 'Pixel Count': 2048, 'acceleration_x_mean': -16225, 'acceleration_x_std': 40, 'acceleration_y_mean': -1224, 'acceleration_y_std': 47, 'acceleration_z_mean': 28, 'acceleration_z_std': 128} -2023-05-08 10:55:52,799 : DEBUG : Wavelength range: 163.96437072753906 -1145.4518535429638 -2023-05-08 10:55:52,917 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 10:55:52,917 : DEBUG : Reading Data spectra ... -2023-05-08 10:55:52,919 : DEBUG : Reading CRC32 ... -2023-05-08 10:55:52,921 : ERROR : Lattitude is not found, using default values instead for lat, lon, sza and saa. -2023-05-08 10:55:52,924 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:52,924 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220922T142545Failed: UnboundLocalError("local variable 'vaa' referenced before assignment") -2023-05-08 10:55:52,925 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 89, in process_sequence - l0_irr,l0_rad,l0_bla = reader.read_sequence(sequence_path,calibration_data_rad,calibration_data_irr) - File "/home/cgoyens/hypernets_processor/hypernets_processor/data_io/hypernets_reader.py", line 1041, in read_sequence - l0_irr = self.read_series( - File "/home/cgoyens/hypernets_processor/hypernets_processor/data_io/hypernets_reader.py", line 378, in read_series - ds["viewing_azimuth_angle"][scan_number] = vaa -UnboundLocalError: local variable 'vaa' referenced before assignment - -2023-05-08 10:55:52,927 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220922T161554 -2023-05-08 10:55:53,168 : INFO : Reading raw data... -2023-05-08 10:55:53,175 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 10:55:53,175 : DEBUG : {'Total Dataset Length': 4131, 'Spectrum Type Information': 136, 'acquisition_time': 1663863869957, 'integration_time': 16384, 'temperature': 29.504924774169922, 'Pixel Count': 2048, 'acceleration_x_mean': -16177, 'acceleration_x_std': 43, 'acceleration_y_mean': -1213, 'acceleration_y_std': 47, 'acceleration_z_mean': -893, 'acceleration_z_std': 93} -2023-05-08 10:55:53,179 : DEBUG : Wavelength range: 163.96437072753906 -1145.4518535429638 -2023-05-08 10:55:53,281 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 10:55:53,282 : DEBUG : Reading Data spectra ... -2023-05-08 10:55:53,283 : DEBUG : Reading CRC32 ... -2023-05-08 10:55:53,284 : ERROR : Lattitude is not found, using default values instead for lat, lon, sza and saa. -2023-05-08 10:55:53,288 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:53,288 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220922T161554Failed: UnboundLocalError("local variable 'vaa' referenced before assignment") -2023-05-08 10:55:53,289 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 89, in process_sequence - l0_irr,l0_rad,l0_bla = reader.read_sequence(sequence_path,calibration_data_rad,calibration_data_irr) - File "/home/cgoyens/hypernets_processor/hypernets_processor/data_io/hypernets_reader.py", line 1041, in read_sequence - l0_irr = self.read_series( - File "/home/cgoyens/hypernets_processor/hypernets_processor/data_io/hypernets_reader.py", line 378, in read_series - ds["viewing_azimuth_angle"][scan_number] = vaa -UnboundLocalError: local variable 'vaa' referenced before assignment - -2023-05-08 10:55:53,291 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220922T163147 -2023-05-08 10:55:53,532 : INFO : Reading raw data... -2023-05-08 10:55:53,539 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 10:55:53,539 : DEBUG : {'Total Dataset Length': 4131, 'Spectrum Type Information': 136, 'acquisition_time': 1663864871989, 'integration_time': 16384, 'temperature': 30.737958908081055, 'Pixel Count': 2048, 'acceleration_x_mean': -16177, 'acceleration_x_std': 41, 'acceleration_y_mean': -1216, 'acceleration_y_std': 49, 'acceleration_z_mean': -867, 'acceleration_z_std': 97} -2023-05-08 10:55:53,542 : DEBUG : Wavelength range: 163.96437072753906 -1145.4518535429638 -2023-05-08 10:55:53,650 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 10:55:53,650 : DEBUG : Reading Data spectra ... -2023-05-08 10:55:53,652 : DEBUG : Reading CRC32 ... -2023-05-08 10:55:53,653 : ERROR : Lattitude is not found, using default values instead for lat, lon, sza and saa. -2023-05-08 10:55:53,657 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:53,657 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220922T163147Failed: UnboundLocalError("local variable 'vaa' referenced before assignment") -2023-05-08 10:55:53,657 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 89, in process_sequence - l0_irr,l0_rad,l0_bla = reader.read_sequence(sequence_path,calibration_data_rad,calibration_data_irr) - File "/home/cgoyens/hypernets_processor/hypernets_processor/data_io/hypernets_reader.py", line 1041, in read_sequence - l0_irr = self.read_series( - File "/home/cgoyens/hypernets_processor/hypernets_processor/data_io/hypernets_reader.py", line 378, in read_series - ds["viewing_azimuth_angle"][scan_number] = vaa -UnboundLocalError: local variable 'vaa' referenced before assignment - -2023-05-08 10:55:53,660 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220922T170140 -2023-05-08 10:55:53,896 : INFO : Reading raw data... -2023-05-08 10:55:53,903 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 10:55:53,904 : DEBUG : {'Total Dataset Length': 4131, 'Spectrum Type Information': 136, 'acquisition_time': 1663866673622, 'integration_time': 16384, 'temperature': 30.714963912963867, 'Pixel Count': 2048, 'acceleration_x_mean': -16179, 'acceleration_x_std': 42, 'acceleration_y_mean': -1186, 'acceleration_y_std': 47, 'acceleration_z_mean': -850, 'acceleration_z_std': 97} -2023-05-08 10:55:53,913 : DEBUG : Wavelength range: 163.96437072753906 -1145.4518535429638 -2023-05-08 10:55:54,026 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 10:55:54,026 : DEBUG : Reading Data spectra ... -2023-05-08 10:55:54,028 : DEBUG : Reading CRC32 ... -2023-05-08 10:55:54,030 : ERROR : Lattitude is not found, using default values instead for lat, lon, sza and saa. -2023-05-08 10:55:54,033 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:54,034 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220922T170140Failed: UnboundLocalError("local variable 'vaa' referenced before assignment") -2023-05-08 10:55:54,034 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 89, in process_sequence - l0_irr,l0_rad,l0_bla = reader.read_sequence(sequence_path,calibration_data_rad,calibration_data_irr) - File "/home/cgoyens/hypernets_processor/hypernets_processor/data_io/hypernets_reader.py", line 1041, in read_sequence - l0_irr = self.read_series( - File "/home/cgoyens/hypernets_processor/hypernets_processor/data_io/hypernets_reader.py", line 378, in read_series - ds["viewing_azimuth_angle"][scan_number] = vaa -UnboundLocalError: local variable 'vaa' referenced before assignment - -2023-05-08 10:55:54,036 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220922T173139 -2023-05-08 10:55:54,315 : INFO : Reading raw data... -2023-05-08 10:55:54,322 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 10:55:54,322 : DEBUG : {'Total Dataset Length': 4131, 'Spectrum Type Information': 136, 'acquisition_time': 1663868666015, 'integration_time': 16384, 'temperature': 31.083660125732422, 'Pixel Count': 2048, 'acceleration_x_mean': -16185, 'acceleration_x_std': 44, 'acceleration_y_mean': -1165, 'acceleration_y_std': 52, 'acceleration_z_mean': -844, 'acceleration_z_std': 99} -2023-05-08 10:55:54,326 : DEBUG : Wavelength range: 163.96437072753906 -1145.4518535429638 -2023-05-08 10:55:54,441 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 10:55:54,441 : DEBUG : Reading Data spectra ... -2023-05-08 10:55:54,443 : DEBUG : Reading CRC32 ... -2023-05-08 10:55:54,444 : ERROR : Lattitude is not found, using default values instead for lat, lon, sza and saa. -2023-05-08 10:55:54,447 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:54,448 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220922T173139Failed: UnboundLocalError("local variable 'vaa' referenced before assignment") -2023-05-08 10:55:54,448 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 89, in process_sequence - l0_irr,l0_rad,l0_bla = reader.read_sequence(sequence_path,calibration_data_rad,calibration_data_irr) - File "/home/cgoyens/hypernets_processor/hypernets_processor/data_io/hypernets_reader.py", line 1041, in read_sequence - l0_irr = self.read_series( - File "/home/cgoyens/hypernets_processor/hypernets_processor/data_io/hypernets_reader.py", line 378, in read_series - ds["viewing_azimuth_angle"][scan_number] = vaa -UnboundLocalError: local variable 'vaa' referenced before assignment - -2023-05-08 10:55:54,450 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220922T180144 -2023-05-08 10:55:54,694 : INFO : Reading raw data... -2023-05-08 10:55:54,700 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 10:55:54,701 : DEBUG : {'Total Dataset Length': 4131, 'Spectrum Type Information': 136, 'acquisition_time': 1663870470435, 'integration_time': 16384, 'temperature': 31.268632888793945, 'Pixel Count': 2048, 'acceleration_x_mean': -16163, 'acceleration_x_std': 44, 'acceleration_y_mean': -1167, 'acceleration_y_std': 49, 'acceleration_z_mean': -858, 'acceleration_z_std': 100} -2023-05-08 10:55:54,704 : DEBUG : Wavelength range: 163.96437072753906 -1145.4518535429638 -2023-05-08 10:55:54,809 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 10:55:54,810 : DEBUG : Reading Data spectra ... -2023-05-08 10:55:54,811 : DEBUG : Reading CRC32 ... -2023-05-08 10:55:54,812 : ERROR : Lattitude is not found, using default values instead for lat, lon, sza and saa. -2023-05-08 10:55:54,816 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:54,816 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220922T180144Failed: UnboundLocalError("local variable 'vaa' referenced before assignment") -2023-05-08 10:55:54,816 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 89, in process_sequence - l0_irr,l0_rad,l0_bla = reader.read_sequence(sequence_path,calibration_data_rad,calibration_data_irr) - File "/home/cgoyens/hypernets_processor/hypernets_processor/data_io/hypernets_reader.py", line 1041, in read_sequence - l0_irr = self.read_series( - File "/home/cgoyens/hypernets_processor/hypernets_processor/data_io/hypernets_reader.py", line 378, in read_series - ds["viewing_azimuth_angle"][scan_number] = vaa -UnboundLocalError: local variable 'vaa' referenced before assignment - -2023-05-08 10:55:54,819 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220922T183142 -2023-05-08 10:55:55,052 : INFO : Reading raw data... -2023-05-08 10:55:55,059 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 10:55:55,059 : DEBUG : {'Total Dataset Length': 4131, 'Spectrum Type Information': 136, 'acquisition_time': 1663872270849, 'integration_time': 16384, 'temperature': 31.19921875, 'Pixel Count': 2048, 'acceleration_x_mean': -16173, 'acceleration_x_std': 42, 'acceleration_y_mean': -1191, 'acceleration_y_std': 51, 'acceleration_z_mean': -842, 'acceleration_z_std': 97} -2023-05-08 10:55:55,063 : DEBUG : Wavelength range: 163.96437072753906 -1145.4518535429638 -2023-05-08 10:55:55,177 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 10:55:55,178 : DEBUG : Reading Data spectra ... -2023-05-08 10:55:55,179 : DEBUG : Reading CRC32 ... -2023-05-08 10:55:55,180 : ERROR : Lattitude is not found, using default values instead for lat, lon, sza and saa. -2023-05-08 10:55:55,184 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:55,184 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220922T183142Failed: UnboundLocalError("local variable 'vaa' referenced before assignment") -2023-05-08 10:55:55,185 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 89, in process_sequence - l0_irr,l0_rad,l0_bla = reader.read_sequence(sequence_path,calibration_data_rad,calibration_data_irr) - File "/home/cgoyens/hypernets_processor/hypernets_processor/data_io/hypernets_reader.py", line 1041, in read_sequence - l0_irr = self.read_series( - File "/home/cgoyens/hypernets_processor/hypernets_processor/data_io/hypernets_reader.py", line 378, in read_series - ds["viewing_azimuth_angle"][scan_number] = vaa -UnboundLocalError: local variable 'vaa' referenced before assignment - -2023-05-08 10:55:55,187 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220922T190149 -2023-05-08 10:55:55,427 : INFO : Reading raw data... -2023-05-08 10:55:55,434 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 10:55:55,434 : DEBUG : {'Total Dataset Length': 4131, 'Spectrum Type Information': 136, 'acquisition_time': 1663874075556, 'integration_time': 16384, 'temperature': 31.129863739013672, 'Pixel Count': 2048, 'acceleration_x_mean': -16179, 'acceleration_x_std': 43, 'acceleration_y_mean': -1173, 'acceleration_y_std': 50, 'acceleration_z_mean': -846, 'acceleration_z_std': 97} -2023-05-08 10:55:55,443 : DEBUG : Wavelength range: 163.96437072753906 -1145.4518535429638 -2023-05-08 10:55:55,559 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 10:55:55,559 : DEBUG : Reading Data spectra ... -2023-05-08 10:55:55,560 : DEBUG : Reading CRC32 ... -2023-05-08 10:55:55,562 : ERROR : Lattitude is not found, using default values instead for lat, lon, sza and saa. -2023-05-08 10:55:55,566 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:55,566 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220922T190149Failed: UnboundLocalError("local variable 'vaa' referenced before assignment") -2023-05-08 10:55:55,566 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 89, in process_sequence - l0_irr,l0_rad,l0_bla = reader.read_sequence(sequence_path,calibration_data_rad,calibration_data_irr) - File "/home/cgoyens/hypernets_processor/hypernets_processor/data_io/hypernets_reader.py", line 1041, in read_sequence - l0_irr = self.read_series( - File "/home/cgoyens/hypernets_processor/hypernets_processor/data_io/hypernets_reader.py", line 378, in read_series - ds["viewing_azimuth_angle"][scan_number] = vaa -UnboundLocalError: local variable 'vaa' referenced before assignment - -2023-05-08 10:55:55,568 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220922T193141 -2023-05-08 10:55:55,800 : INFO : Reading raw data... -2023-05-08 10:55:55,808 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 10:55:55,808 : DEBUG : {'Total Dataset Length': 4131, 'Spectrum Type Information': 136, 'acquisition_time': 1663875883860, 'integration_time': 16384, 'temperature': 31.083660125732422, 'Pixel Count': 2048, 'acceleration_x_mean': -16188, 'acceleration_x_std': 43, 'acceleration_y_mean': -1149, 'acceleration_y_std': 54, 'acceleration_z_mean': -842, 'acceleration_z_std': 99} -2023-05-08 10:55:55,811 : DEBUG : Wavelength range: 163.96437072753906 -1145.4518535429638 -2023-05-08 10:55:55,913 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 10:55:55,913 : DEBUG : Reading Data spectra ... -2023-05-08 10:55:55,914 : DEBUG : Reading CRC32 ... -2023-05-08 10:55:55,916 : ERROR : Lattitude is not found, using default values instead for lat, lon, sza and saa. -2023-05-08 10:55:55,919 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:55,919 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220922T193141Failed: UnboundLocalError("local variable 'vaa' referenced before assignment") -2023-05-08 10:55:55,920 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 89, in process_sequence - l0_irr,l0_rad,l0_bla = reader.read_sequence(sequence_path,calibration_data_rad,calibration_data_irr) - File "/home/cgoyens/hypernets_processor/hypernets_processor/data_io/hypernets_reader.py", line 1041, in read_sequence - l0_irr = self.read_series( - File "/home/cgoyens/hypernets_processor/hypernets_processor/data_io/hypernets_reader.py", line 378, in read_series - ds["viewing_azimuth_angle"][scan_number] = vaa -UnboundLocalError: local variable 'vaa' referenced before assignment - -2023-05-08 10:55:55,924 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220922T200122 -2023-05-08 10:55:56,158 : INFO : Reading raw data... -2023-05-08 10:55:56,166 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 10:55:56,166 : DEBUG : {'Total Dataset Length': 4131, 'Spectrum Type Information': 136, 'acquisition_time': 1663877655891, 'integration_time': 16384, 'temperature': 31.03748321533203, 'Pixel Count': 2048, 'acceleration_x_mean': -16169, 'acceleration_x_std': 45, 'acceleration_y_mean': -1187, 'acceleration_y_std': 51, 'acceleration_z_mean': -839, 'acceleration_z_std': 98} -2023-05-08 10:55:56,169 : DEBUG : Wavelength range: 163.96437072753906 -1145.4518535429638 -2023-05-08 10:55:56,271 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 10:55:56,271 : DEBUG : Reading Data spectra ... -2023-05-08 10:55:56,272 : DEBUG : Reading CRC32 ... -2023-05-08 10:55:56,273 : ERROR : Lattitude is not found, using default values instead for lat, lon, sza and saa. -2023-05-08 10:55:56,278 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:56,278 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220922T200122Failed: UnboundLocalError("local variable 'vaa' referenced before assignment") -2023-05-08 10:55:56,278 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 89, in process_sequence - l0_irr,l0_rad,l0_bla = reader.read_sequence(sequence_path,calibration_data_rad,calibration_data_irr) - File "/home/cgoyens/hypernets_processor/hypernets_processor/data_io/hypernets_reader.py", line 1041, in read_sequence - l0_irr = self.read_series( - File "/home/cgoyens/hypernets_processor/hypernets_processor/data_io/hypernets_reader.py", line 378, in read_series - ds["viewing_azimuth_angle"][scan_number] = vaa -UnboundLocalError: local variable 'vaa' referenced before assignment - -2023-05-08 10:55:56,280 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220922T203136 -2023-05-08 10:55:56,514 : INFO : Reading raw data... -2023-05-08 10:55:56,522 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 10:55:56,522 : DEBUG : {'Total Dataset Length': 4131, 'Spectrum Type Information': 136, 'acquisition_time': 1663879464431, 'integration_time': 16384, 'temperature': 30.899105072021484, 'Pixel Count': 2048, 'acceleration_x_mean': -16172, 'acceleration_x_std': 43, 'acceleration_y_mean': -1180, 'acceleration_y_std': 50, 'acceleration_z_mean': -860, 'acceleration_z_std': 104} -2023-05-08 10:55:56,525 : DEBUG : Wavelength range: 163.96437072753906 -1145.4518535429638 -2023-05-08 10:55:56,642 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 10:55:56,642 : DEBUG : Reading Data spectra ... -2023-05-08 10:55:56,643 : DEBUG : Reading CRC32 ... -2023-05-08 10:55:56,645 : ERROR : Lattitude is not found, using default values instead for lat, lon, sza and saa. -2023-05-08 10:55:56,649 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:56,649 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220922T203136Failed: UnboundLocalError("local variable 'vaa' referenced before assignment") -2023-05-08 10:55:56,649 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 89, in process_sequence - l0_irr,l0_rad,l0_bla = reader.read_sequence(sequence_path,calibration_data_rad,calibration_data_irr) - File "/home/cgoyens/hypernets_processor/hypernets_processor/data_io/hypernets_reader.py", line 1041, in read_sequence - l0_irr = self.read_series( - File "/home/cgoyens/hypernets_processor/hypernets_processor/data_io/hypernets_reader.py", line 378, in read_series - ds["viewing_azimuth_angle"][scan_number] = vaa -UnboundLocalError: local variable 'vaa' referenced before assignment - -2023-05-08 10:55:56,651 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220922T210149 -2023-05-08 10:55:56,885 : INFO : Reading raw data... -2023-05-08 10:55:56,893 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 10:55:56,893 : DEBUG : {'Total Dataset Length': 4131, 'Spectrum Type Information': 136, 'acquisition_time': 1663881276173, 'integration_time': 16384, 'temperature': 30.83000373840332, 'Pixel Count': 2048, 'acceleration_x_mean': -16174, 'acceleration_x_std': 44, 'acceleration_y_mean': -1166, 'acceleration_y_std': 50, 'acceleration_z_mean': -826, 'acceleration_z_std': 103} -2023-05-08 10:55:56,897 : DEBUG : Wavelength range: 163.96437072753906 -1145.4518535429638 -2023-05-08 10:55:57,013 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 10:55:57,013 : DEBUG : Reading Data spectra ... -2023-05-08 10:55:57,014 : DEBUG : Reading CRC32 ... -2023-05-08 10:55:57,016 : ERROR : Lattitude is not found, using default values instead for lat, lon, sza and saa. -2023-05-08 10:55:57,020 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:57,020 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220922T210149Failed: UnboundLocalError("local variable 'vaa' referenced before assignment") -2023-05-08 10:55:57,021 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 89, in process_sequence - l0_irr,l0_rad,l0_bla = reader.read_sequence(sequence_path,calibration_data_rad,calibration_data_irr) - File "/home/cgoyens/hypernets_processor/hypernets_processor/data_io/hypernets_reader.py", line 1041, in read_sequence - l0_irr = self.read_series( - File "/home/cgoyens/hypernets_processor/hypernets_processor/data_io/hypernets_reader.py", line 378, in read_series - ds["viewing_azimuth_angle"][scan_number] = vaa -UnboundLocalError: local variable 'vaa' referenced before assignment - -2023-05-08 10:55:57,023 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220922T213130 -2023-05-08 10:55:57,264 : INFO : Reading raw data... -2023-05-08 10:55:57,271 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 10:55:57,271 : DEBUG : {'Total Dataset Length': 4131, 'Spectrum Type Information': 136, 'acquisition_time': 1663883058853, 'integration_time': 16384, 'temperature': 30.714963912963867, 'Pixel Count': 2048, 'acceleration_x_mean': -16175, 'acceleration_x_std': 44, 'acceleration_y_mean': -1166, 'acceleration_y_std': 50, 'acceleration_z_mean': -835, 'acceleration_z_std': 94} -2023-05-08 10:55:57,274 : DEBUG : Wavelength range: 163.96437072753906 -1145.4518535429638 -2023-05-08 10:55:57,409 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 10:55:57,409 : DEBUG : Reading Data spectra ... -2023-05-08 10:55:57,411 : DEBUG : Reading CRC32 ... -2023-05-08 10:55:57,412 : ERROR : Lattitude is not found, using default values instead for lat, lon, sza and saa. -2023-05-08 10:55:57,416 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:57,416 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220922T213130Failed: UnboundLocalError("local variable 'vaa' referenced before assignment") -2023-05-08 10:55:57,417 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 89, in process_sequence - l0_irr,l0_rad,l0_bla = reader.read_sequence(sequence_path,calibration_data_rad,calibration_data_irr) - File "/home/cgoyens/hypernets_processor/hypernets_processor/data_io/hypernets_reader.py", line 1041, in read_sequence - l0_irr = self.read_series( - File "/home/cgoyens/hypernets_processor/hypernets_processor/data_io/hypernets_reader.py", line 378, in read_series - ds["viewing_azimuth_angle"][scan_number] = vaa -UnboundLocalError: local variable 'vaa' referenced before assignment - -2023-05-08 10:55:57,419 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220922T220141 -2023-05-08 10:55:57,698 : INFO : Reading raw data... -2023-05-08 10:55:57,704 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 10:55:57,704 : DEBUG : {'Total Dataset Length': 4131, 'Spectrum Type Information': 136, 'acquisition_time': 1663884870728, 'integration_time': 16384, 'temperature': 30.577123641967773, 'Pixel Count': 2048, 'acceleration_x_mean': -16188, 'acceleration_x_std': 41, 'acceleration_y_mean': -1169, 'acceleration_y_std': 47, 'acceleration_z_mean': -836, 'acceleration_z_std': 97} -2023-05-08 10:55:57,708 : DEBUG : Wavelength range: 163.96437072753906 -1145.4518535429638 -2023-05-08 10:55:57,822 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 10:55:57,822 : DEBUG : Reading Data spectra ... -2023-05-08 10:55:57,823 : DEBUG : Reading CRC32 ... -2023-05-08 10:55:57,825 : ERROR : Lattitude is not found, using default values instead for lat, lon, sza and saa. -2023-05-08 10:55:57,829 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:57,829 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220922T220141Failed: UnboundLocalError("local variable 'vaa' referenced before assignment") -2023-05-08 10:55:57,829 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 89, in process_sequence - l0_irr,l0_rad,l0_bla = reader.read_sequence(sequence_path,calibration_data_rad,calibration_data_irr) - File "/home/cgoyens/hypernets_processor/hypernets_processor/data_io/hypernets_reader.py", line 1041, in read_sequence - l0_irr = self.read_series( - File "/home/cgoyens/hypernets_processor/hypernets_processor/data_io/hypernets_reader.py", line 378, in read_series - ds["viewing_azimuth_angle"][scan_number] = vaa -UnboundLocalError: local variable 'vaa' referenced before assignment - -2023-05-08 10:55:57,831 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220922T223149 -2023-05-08 10:55:58,148 : INFO : Reading raw data... -2023-05-08 10:55:58,157 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 10:55:58,158 : DEBUG : {'Total Dataset Length': 4131, 'Spectrum Type Information': 136, 'acquisition_time': 1663886675916, 'integration_time': 16384, 'temperature': 30.531227111816406, 'Pixel Count': 2048, 'acceleration_x_mean': -16205, 'acceleration_x_std': 45, 'acceleration_y_mean': -1184, 'acceleration_y_std': 52, 'acceleration_z_mean': -828, 'acceleration_z_std': 98} -2023-05-08 10:55:58,166 : DEBUG : Wavelength range: 163.96437072753906 -1145.4518535429638 -2023-05-08 10:55:58,273 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 10:55:58,273 : DEBUG : Reading Data spectra ... -2023-05-08 10:55:58,274 : DEBUG : Reading CRC32 ... -2023-05-08 10:55:58,275 : ERROR : Lattitude is not found, using default values instead for lat, lon, sza and saa. -2023-05-08 10:55:58,282 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:58,282 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220922T223149Failed: UnboundLocalError("local variable 'vaa' referenced before assignment") -2023-05-08 10:55:58,283 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 89, in process_sequence - l0_irr,l0_rad,l0_bla = reader.read_sequence(sequence_path,calibration_data_rad,calibration_data_irr) - File "/home/cgoyens/hypernets_processor/hypernets_processor/data_io/hypernets_reader.py", line 1041, in read_sequence - l0_irr = self.read_series( - File "/home/cgoyens/hypernets_processor/hypernets_processor/data_io/hypernets_reader.py", line 378, in read_series - ds["viewing_azimuth_angle"][scan_number] = vaa -UnboundLocalError: local variable 'vaa' referenced before assignment - -2023-05-08 10:55:58,285 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220922T230123 -2023-05-08 10:55:58,527 : INFO : Reading raw data... -2023-05-08 10:55:58,534 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 10:55:58,534 : DEBUG : {'Total Dataset Length': 4131, 'Spectrum Type Information': 136, 'acquisition_time': 1663888565152, 'integration_time': 16384, 'temperature': 30.554170608520508, 'Pixel Count': 2048, 'acceleration_x_mean': -16193, 'acceleration_x_std': 39, 'acceleration_y_mean': -1174, 'acceleration_y_std': 45, 'acceleration_z_mean': -823, 'acceleration_z_std': 94} -2023-05-08 10:55:58,537 : DEBUG : Wavelength range: 163.96437072753906 -1145.4518535429638 -2023-05-08 10:55:58,657 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 10:55:58,657 : DEBUG : Reading Data spectra ... -2023-05-08 10:55:58,659 : DEBUG : Reading CRC32 ... -2023-05-08 10:55:58,660 : ERROR : Lattitude is not found, using default values instead for lat, lon, sza and saa. -2023-05-08 10:55:58,665 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:58,665 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220922T230123Failed: UnboundLocalError("local variable 'vaa' referenced before assignment") -2023-05-08 10:55:58,665 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 89, in process_sequence - l0_irr,l0_rad,l0_bla = reader.read_sequence(sequence_path,calibration_data_rad,calibration_data_irr) - File "/home/cgoyens/hypernets_processor/hypernets_processor/data_io/hypernets_reader.py", line 1041, in read_sequence - l0_irr = self.read_series( - File "/home/cgoyens/hypernets_processor/hypernets_processor/data_io/hypernets_reader.py", line 378, in read_series - ds["viewing_azimuth_angle"][scan_number] = vaa -UnboundLocalError: local variable 'vaa' referenced before assignment - -2023-05-08 10:55:58,667 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220922T233026 -2023-05-08 10:55:58,895 : INFO : Reading raw data... -2023-05-08 10:55:58,903 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 10:55:58,903 : DEBUG : {'Total Dataset Length': 4131, 'Spectrum Type Information': 136, 'acquisition_time': 1663890193197, 'integration_time': 16384, 'temperature': 30.485353469848633, 'Pixel Count': 2048, 'acceleration_x_mean': -16193, 'acceleration_x_std': 44, 'acceleration_y_mean': -1155, 'acceleration_y_std': 54, 'acceleration_z_mean': -818, 'acceleration_z_std': 102} -2023-05-08 10:55:58,908 : DEBUG : Wavelength range: 163.96437072753906 -1145.4518535429638 -2023-05-08 10:55:59,015 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 10:55:59,015 : DEBUG : Reading Data spectra ... -2023-05-08 10:55:59,017 : DEBUG : Reading CRC32 ... -2023-05-08 10:55:59,018 : ERROR : Lattitude is not found, using default values instead for lat, lon, sza and saa. -2023-05-08 10:55:59,022 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:59,022 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220922T233026Failed: UnboundLocalError("local variable 'vaa' referenced before assignment") -2023-05-08 10:55:59,022 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 89, in process_sequence - l0_irr,l0_rad,l0_bla = reader.read_sequence(sequence_path,calibration_data_rad,calibration_data_irr) - File "/home/cgoyens/hypernets_processor/hypernets_processor/data_io/hypernets_reader.py", line 1041, in read_sequence - l0_irr = self.read_series( - File "/home/cgoyens/hypernets_processor/hypernets_processor/data_io/hypernets_reader.py", line 378, in read_series - ds["viewing_azimuth_angle"][scan_number] = vaa -UnboundLocalError: local variable 'vaa' referenced before assignment - -2023-05-08 10:55:59,024 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220923T000105 -2023-05-08 10:55:59,271 : INFO : Reading raw data... -2023-05-08 10:55:59,278 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 10:55:59,278 : DEBUG : {'Total Dataset Length': 4131, 'Spectrum Type Information': 136, 'acquisition_time': 1663892144626, 'integration_time': 16384, 'temperature': 30.347890853881836, 'Pixel Count': 2048, 'acceleration_x_mean': -16181, 'acceleration_x_std': 43, 'acceleration_y_mean': -772, 'acceleration_y_std': 46, 'acceleration_z_mean': 1038, 'acceleration_z_std': 106} -2023-05-08 10:55:59,281 : DEBUG : Wavelength range: 163.96437072753906 -1145.4518535429638 -2023-05-08 10:55:59,392 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 10:55:59,392 : DEBUG : Reading Data spectra ... -2023-05-08 10:55:59,394 : DEBUG : Reading CRC32 ... -2023-05-08 10:55:59,395 : ERROR : Lattitude is not found, using default values instead for lat, lon, sza and saa. -2023-05-08 10:55:59,399 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:59,399 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220923T000105Failed: UnboundLocalError("local variable 'vaa' referenced before assignment") -2023-05-08 10:55:59,400 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 89, in process_sequence - l0_irr,l0_rad,l0_bla = reader.read_sequence(sequence_path,calibration_data_rad,calibration_data_irr) - File "/home/cgoyens/hypernets_processor/hypernets_processor/data_io/hypernets_reader.py", line 1041, in read_sequence - l0_irr = self.read_series( - File "/home/cgoyens/hypernets_processor/hypernets_processor/data_io/hypernets_reader.py", line 378, in read_series - ds["viewing_azimuth_angle"][scan_number] = vaa -UnboundLocalError: local variable 'vaa' referenced before assignment - -2023-05-08 10:55:59,402 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220923T003034 -2023-05-08 10:55:59,632 : INFO : Reading raw data... -2023-05-08 10:55:59,639 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 10:55:59,639 : DEBUG : {'Total Dataset Length': 4131, 'Spectrum Type Information': 136, 'acquisition_time': 1663893913420, 'integration_time': 16384, 'temperature': 30.393686294555664, 'Pixel Count': 2048, 'acceleration_x_mean': -16206, 'acceleration_x_std': 44, 'acceleration_y_mean': -770, 'acceleration_y_std': 48, 'acceleration_z_mean': 1035, 'acceleration_z_std': 99} -2023-05-08 10:55:59,643 : DEBUG : Wavelength range: 163.96437072753906 -1145.4518535429638 -2023-05-08 10:55:59,745 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 10:55:59,745 : DEBUG : Reading Data spectra ... -2023-05-08 10:55:59,746 : DEBUG : Reading CRC32 ... -2023-05-08 10:55:59,747 : ERROR : Lattitude is not found, using default values instead for lat, lon, sza and saa. -2023-05-08 10:55:59,751 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:55:59,751 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220923T003034Failed: UnboundLocalError("local variable 'vaa' referenced before assignment") -2023-05-08 10:55:59,752 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 89, in process_sequence - l0_irr,l0_rad,l0_bla = reader.read_sequence(sequence_path,calibration_data_rad,calibration_data_irr) - File "/home/cgoyens/hypernets_processor/hypernets_processor/data_io/hypernets_reader.py", line 1041, in read_sequence - l0_irr = self.read_series( - File "/home/cgoyens/hypernets_processor/hypernets_processor/data_io/hypernets_reader.py", line 378, in read_series - ds["viewing_azimuth_angle"][scan_number] = vaa -UnboundLocalError: local variable 'vaa' referenced before assignment - -2023-05-08 10:55:59,754 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220923T010149 -2023-05-08 10:55:59,989 : INFO : Reading raw data... -2023-05-08 10:55:59,996 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 10:55:59,996 : DEBUG : {'Total Dataset Length': 4131, 'Spectrum Type Information': 136, 'acquisition_time': 1663895807573, 'integration_time': 16384, 'temperature': 30.279239654541016, 'Pixel Count': 2048, 'acceleration_x_mean': -16183, 'acceleration_x_std': 44, 'acceleration_y_mean': -707, 'acceleration_y_std': 48, 'acceleration_z_mean': 1027, 'acceleration_z_std': 102} -2023-05-08 10:56:00,000 : DEBUG : Wavelength range: 163.96437072753906 -1145.4518535429638 -2023-05-08 10:56:00,127 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 10:56:00,127 : DEBUG : Reading Data spectra ... -2023-05-08 10:56:00,129 : DEBUG : Reading CRC32 ... -2023-05-08 10:56:00,130 : ERROR : Lattitude is not found, using default values instead for lat, lon, sza and saa. -2023-05-08 10:56:00,134 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:56:00,134 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220923T010149Failed: UnboundLocalError("local variable 'vaa' referenced before assignment") -2023-05-08 10:56:00,135 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 89, in process_sequence - l0_irr,l0_rad,l0_bla = reader.read_sequence(sequence_path,calibration_data_rad,calibration_data_irr) - File "/home/cgoyens/hypernets_processor/hypernets_processor/data_io/hypernets_reader.py", line 1041, in read_sequence - l0_irr = self.read_series( - File "/home/cgoyens/hypernets_processor/hypernets_processor/data_io/hypernets_reader.py", line 378, in read_series - ds["viewing_azimuth_angle"][scan_number] = vaa -UnboundLocalError: local variable 'vaa' referenced before assignment - -2023-05-08 10:56:00,137 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220923T013121 -2023-05-08 10:56:00,367 : INFO : Reading raw data... -2023-05-08 10:56:00,374 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 10:56:00,374 : DEBUG : {'Total Dataset Length': 4131, 'Spectrum Type Information': 136, 'acquisition_time': 1663897566810, 'integration_time': 16384, 'temperature': 30.256366729736328, 'Pixel Count': 2048, 'acceleration_x_mean': -16171, 'acceleration_x_std': 46, 'acceleration_y_mean': -722, 'acceleration_y_std': 47, 'acceleration_z_mean': 1049, 'acceleration_z_std': 102} -2023-05-08 10:56:00,377 : DEBUG : Wavelength range: 163.96437072753906 -1145.4518535429638 -2023-05-08 10:56:00,492 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 10:56:00,492 : DEBUG : Reading Data spectra ... -2023-05-08 10:56:00,493 : DEBUG : Reading CRC32 ... -2023-05-08 10:56:00,495 : ERROR : Lattitude is not found, using default values instead for lat, lon, sza and saa. -2023-05-08 10:56:00,499 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:56:00,499 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220923T013121Failed: UnboundLocalError("local variable 'vaa' referenced before assignment") -2023-05-08 10:56:00,500 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 89, in process_sequence - l0_irr,l0_rad,l0_bla = reader.read_sequence(sequence_path,calibration_data_rad,calibration_data_irr) - File "/home/cgoyens/hypernets_processor/hypernets_processor/data_io/hypernets_reader.py", line 1041, in read_sequence - l0_irr = self.read_series( - File "/home/cgoyens/hypernets_processor/hypernets_processor/data_io/hypernets_reader.py", line 378, in read_series - ds["viewing_azimuth_angle"][scan_number] = vaa -UnboundLocalError: local variable 'vaa' referenced before assignment - -2023-05-08 10:56:00,502 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220923T020144 -2023-05-08 10:56:00,738 : INFO : Reading raw data... -2023-05-08 10:56:00,745 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 10:56:00,745 : DEBUG : {'Total Dataset Length': 4131, 'Spectrum Type Information': 136, 'acquisition_time': 1663899392065, 'integration_time': 16384, 'temperature': 30.187789916992188, 'Pixel Count': 2048, 'acceleration_x_mean': -16203, 'acceleration_x_std': 43, 'acceleration_y_mean': -717, 'acceleration_y_std': 46, 'acceleration_z_mean': 1037, 'acceleration_z_std': 96} -2023-05-08 10:56:00,749 : DEBUG : Wavelength range: 163.96437072753906 -1145.4518535429638 -2023-05-08 10:56:00,860 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 10:56:00,860 : DEBUG : Reading Data spectra ... -2023-05-08 10:56:00,861 : DEBUG : Reading CRC32 ... -2023-05-08 10:56:00,862 : ERROR : Lattitude is not found, using default values instead for lat, lon, sza and saa. -2023-05-08 10:56:00,867 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:56:00,867 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220923T020144Failed: UnboundLocalError("local variable 'vaa' referenced before assignment") -2023-05-08 10:56:00,868 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 89, in process_sequence - l0_irr,l0_rad,l0_bla = reader.read_sequence(sequence_path,calibration_data_rad,calibration_data_irr) - File "/home/cgoyens/hypernets_processor/hypernets_processor/data_io/hypernets_reader.py", line 1041, in read_sequence - l0_irr = self.read_series( - File "/home/cgoyens/hypernets_processor/hypernets_processor/data_io/hypernets_reader.py", line 378, in read_series - ds["viewing_azimuth_angle"][scan_number] = vaa -UnboundLocalError: local variable 'vaa' referenced before assignment - -2023-05-08 10:56:00,870 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220923T023142 -2023-05-08 10:56:01,126 : INFO : Reading raw data... -2023-05-08 10:56:01,133 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 10:56:01,133 : DEBUG : {'Total Dataset Length': 4131, 'Spectrum Type Information': 136, 'acquisition_time': 1663901186184, 'integration_time': 16384, 'temperature': 30.142099380493164, 'Pixel Count': 2048, 'acceleration_x_mean': -16203, 'acceleration_x_std': 44, 'acceleration_y_mean': -712, 'acceleration_y_std': 46, 'acceleration_z_mean': 1026, 'acceleration_z_std': 98} -2023-05-08 10:56:01,136 : DEBUG : Wavelength range: 163.96437072753906 -1145.4518535429638 -2023-05-08 10:56:01,262 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 10:56:01,262 : DEBUG : Reading Data spectra ... -2023-05-08 10:56:01,263 : DEBUG : Reading CRC32 ... -2023-05-08 10:56:01,265 : ERROR : Lattitude is not found, using default values instead for lat, lon, sza and saa. -2023-05-08 10:56:01,269 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:56:01,269 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220923T023142Failed: UnboundLocalError("local variable 'vaa' referenced before assignment") -2023-05-08 10:56:01,270 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 89, in process_sequence - l0_irr,l0_rad,l0_bla = reader.read_sequence(sequence_path,calibration_data_rad,calibration_data_irr) - File "/home/cgoyens/hypernets_processor/hypernets_processor/data_io/hypernets_reader.py", line 1041, in read_sequence - l0_irr = self.read_series( - File "/home/cgoyens/hypernets_processor/hypernets_processor/data_io/hypernets_reader.py", line 378, in read_series - ds["viewing_azimuth_angle"][scan_number] = vaa -UnboundLocalError: local variable 'vaa' referenced before assignment - -2023-05-08 10:56:01,272 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220923T030149 -2023-05-08 10:56:01,523 : INFO : Reading raw data... -2023-05-08 10:56:01,531 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 10:56:01,531 : DEBUG : {'Total Dataset Length': 4131, 'Spectrum Type Information': 136, 'acquisition_time': 1663902999005, 'integration_time': 16384, 'temperature': 30.119266510009766, 'Pixel Count': 2048, 'acceleration_x_mean': -16197, 'acceleration_x_std': 41, 'acceleration_y_mean': -715, 'acceleration_y_std': 48, 'acceleration_z_mean': 1056, 'acceleration_z_std': 104} -2023-05-08 10:56:01,535 : DEBUG : Wavelength range: 163.96437072753906 -1145.4518535429638 -2023-05-08 10:56:01,679 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 10:56:01,679 : DEBUG : Reading Data spectra ... -2023-05-08 10:56:01,680 : DEBUG : Reading CRC32 ... -2023-05-08 10:56:01,682 : ERROR : Lattitude is not found, using default values instead for lat, lon, sza and saa. -2023-05-08 10:56:01,686 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:56:01,686 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220923T030149Failed: UnboundLocalError("local variable 'vaa' referenced before assignment") -2023-05-08 10:56:01,687 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 89, in process_sequence - l0_irr,l0_rad,l0_bla = reader.read_sequence(sequence_path,calibration_data_rad,calibration_data_irr) - File "/home/cgoyens/hypernets_processor/hypernets_processor/data_io/hypernets_reader.py", line 1041, in read_sequence - l0_irr = self.read_series( - File "/home/cgoyens/hypernets_processor/hypernets_processor/data_io/hypernets_reader.py", line 378, in read_series - ds["viewing_azimuth_angle"][scan_number] = vaa -UnboundLocalError: local variable 'vaa' referenced before assignment - -2023-05-08 10:56:01,689 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220923T033032 -2023-05-08 10:56:01,955 : INFO : Reading raw data... -2023-05-08 10:56:01,962 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 10:56:01,963 : DEBUG : {'Total Dataset Length': 4131, 'Spectrum Type Information': 136, 'acquisition_time': 1663904698031, 'integration_time': 16384, 'temperature': 30.164941787719727, 'Pixel Count': 2048, 'acceleration_x_mean': -16211, 'acceleration_x_std': 47, 'acceleration_y_mean': -726, 'acceleration_y_std': 46, 'acceleration_z_mean': 1019, 'acceleration_z_std': 106} -2023-05-08 10:56:01,966 : DEBUG : Wavelength range: 163.96437072753906 -1145.4518535429638 -2023-05-08 10:56:02,079 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 10:56:02,079 : DEBUG : Reading Data spectra ... -2023-05-08 10:56:02,082 : DEBUG : Reading CRC32 ... -2023-05-08 10:56:02,084 : ERROR : Lattitude is not found, using default values instead for lat, lon, sza and saa. -2023-05-08 10:56:02,092 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:56:02,093 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220923T033032Failed: UnboundLocalError("local variable 'vaa' referenced before assignment") -2023-05-08 10:56:02,093 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 89, in process_sequence - l0_irr,l0_rad,l0_bla = reader.read_sequence(sequence_path,calibration_data_rad,calibration_data_irr) - File "/home/cgoyens/hypernets_processor/hypernets_processor/data_io/hypernets_reader.py", line 1041, in read_sequence - l0_irr = self.read_series( - File "/home/cgoyens/hypernets_processor/hypernets_processor/data_io/hypernets_reader.py", line 378, in read_series - ds["viewing_azimuth_angle"][scan_number] = vaa -UnboundLocalError: local variable 'vaa' referenced before assignment - -2023-05-08 10:56:02,097 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220923T040139 -2023-05-08 10:56:02,394 : INFO : Reading raw data... -2023-05-08 10:56:02,401 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 10:56:02,402 : DEBUG : {'Total Dataset Length': 4131, 'Spectrum Type Information': 136, 'acquisition_time': 1663906584557, 'integration_time': 16384, 'temperature': 29.982376098632812, 'Pixel Count': 2048, 'acceleration_x_mean': -16189, 'acceleration_x_std': 43, 'acceleration_y_mean': -722, 'acceleration_y_std': 46, 'acceleration_z_mean': 1044, 'acceleration_z_std': 98} -2023-05-08 10:56:02,411 : DEBUG : Wavelength range: 163.96437072753906 -1145.4518535429638 -2023-05-08 10:56:02,521 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 10:56:02,522 : DEBUG : Reading Data spectra ... -2023-05-08 10:56:02,527 : DEBUG : Reading CRC32 ... -2023-05-08 10:56:02,530 : ERROR : Lattitude is not found, using default values instead for lat, lon, sza and saa. -2023-05-08 10:56:02,537 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:56:02,537 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220923T040139Failed: UnboundLocalError("local variable 'vaa' referenced before assignment") -2023-05-08 10:56:02,538 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 89, in process_sequence - l0_irr,l0_rad,l0_bla = reader.read_sequence(sequence_path,calibration_data_rad,calibration_data_irr) - File "/home/cgoyens/hypernets_processor/hypernets_processor/data_io/hypernets_reader.py", line 1041, in read_sequence - l0_irr = self.read_series( - File "/home/cgoyens/hypernets_processor/hypernets_processor/data_io/hypernets_reader.py", line 378, in read_series - ds["viewing_azimuth_angle"][scan_number] = vaa -UnboundLocalError: local variable 'vaa' referenced before assignment - -2023-05-08 10:56:02,540 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220923T043147 -2023-05-08 10:56:02,781 : INFO : Reading raw data... -2023-05-08 10:56:02,787 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 10:56:02,787 : DEBUG : {'Total Dataset Length': 4131, 'Spectrum Type Information': 136, 'acquisition_time': 1663908390862, 'integration_time': 16384, 'temperature': 29.982376098632812, 'Pixel Count': 2048, 'acceleration_x_mean': -16211, 'acceleration_x_std': 45, 'acceleration_y_mean': -707, 'acceleration_y_std': 46, 'acceleration_z_mean': 1050, 'acceleration_z_std': 105} -2023-05-08 10:56:02,791 : DEBUG : Wavelength range: 163.96437072753906 -1145.4518535429638 -2023-05-08 10:56:02,903 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 10:56:02,903 : DEBUG : Reading Data spectra ... -2023-05-08 10:56:02,905 : DEBUG : Reading CRC32 ... -2023-05-08 10:56:02,906 : ERROR : Lattitude is not found, using default values instead for lat, lon, sza and saa. -2023-05-08 10:56:02,910 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:56:02,910 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220923T043147Failed: UnboundLocalError("local variable 'vaa' referenced before assignment") -2023-05-08 10:56:02,911 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 89, in process_sequence - l0_irr,l0_rad,l0_bla = reader.read_sequence(sequence_path,calibration_data_rad,calibration_data_irr) - File "/home/cgoyens/hypernets_processor/hypernets_processor/data_io/hypernets_reader.py", line 1041, in read_sequence - l0_irr = self.read_series( - File "/home/cgoyens/hypernets_processor/hypernets_processor/data_io/hypernets_reader.py", line 378, in read_series - ds["viewing_azimuth_angle"][scan_number] = vaa -UnboundLocalError: local variable 'vaa' referenced before assignment - -2023-05-08 10:56:02,913 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220923T050147 -2023-05-08 10:56:03,154 : INFO : Reading raw data... -2023-05-08 10:56:03,163 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 10:56:03,163 : DEBUG : {'Total Dataset Length': 4131, 'Spectrum Type Information': 136, 'acquisition_time': 1663910195633, 'integration_time': 16384, 'temperature': 30.005178451538086, 'Pixel Count': 2048, 'acceleration_x_mean': -16182, 'acceleration_x_std': 43, 'acceleration_y_mean': -721, 'acceleration_y_std': 47, 'acceleration_z_mean': 1032, 'acceleration_z_std': 99} -2023-05-08 10:56:03,166 : DEBUG : Wavelength range: 163.96437072753906 -1145.4518535429638 -2023-05-08 10:56:03,271 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 10:56:03,271 : DEBUG : Reading Data spectra ... -2023-05-08 10:56:03,272 : DEBUG : Reading CRC32 ... -2023-05-08 10:56:03,273 : ERROR : Lattitude is not found, using default values instead for lat, lon, sza and saa. -2023-05-08 10:56:03,277 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:56:03,278 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220923T050147Failed: UnboundLocalError("local variable 'vaa' referenced before assignment") -2023-05-08 10:56:03,278 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 89, in process_sequence - l0_irr,l0_rad,l0_bla = reader.read_sequence(sequence_path,calibration_data_rad,calibration_data_irr) - File "/home/cgoyens/hypernets_processor/hypernets_processor/data_io/hypernets_reader.py", line 1041, in read_sequence - l0_irr = self.read_series( - File "/home/cgoyens/hypernets_processor/hypernets_processor/data_io/hypernets_reader.py", line 378, in read_series - ds["viewing_azimuth_angle"][scan_number] = vaa -UnboundLocalError: local variable 'vaa' referenced before assignment - -2023-05-08 10:56:03,280 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220923T053026 -2023-05-08 10:56:03,519 : INFO : Reading raw data... -2023-05-08 10:56:03,526 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 10:56:03,526 : DEBUG : {'Total Dataset Length': 4131, 'Spectrum Type Information': 136, 'acquisition_time': 1663911905014, 'integration_time': 16384, 'temperature': 30.005178451538086, 'Pixel Count': 2048, 'acceleration_x_mean': -16179, 'acceleration_x_std': 44, 'acceleration_y_mean': -730, 'acceleration_y_std': 45, 'acceleration_z_mean': 1038, 'acceleration_z_std': 100} -2023-05-08 10:56:03,530 : DEBUG : Wavelength range: 163.96437072753906 -1145.4518535429638 -2023-05-08 10:56:03,639 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 10:56:03,639 : DEBUG : Reading Data spectra ... -2023-05-08 10:56:03,640 : DEBUG : Reading CRC32 ... -2023-05-08 10:56:03,642 : ERROR : Lattitude is not found, using default values instead for lat, lon, sza and saa. -2023-05-08 10:56:03,648 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:56:03,648 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220923T053026Failed: UnboundLocalError("local variable 'vaa' referenced before assignment") -2023-05-08 10:56:03,648 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 89, in process_sequence - l0_irr,l0_rad,l0_bla = reader.read_sequence(sequence_path,calibration_data_rad,calibration_data_irr) - File "/home/cgoyens/hypernets_processor/hypernets_processor/data_io/hypernets_reader.py", line 1041, in read_sequence - l0_irr = self.read_series( - File "/home/cgoyens/hypernets_processor/hypernets_processor/data_io/hypernets_reader.py", line 378, in read_series - ds["viewing_azimuth_angle"][scan_number] = vaa -UnboundLocalError: local variable 'vaa' referenced before assignment - -2023-05-08 10:56:03,650 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220923T060141 -2023-05-08 10:56:03,894 : INFO : Reading raw data... -2023-05-08 10:56:03,901 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 10:56:03,901 : DEBUG : {'Total Dataset Length': 4131, 'Spectrum Type Information': 136, 'acquisition_time': 1663913761351, 'integration_time': 16384, 'temperature': 29.822946548461914, 'Pixel Count': 2048, 'acceleration_x_mean': -16201, 'acceleration_x_std': 40, 'acceleration_y_mean': -737, 'acceleration_y_std': 45, 'acceleration_z_mean': 1041, 'acceleration_z_std': 100} -2023-05-08 10:56:03,904 : DEBUG : Wavelength range: 163.96437072753906 -1145.4518535429638 -2023-05-08 10:56:04,009 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 10:56:04,009 : DEBUG : Reading Data spectra ... -2023-05-08 10:56:04,010 : DEBUG : Reading CRC32 ... -2023-05-08 10:56:04,012 : ERROR : Lattitude is not found, using default values instead for lat, lon, sza and saa. -2023-05-08 10:56:04,016 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:56:04,016 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220923T060141Failed: UnboundLocalError("local variable 'vaa' referenced before assignment") -2023-05-08 10:56:04,016 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 89, in process_sequence - l0_irr,l0_rad,l0_bla = reader.read_sequence(sequence_path,calibration_data_rad,calibration_data_irr) - File "/home/cgoyens/hypernets_processor/hypernets_processor/data_io/hypernets_reader.py", line 1041, in read_sequence - l0_irr = self.read_series( - File "/home/cgoyens/hypernets_processor/hypernets_processor/data_io/hypernets_reader.py", line 378, in read_series - ds["viewing_azimuth_angle"][scan_number] = vaa -UnboundLocalError: local variable 'vaa' referenced before assignment - -2023-05-08 10:56:04,018 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220923T063149 -2023-05-08 10:56:04,255 : INFO : Reading raw data... -2023-05-08 10:56:04,262 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 10:56:04,262 : DEBUG : {'Total Dataset Length': 4131, 'Spectrum Type Information': 136, 'acquisition_time': 1663915610769, 'integration_time': 16384, 'temperature': 29.914012908935547, 'Pixel Count': 2048, 'acceleration_x_mean': -16204, 'acceleration_x_std': 44, 'acceleration_y_mean': -730, 'acceleration_y_std': 44, 'acceleration_z_mean': 1039, 'acceleration_z_std': 96} -2023-05-08 10:56:04,268 : DEBUG : Wavelength range: 163.96437072753906 -1145.4518535429638 -2023-05-08 10:56:04,371 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 10:56:04,371 : DEBUG : Reading Data spectra ... -2023-05-08 10:56:04,372 : DEBUG : Reading CRC32 ... -2023-05-08 10:56:04,373 : ERROR : Lattitude is not found, using default values instead for lat, lon, sza and saa. -2023-05-08 10:56:04,378 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:56:04,378 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220923T063149Failed: UnboundLocalError("local variable 'vaa' referenced before assignment") -2023-05-08 10:56:04,378 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 89, in process_sequence - l0_irr,l0_rad,l0_bla = reader.read_sequence(sequence_path,calibration_data_rad,calibration_data_irr) - File "/home/cgoyens/hypernets_processor/hypernets_processor/data_io/hypernets_reader.py", line 1041, in read_sequence - l0_irr = self.read_series( - File "/home/cgoyens/hypernets_processor/hypernets_processor/data_io/hypernets_reader.py", line 378, in read_series - ds["viewing_azimuth_angle"][scan_number] = vaa -UnboundLocalError: local variable 'vaa' referenced before assignment - -2023-05-08 10:56:04,380 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220923T070151 -2023-05-08 10:56:04,638 : INFO : Reading raw data... -2023-05-08 10:56:04,645 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 10:56:04,645 : DEBUG : {'Total Dataset Length': 4131, 'Spectrum Type Information': 136, 'acquisition_time': 1663917247903, 'integration_time': 16384, 'temperature': 29.754701614379883, 'Pixel Count': 2048, 'acceleration_x_mean': -16185, 'acceleration_x_std': 42, 'acceleration_y_mean': -728, 'acceleration_y_std': 46, 'acceleration_z_mean': 1062, 'acceleration_z_std': 97} -2023-05-08 10:56:04,649 : DEBUG : Wavelength range: 163.96437072753906 -1145.4518535429638 -2023-05-08 10:56:04,763 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 10:56:04,764 : DEBUG : Reading Data spectra ... -2023-05-08 10:56:04,765 : DEBUG : Reading CRC32 ... -2023-05-08 10:56:04,767 : ERROR : Lattitude is not found, using default values instead for lat, lon, sza and saa. -2023-05-08 10:56:04,771 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:56:04,772 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220923T070151Failed: UnboundLocalError("local variable 'vaa' referenced before assignment") -2023-05-08 10:56:04,772 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 89, in process_sequence - l0_irr,l0_rad,l0_bla = reader.read_sequence(sequence_path,calibration_data_rad,calibration_data_irr) - File "/home/cgoyens/hypernets_processor/hypernets_processor/data_io/hypernets_reader.py", line 1041, in read_sequence - l0_irr = self.read_series( - File "/home/cgoyens/hypernets_processor/hypernets_processor/data_io/hypernets_reader.py", line 378, in read_series - ds["viewing_azimuth_angle"][scan_number] = vaa -UnboundLocalError: local variable 'vaa' referenced before assignment - -2023-05-08 10:56:04,775 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220923T073150 -2023-05-08 10:56:05,028 : INFO : Reading raw data... -2023-05-08 10:56:05,035 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 10:56:05,035 : DEBUG : {'Total Dataset Length': 4131, 'Spectrum Type Information': 136, 'acquisition_time': 1663918718137, 'integration_time': 2048, 'temperature': 28.962291717529297, 'Pixel Count': 2048, 'acceleration_x_mean': -16201, 'acceleration_x_std': 38, 'acceleration_y_mean': -712, 'acceleration_y_std': 45, 'acceleration_z_mean': 1056, 'acceleration_z_std': 92} -2023-05-08 10:56:05,040 : DEBUG : Wavelength range: 163.96437072753906 -1145.4518535429638 -2023-05-08 10:56:05,172 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 10:56:05,172 : DEBUG : Reading Data spectra ... -2023-05-08 10:56:05,174 : DEBUG : Reading CRC32 ... -2023-05-08 10:56:05,176 : ERROR : Lattitude is not found, using default values instead for lat, lon, sza and saa. -2023-05-08 10:56:05,181 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:56:05,182 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220923T073150Failed: UnboundLocalError("local variable 'vaa' referenced before assignment") -2023-05-08 10:56:05,182 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 89, in process_sequence - l0_irr,l0_rad,l0_bla = reader.read_sequence(sequence_path,calibration_data_rad,calibration_data_irr) - File "/home/cgoyens/hypernets_processor/hypernets_processor/data_io/hypernets_reader.py", line 1041, in read_sequence - l0_irr = self.read_series( - File "/home/cgoyens/hypernets_processor/hypernets_processor/data_io/hypernets_reader.py", line 378, in read_series - ds["viewing_azimuth_angle"][scan_number] = vaa -UnboundLocalError: local variable 'vaa' referenced before assignment - -2023-05-08 10:56:05,185 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220923T080032 -2023-05-08 10:56:05,466 : INFO : Reading raw data... -2023-05-08 10:56:05,473 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 10:56:05,474 : DEBUG : {'Total Dataset Length': 4131, 'Spectrum Type Information': 136, 'acquisition_time': 1663920440509, 'integration_time': 2048, 'temperature': 28.782100677490234, 'Pixel Count': 2048, 'acceleration_x_mean': -16194, 'acceleration_x_std': 45, 'acceleration_y_mean': -717, 'acceleration_y_std': 45, 'acceleration_z_mean': 1037, 'acceleration_z_std': 94} -2023-05-08 10:56:05,478 : DEBUG : Wavelength range: 163.96437072753906 -1145.4518535429638 -2023-05-08 10:56:05,582 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 10:56:05,582 : DEBUG : Reading Data spectra ... -2023-05-08 10:56:05,583 : DEBUG : Reading CRC32 ... -2023-05-08 10:56:05,586 : ERROR : Lattitude is not found, using default values instead for lat, lon, sza and saa. -2023-05-08 10:56:05,595 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:56:05,596 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220923T080032Failed: UnboundLocalError("local variable 'vaa' referenced before assignment") -2023-05-08 10:56:05,596 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 89, in process_sequence - l0_irr,l0_rad,l0_bla = reader.read_sequence(sequence_path,calibration_data_rad,calibration_data_irr) - File "/home/cgoyens/hypernets_processor/hypernets_processor/data_io/hypernets_reader.py", line 1041, in read_sequence - l0_irr = self.read_series( - File "/home/cgoyens/hypernets_processor/hypernets_processor/data_io/hypernets_reader.py", line 378, in read_series - ds["viewing_azimuth_angle"][scan_number] = vaa -UnboundLocalError: local variable 'vaa' referenced before assignment - -2023-05-08 10:56:05,600 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220923T090300 -2023-05-08 10:56:05,865 : INFO : Reading raw data... -2023-05-08 10:56:05,872 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 10:56:05,872 : DEBUG : {'Total Dataset Length': 4131, 'Spectrum Type Information': 136, 'acquisition_time': 1663923976336, 'integration_time': 2048, 'temperature': 27.97527313232422, 'Pixel Count': 2048, 'acceleration_x_mean': -16244, 'acceleration_x_std': 43, 'acceleration_y_mean': -662, 'acceleration_y_std': 43, 'acceleration_z_mean': 463, 'acceleration_z_std': 123} -2023-05-08 10:56:05,876 : DEBUG : Wavelength range: 163.96437072753906 -1145.4518535429638 -2023-05-08 10:56:06,026 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 10:56:06,026 : DEBUG : Reading Data spectra ... -2023-05-08 10:56:06,028 : DEBUG : Reading CRC32 ... -2023-05-08 10:56:06,030 : ERROR : Lattitude is not found, using default values instead for lat, lon, sza and saa. -2023-05-08 10:56:06,034 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:56:06,034 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220923T090300Failed: UnboundLocalError("local variable 'vaa' referenced before assignment") -2023-05-08 10:56:06,035 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 89, in process_sequence - l0_irr,l0_rad,l0_bla = reader.read_sequence(sequence_path,calibration_data_rad,calibration_data_irr) - File "/home/cgoyens/hypernets_processor/hypernets_processor/data_io/hypernets_reader.py", line 1041, in read_sequence - l0_irr = self.read_series( - File "/home/cgoyens/hypernets_processor/hypernets_processor/data_io/hypernets_reader.py", line 378, in read_series - ds["viewing_azimuth_angle"][scan_number] = vaa -UnboundLocalError: local variable 'vaa' referenced before assignment - -2023-05-08 10:56:06,037 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220923T093032 -2023-05-08 10:56:06,286 : INFO : Reading raw data... -2023-05-08 10:56:06,294 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 10:56:06,294 : DEBUG : {'Total Dataset Length': 4131, 'Spectrum Type Information': 136, 'acquisition_time': 1663925652832, 'integration_time': 2048, 'temperature': 28.782100677490234, 'Pixel Count': 2048, 'acceleration_x_mean': -16228, 'acceleration_x_std': 44, 'acceleration_y_mean': -646, 'acceleration_y_std': 50, 'acceleration_z_mean': 463, 'acceleration_z_std': 122} -2023-05-08 10:56:06,297 : DEBUG : Wavelength range: 163.96437072753906 -1145.4518535429638 -2023-05-08 10:56:06,412 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 10:56:06,412 : DEBUG : Reading Data spectra ... -2023-05-08 10:56:06,413 : DEBUG : Reading CRC32 ... -2023-05-08 10:56:06,415 : ERROR : Lattitude is not found, using default values instead for lat, lon, sza and saa. -2023-05-08 10:56:06,419 : INFO : Processing Anomalies: ['x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'] -2023-05-08 10:56:06,419 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220923T093032Failed: UnboundLocalError("local variable 'vaa' referenced before assignment") -2023-05-08 10:56:06,419 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 89, in process_sequence - l0_irr,l0_rad,l0_bla = reader.read_sequence(sequence_path,calibration_data_rad,calibration_data_irr) - File "/home/cgoyens/hypernets_processor/hypernets_processor/data_io/hypernets_reader.py", line 1041, in read_sequence - l0_irr = self.read_series( - File "/home/cgoyens/hypernets_processor/hypernets_processor/data_io/hypernets_reader.py", line 378, in read_series - ds["viewing_azimuth_angle"][scan_number] = vaa -UnboundLocalError: local variable 'vaa' referenced before assignment - -2023-05-08 10:56:06,422 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/DATA/SEQ20220923T100025 -2023-05-08 10:56:26,212 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/Test/SEQ20230410T143155 -2023-05-08 10:56:26,550 : INFO : Reading raw data... -2023-05-08 10:56:26,557 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 10:56:26,557 : DEBUG : {'Total Dataset Length': 4131, 'Spectrum Type Information': 136, 'acquisition_time': 1681137276013, 'integration_time': 1024, 'temperature': 16.44601821899414, 'Pixel Count': 2048, 'acceleration_x_mean': -16327, 'acceleration_x_std': 91, 'acceleration_y_mean': -396, 'acceleration_y_std': 243, 'acceleration_z_mean': 116, 'acceleration_z_std': 453} -2023-05-08 10:56:26,561 : DEBUG : Wavelength range: 163.96437072753906 -1145.4518535429638 -2023-05-08 10:56:26,695 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 10:56:26,695 : DEBUG : Reading Data spectra ... -2023-05-08 10:56:26,698 : DEBUG : Reading CRC32 ... -2023-05-08 10:56:26,728 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 10:56:26,728 : DEBUG : Reading Data spectra ... -2023-05-08 10:56:26,731 : DEBUG : Reading CRC32 ... -2023-05-08 10:56:26,758 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 10:56:26,759 : DEBUG : Reading Data spectra ... -2023-05-08 10:56:26,761 : DEBUG : Reading CRC32 ... -2023-05-08 10:56:26,790 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 10:56:26,790 : DEBUG : Reading Data spectra ... -2023-05-08 10:56:26,792 : DEBUG : Reading CRC32 ... -2023-05-08 10:56:26,820 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 10:56:26,820 : DEBUG : Reading Data spectra ... -2023-05-08 10:56:26,823 : DEBUG : Reading CRC32 ... -2023-05-08 10:56:26,848 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 10:56:26,848 : DEBUG : Reading Data spectra ... -2023-05-08 10:56:26,851 : DEBUG : Reading CRC32 ... -2023-05-08 10:56:27,046 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 10:56:27,046 : DEBUG : {'Total Dataset Length': 4131, 'Spectrum Type Information': 144, 'acquisition_time': 1681137227706, 'integration_time': 256, 'temperature': 15.911444664001465, 'Pixel Count': 2048, 'acceleration_x_mean': 12620, 'acceleration_x_std': 638, 'acceleration_y_mean': -10200, 'acceleration_y_std': 784, 'acceleration_z_mean': -2932, 'acceleration_z_std': 499} -2023-05-08 10:56:27,054 : DEBUG : Wavelength range: 163.85150146484375 -1145.61306723725 -2023-05-08 10:56:27,172 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 10:56:27,172 : DEBUG : Reading Data spectra ... -2023-05-08 10:56:27,173 : DEBUG : Reading CRC32 ... -2023-05-08 10:56:27,189 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 10:56:27,189 : DEBUG : Reading Data spectra ... -2023-05-08 10:56:27,190 : DEBUG : Reading CRC32 ... -2023-05-08 10:56:27,206 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 10:56:27,206 : DEBUG : Reading Data spectra ... -2023-05-08 10:56:27,207 : DEBUG : Reading CRC32 ... -2023-05-08 10:56:27,222 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 10:56:27,223 : DEBUG : Reading Data spectra ... -2023-05-08 10:56:27,224 : DEBUG : Reading CRC32 ... -2023-05-08 10:56:27,239 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 10:56:27,239 : DEBUG : Reading Data spectra ... -2023-05-08 10:56:27,240 : DEBUG : Reading CRC32 ... -2023-05-08 10:56:27,255 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 10:56:27,256 : DEBUG : Reading Data spectra ... -2023-05-08 10:56:27,257 : DEBUG : Reading CRC32 ... -2023-05-08 10:56:27,272 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 10:56:27,272 : DEBUG : Reading Data spectra ... -2023-05-08 10:56:27,274 : DEBUG : Reading CRC32 ... -2023-05-08 10:56:27,289 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 10:56:27,289 : DEBUG : Reading Data spectra ... -2023-05-08 10:56:27,290 : DEBUG : Reading CRC32 ... -2023-05-08 10:56:27,305 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 10:56:27,305 : DEBUG : Reading Data spectra ... -2023-05-08 10:56:27,307 : DEBUG : Reading CRC32 ... -2023-05-08 10:56:27,325 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 10:56:27,325 : DEBUG : Reading Data spectra ... -2023-05-08 10:56:27,326 : DEBUG : Reading CRC32 ... -2023-05-08 10:56:27,341 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 10:56:27,341 : DEBUG : Reading Data spectra ... -2023-05-08 10:56:27,343 : DEBUG : Reading CRC32 ... -2023-05-08 10:56:27,357 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 10:56:27,358 : DEBUG : Reading Data spectra ... -2023-05-08 10:56:27,359 : DEBUG : Reading CRC32 ... -2023-05-08 10:56:27,498 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 10:56:27,498 : DEBUG : {'Total Dataset Length': 4131, 'Spectrum Type Information': 128, 'acquisition_time': 1681137201860, 'integration_time': 32, 'temperature': 15.547460556030273, 'Pixel Count': 2048, 'acceleration_x_mean': -12608, 'acceleration_x_std': 0, 'acceleration_y_mean': -10176, 'acceleration_y_std': 0, 'acceleration_z_mean': -2752, 'acceleration_z_std': 0} -2023-05-08 10:56:27,502 : DEBUG : Wavelength range: 163.85150146484375 -1145.61306723725 -2023-05-08 10:56:27,606 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 10:56:27,606 : DEBUG : Reading Data spectra ... -2023-05-08 10:56:27,609 : DEBUG : Reading CRC32 ... -2023-05-08 10:56:27,629 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 10:56:27,630 : DEBUG : Reading Data spectra ... -2023-05-08 10:56:27,631 : DEBUG : Reading CRC32 ... -2023-05-08 10:56:27,646 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 10:56:27,646 : DEBUG : Reading Data spectra ... -2023-05-08 10:56:27,647 : DEBUG : Reading CRC32 ... -2023-05-08 10:56:27,663 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 10:56:27,663 : DEBUG : Reading Data spectra ... -2023-05-08 10:56:27,665 : DEBUG : Reading CRC32 ... -2023-05-08 10:56:27,679 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 10:56:27,680 : DEBUG : Reading Data spectra ... -2023-05-08 10:56:27,681 : DEBUG : Reading CRC32 ... -2023-05-08 10:56:27,696 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 10:56:27,696 : DEBUG : Reading Data spectra ... -2023-05-08 10:56:27,697 : DEBUG : Reading CRC32 ... -2023-05-08 10:56:27,713 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 10:56:27,713 : DEBUG : Reading Data spectra ... -2023-05-08 10:56:27,715 : DEBUG : Reading CRC32 ... -2023-05-08 10:56:27,730 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 10:56:27,730 : DEBUG : Reading Data spectra ... -2023-05-08 10:56:27,731 : DEBUG : Reading CRC32 ... -2023-05-08 10:56:27,746 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 10:56:27,746 : DEBUG : Reading Data spectra ... -2023-05-08 10:56:27,747 : DEBUG : Reading CRC32 ... -2023-05-08 10:56:27,762 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 10:56:27,763 : DEBUG : Reading Data spectra ... -2023-05-08 10:56:27,764 : DEBUG : Reading CRC32 ... -2023-05-08 10:56:27,779 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 10:56:27,779 : DEBUG : Reading Data spectra ... -2023-05-08 10:56:27,780 : DEBUG : Reading CRC32 ... -2023-05-08 10:56:27,795 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 10:56:27,795 : DEBUG : Reading Data spectra ... -2023-05-08 10:56:27,796 : DEBUG : Reading CRC32 ... -2023-05-08 10:56:27,812 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 10:56:27,813 : DEBUG : Reading Data spectra ... -2023-05-08 10:56:27,814 : DEBUG : Reading CRC32 ... -2023-05-08 10:56:27,832 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 10:56:27,832 : DEBUG : Reading Data spectra ... -2023-05-08 10:56:27,834 : DEBUG : Reading CRC32 ... -2023-05-08 10:56:27,857 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 10:56:27,857 : DEBUG : Reading Data spectra ... -2023-05-08 10:56:27,859 : DEBUG : Reading CRC32 ... -2023-05-08 10:56:27,997 : INFO : Done -2023-05-08 10:56:27,997 : INFO : Processing to L1a... -2023-05-08 10:56:28,883 : DEBUG : preprocessing done -2023-05-08 10:56:57,810 : DEBUG : preprocessing done -2023-05-08 10:57:26,934 : INFO : Done -2023-05-08 10:57:26,935 : INFO : Processing to L1b radiance... -2023-05-08 10:57:34,920 : INFO : Done -2023-05-08 10:57:34,920 : INFO : Processing to L1b irradiance... -2023-05-08 10:57:40,985 : INFO : Done -2023-05-08 10:57:40,986 : INFO : Processing to L1c... -2023-05-08 10:57:41,579 : DEBUG : WaterNetworkProtocol -2023-05-08 10:57:41,662 : DEBUG : 1681137226 -2023-05-08 10:57:41,668 : DEBUG : 1681137226 -2023-05-08 10:57:41,674 : DEBUG : 1681137226 -2023-05-08 10:57:41,679 : DEBUG : 1681137226 -2023-05-08 10:57:41,685 : DEBUG : 1681137226 -2023-05-08 10:57:41,691 : DEBUG : 1681137226 -2023-05-08 10:57:49,587 : INFO : interpolate sky radiance -2023-05-08 10:58:09,440 : INFO : interpolate irradiances -2023-05-08 11:17:23,952 : INFO : Processing Anomalies: ['x'] -2023-05-08 11:17:23,956 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/Test/SEQ20230410T143155Failed: UnboundLocalError("local variable 'wind' referenced before assignment") -2023-05-08 11:17:23,986 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 142, in process_sequence - L1c = surf.process_l1c(L1c_int, L1b_irr) - File "/home/cgoyens/hypernets_processor/hypernets_processor/surface_reflectance/surface_reflectance.py", line 51, in process_l1c - dataset_l1c = self.rh.get_wind(dataset_l1c) - File "/home/cgoyens/hypernets_processor/hypernets_processor/rhymer/rhymer/hypstar/rhymer_hypstar.py", line 299, in get_wind - anc_wind = self.rhymeranc.ts_wind(isotime, l1b.attrs['site_id']) - File "/home/cgoyens/hypernets_processor/hypernets_processor/rhymer/rhymer/ancillary/rhymer_ancillary.py", line 59, in ts_wind - return (wind) -UnboundLocalError: local variable 'wind' referenced before assignment - -2023-05-08 11:17:23,995 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/Test/SEQ20230410T140203 -2023-05-08 11:17:24,310 : INFO : Reading raw data... -2023-05-08 11:17:24,316 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 11:17:24,316 : DEBUG : {'Total Dataset Length': 4131, 'Spectrum Type Information': 136, 'acquisition_time': 1681135485382, 'integration_time': 1024, 'temperature': 16.488752365112305, 'Pixel Count': 2048, 'acceleration_x_mean': -16303, 'acceleration_x_std': 131, 'acceleration_y_mean': -429, 'acceleration_y_std': 595, 'acceleration_z_mean': -29, 'acceleration_z_std': 643} -2023-05-08 11:17:24,320 : DEBUG : Wavelength range: 163.96437072753906 -1145.4518535429638 -2023-05-08 11:17:24,447 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 11:17:24,447 : DEBUG : Reading Data spectra ... -2023-05-08 11:17:24,449 : DEBUG : Reading CRC32 ... -2023-05-08 11:17:24,465 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 11:17:24,465 : DEBUG : Reading Data spectra ... -2023-05-08 11:17:24,467 : DEBUG : Reading CRC32 ... -2023-05-08 11:17:24,482 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 11:17:24,482 : DEBUG : Reading Data spectra ... -2023-05-08 11:17:24,484 : DEBUG : Reading CRC32 ... -2023-05-08 11:17:24,499 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 11:17:24,500 : DEBUG : Reading Data spectra ... -2023-05-08 11:17:24,501 : DEBUG : Reading CRC32 ... -2023-05-08 11:17:24,516 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 11:17:24,516 : DEBUG : Reading Data spectra ... -2023-05-08 11:17:24,517 : DEBUG : Reading CRC32 ... -2023-05-08 11:17:24,532 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 11:17:24,533 : DEBUG : Reading Data spectra ... -2023-05-08 11:17:24,534 : DEBUG : Reading CRC32 ... -2023-05-08 11:17:24,645 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 11:17:24,645 : DEBUG : {'Total Dataset Length': 4131, 'Spectrum Type Information': 144, 'acquisition_time': 1681135438327, 'integration_time': 256, 'temperature': 16.039813995361328, 'Pixel Count': 2048, 'acceleration_x_mean': 12473, 'acceleration_x_std': 262, 'acceleration_y_mean': -10256, 'acceleration_y_std': 292, 'acceleration_z_mean': -2960, 'acceleration_z_std': 469} -2023-05-08 11:17:24,649 : DEBUG : Wavelength range: 163.85150146484375 -1145.61306723725 -2023-05-08 11:17:24,774 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 11:17:24,774 : DEBUG : Reading Data spectra ... -2023-05-08 11:17:24,776 : DEBUG : Reading CRC32 ... -2023-05-08 11:17:24,792 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 11:17:24,792 : DEBUG : Reading Data spectra ... -2023-05-08 11:17:24,794 : DEBUG : Reading CRC32 ... -2023-05-08 11:17:24,810 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 11:17:24,810 : DEBUG : Reading Data spectra ... -2023-05-08 11:17:24,812 : DEBUG : Reading CRC32 ... -2023-05-08 11:17:24,829 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 11:17:24,829 : DEBUG : Reading Data spectra ... -2023-05-08 11:17:24,830 : DEBUG : Reading CRC32 ... -2023-05-08 11:17:24,853 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 11:17:24,853 : DEBUG : Reading Data spectra ... -2023-05-08 11:17:24,854 : DEBUG : Reading CRC32 ... -2023-05-08 11:17:24,870 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 11:17:24,870 : DEBUG : Reading Data spectra ... -2023-05-08 11:17:24,872 : DEBUG : Reading CRC32 ... -2023-05-08 11:17:24,887 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 11:17:24,887 : DEBUG : Reading Data spectra ... -2023-05-08 11:17:24,888 : DEBUG : Reading CRC32 ... -2023-05-08 11:17:24,904 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 11:17:24,904 : DEBUG : Reading Data spectra ... -2023-05-08 11:17:24,905 : DEBUG : Reading CRC32 ... -2023-05-08 11:17:24,920 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 11:17:24,920 : DEBUG : Reading Data spectra ... -2023-05-08 11:17:24,922 : DEBUG : Reading CRC32 ... -2023-05-08 11:17:24,938 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 11:17:24,939 : DEBUG : Reading Data spectra ... -2023-05-08 11:17:24,940 : DEBUG : Reading CRC32 ... -2023-05-08 11:17:24,955 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 11:17:24,955 : DEBUG : Reading Data spectra ... -2023-05-08 11:17:24,957 : DEBUG : Reading CRC32 ... -2023-05-08 11:17:24,972 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 11:17:24,972 : DEBUG : Reading Data spectra ... -2023-05-08 11:17:24,973 : DEBUG : Reading CRC32 ... -2023-05-08 11:17:25,107 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 11:17:25,108 : DEBUG : {'Total Dataset Length': 4131, 'Spectrum Type Information': 128, 'acquisition_time': 1681135412063, 'integration_time': 32, 'temperature': 15.654559135437012, 'Pixel Count': 2048, 'acceleration_x_mean': -12992, 'acceleration_x_std': 0, 'acceleration_y_mean': -9856, 'acceleration_y_std': 0, 'acceleration_z_mean': -2048, 'acceleration_z_std': 0} -2023-05-08 11:17:25,111 : DEBUG : Wavelength range: 163.85150146484375 -1145.61306723725 -2023-05-08 11:17:25,230 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 11:17:25,230 : DEBUG : Reading Data spectra ... -2023-05-08 11:17:25,232 : DEBUG : Reading CRC32 ... -2023-05-08 11:17:25,249 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 11:17:25,249 : DEBUG : Reading Data spectra ... -2023-05-08 11:17:25,251 : DEBUG : Reading CRC32 ... -2023-05-08 11:17:25,267 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 11:17:25,267 : DEBUG : Reading Data spectra ... -2023-05-08 11:17:25,269 : DEBUG : Reading CRC32 ... -2023-05-08 11:17:25,285 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 11:17:25,285 : DEBUG : Reading Data spectra ... -2023-05-08 11:17:25,287 : DEBUG : Reading CRC32 ... -2023-05-08 11:17:25,303 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 11:17:25,303 : DEBUG : Reading Data spectra ... -2023-05-08 11:17:25,304 : DEBUG : Reading CRC32 ... -2023-05-08 11:17:25,319 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 11:17:25,320 : DEBUG : Reading Data spectra ... -2023-05-08 11:17:25,321 : DEBUG : Reading CRC32 ... -2023-05-08 11:17:25,340 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 11:17:25,340 : DEBUG : Reading Data spectra ... -2023-05-08 11:17:25,343 : DEBUG : Reading CRC32 ... -2023-05-08 11:17:25,362 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 11:17:25,362 : DEBUG : Reading Data spectra ... -2023-05-08 11:17:25,364 : DEBUG : Reading CRC32 ... -2023-05-08 11:17:25,384 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 11:17:25,384 : DEBUG : Reading Data spectra ... -2023-05-08 11:17:25,387 : DEBUG : Reading CRC32 ... -2023-05-08 11:17:25,403 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 11:17:25,403 : DEBUG : Reading Data spectra ... -2023-05-08 11:17:25,405 : DEBUG : Reading CRC32 ... -2023-05-08 11:17:25,423 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 11:17:25,423 : DEBUG : Reading Data spectra ... -2023-05-08 11:17:25,425 : DEBUG : Reading CRC32 ... -2023-05-08 11:17:25,440 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 11:17:25,440 : DEBUG : Reading Data spectra ... -2023-05-08 11:17:25,442 : DEBUG : Reading CRC32 ... -2023-05-08 11:17:25,461 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 11:17:25,461 : DEBUG : Reading Data spectra ... -2023-05-08 11:17:25,463 : DEBUG : Reading CRC32 ... -2023-05-08 11:17:25,480 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 11:17:25,480 : DEBUG : Reading Data spectra ... -2023-05-08 11:17:25,481 : DEBUG : Reading CRC32 ... -2023-05-08 11:17:25,497 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 11:17:25,497 : DEBUG : Reading Data spectra ... -2023-05-08 11:17:25,498 : DEBUG : Reading CRC32 ... -2023-05-08 11:17:25,616 : INFO : Done -2023-05-08 11:17:25,616 : INFO : Processing to L1a... -2023-05-08 11:17:26,518 : DEBUG : preprocessing done -2023-05-08 11:17:59,153 : DEBUG : preprocessing done -2023-05-08 11:18:34,223 : INFO : Done -2023-05-08 11:18:34,223 : INFO : Processing to L1b radiance... -2023-05-08 11:18:48,292 : INFO : Done -2023-05-08 11:18:48,292 : INFO : Processing to L1b irradiance... -2023-05-08 11:19:02,289 : INFO : Done -2023-05-08 11:19:02,290 : INFO : Processing to L1c... -2023-05-08 11:19:03,096 : DEBUG : WaterNetworkProtocol -2023-05-08 11:19:03,232 : DEBUG : 1681135437 -2023-05-08 11:19:03,241 : DEBUG : 1681135437 -2023-05-08 11:19:03,246 : DEBUG : 1681135437 -2023-05-08 11:19:03,256 : DEBUG : 1681135437 -2023-05-08 11:19:03,262 : DEBUG : 1681135437 -2023-05-08 11:19:03,275 : DEBUG : 1681135437 -2023-05-08 11:19:17,671 : INFO : interpolate sky radiance -2023-05-08 11:19:49,978 : INFO : interpolate irradiances -2023-05-08 11:23:53,631 : INFO : Processing Anomalies: ['x', 'x'] -2023-05-08 11:23:53,632 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/Test/SEQ20230410T140203Failed: UnboundLocalError("local variable 'wind' referenced before assignment") -2023-05-08 11:23:53,634 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 142, in process_sequence - L1c = surf.process_l1c(L1c_int, L1b_irr) - File "/home/cgoyens/hypernets_processor/hypernets_processor/surface_reflectance/surface_reflectance.py", line 51, in process_l1c - def process_l1c(self, dataset, l1birr): - File "/home/cgoyens/hypernets_processor/hypernets_processor/rhymer/rhymer/hypstar/rhymer_hypstar.py", line 299, in get_wind - anc_wind = self.rhymeranc.ts_wind(isotime, l1b.attrs['site_id']) - File "/home/cgoyens/hypernets_processor/hypernets_processor/rhymer/rhymer/ancillary/rhymer_ancillary.py", line 59, in ts_wind - return (wind) -UnboundLocalError: local variable 'wind' referenced before assignment - -2023-05-08 11:26:58,351 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/Test/SEQ20230410T143155 -2023-05-08 11:26:58,709 : INFO : Reading raw data... -2023-05-08 11:26:58,717 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 11:26:58,717 : DEBUG : {'Total Dataset Length': 4131, 'Spectrum Type Information': 136, 'acquisition_time': 1681137276013, 'integration_time': 1024, 'temperature': 16.44601821899414, 'Pixel Count': 2048, 'acceleration_x_mean': -16327, 'acceleration_x_std': 91, 'acceleration_y_mean': -396, 'acceleration_y_std': 243, 'acceleration_z_mean': 116, 'acceleration_z_std': 453} -2023-05-08 11:26:58,721 : DEBUG : Wavelength range: 163.96437072753906 -1145.4518535429638 -2023-05-08 11:26:58,871 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 11:26:58,871 : DEBUG : Reading Data spectra ... -2023-05-08 11:26:58,873 : DEBUG : Reading CRC32 ... -2023-05-08 11:26:58,893 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 11:26:58,893 : DEBUG : Reading Data spectra ... -2023-05-08 11:26:58,896 : DEBUG : Reading CRC32 ... -2023-05-08 11:26:58,918 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 11:26:58,918 : DEBUG : Reading Data spectra ... -2023-05-08 11:26:58,920 : DEBUG : Reading CRC32 ... -2023-05-08 11:26:58,937 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 11:26:58,937 : DEBUG : Reading Data spectra ... -2023-05-08 11:26:58,938 : DEBUG : Reading CRC32 ... -2023-05-08 11:26:58,957 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 11:26:58,957 : DEBUG : Reading Data spectra ... -2023-05-08 11:26:58,958 : DEBUG : Reading CRC32 ... -2023-05-08 11:26:58,975 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 11:26:58,976 : DEBUG : Reading Data spectra ... -2023-05-08 11:26:58,977 : DEBUG : Reading CRC32 ... -2023-05-08 11:26:59,106 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 11:26:59,106 : DEBUG : {'Total Dataset Length': 4131, 'Spectrum Type Information': 144, 'acquisition_time': 1681137227706, 'integration_time': 256, 'temperature': 15.911444664001465, 'Pixel Count': 2048, 'acceleration_x_mean': 12620, 'acceleration_x_std': 638, 'acceleration_y_mean': -10200, 'acceleration_y_std': 784, 'acceleration_z_mean': -2932, 'acceleration_z_std': 499} -2023-05-08 11:26:59,110 : DEBUG : Wavelength range: 163.85150146484375 -1145.61306723725 -2023-05-08 11:26:59,237 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 11:26:59,237 : DEBUG : Reading Data spectra ... -2023-05-08 11:26:59,239 : DEBUG : Reading CRC32 ... -2023-05-08 11:26:59,258 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 11:26:59,258 : DEBUG : Reading Data spectra ... -2023-05-08 11:26:59,259 : DEBUG : Reading CRC32 ... -2023-05-08 11:26:59,276 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 11:26:59,276 : DEBUG : Reading Data spectra ... -2023-05-08 11:26:59,278 : DEBUG : Reading CRC32 ... -2023-05-08 11:26:59,294 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 11:26:59,295 : DEBUG : Reading Data spectra ... -2023-05-08 11:26:59,296 : DEBUG : Reading CRC32 ... -2023-05-08 11:26:59,314 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 11:26:59,314 : DEBUG : Reading Data spectra ... -2023-05-08 11:26:59,315 : DEBUG : Reading CRC32 ... -2023-05-08 11:26:59,337 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 11:26:59,337 : DEBUG : Reading Data spectra ... -2023-05-08 11:26:59,338 : DEBUG : Reading CRC32 ... -2023-05-08 11:26:59,356 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 11:26:59,357 : DEBUG : Reading Data spectra ... -2023-05-08 11:26:59,358 : DEBUG : Reading CRC32 ... -2023-05-08 11:26:59,376 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 11:26:59,376 : DEBUG : Reading Data spectra ... -2023-05-08 11:26:59,378 : DEBUG : Reading CRC32 ... -2023-05-08 11:26:59,414 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 11:26:59,414 : DEBUG : Reading Data spectra ... -2023-05-08 11:26:59,417 : DEBUG : Reading CRC32 ... -2023-05-08 11:26:59,454 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 11:26:59,454 : DEBUG : Reading Data spectra ... -2023-05-08 11:26:59,455 : DEBUG : Reading CRC32 ... -2023-05-08 11:26:59,494 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 11:26:59,495 : DEBUG : Reading Data spectra ... -2023-05-08 11:26:59,496 : DEBUG : Reading CRC32 ... -2023-05-08 11:26:59,522 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 11:26:59,522 : DEBUG : Reading Data spectra ... -2023-05-08 11:26:59,524 : DEBUG : Reading CRC32 ... -2023-05-08 11:26:59,662 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 11:26:59,662 : DEBUG : {'Total Dataset Length': 4131, 'Spectrum Type Information': 128, 'acquisition_time': 1681137201860, 'integration_time': 32, 'temperature': 15.547460556030273, 'Pixel Count': 2048, 'acceleration_x_mean': -12608, 'acceleration_x_std': 0, 'acceleration_y_mean': -10176, 'acceleration_y_std': 0, 'acceleration_z_mean': -2752, 'acceleration_z_std': 0} -2023-05-08 11:26:59,669 : DEBUG : Wavelength range: 163.85150146484375 -1145.61306723725 -2023-05-08 11:26:59,862 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 11:26:59,863 : DEBUG : Reading Data spectra ... -2023-05-08 11:26:59,865 : DEBUG : Reading CRC32 ... -2023-05-08 11:26:59,892 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 11:26:59,892 : DEBUG : Reading Data spectra ... -2023-05-08 11:26:59,895 : DEBUG : Reading CRC32 ... -2023-05-08 11:26:59,910 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 11:26:59,910 : DEBUG : Reading Data spectra ... -2023-05-08 11:26:59,912 : DEBUG : Reading CRC32 ... -2023-05-08 11:26:59,929 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 11:26:59,929 : DEBUG : Reading Data spectra ... -2023-05-08 11:26:59,930 : DEBUG : Reading CRC32 ... -2023-05-08 11:26:59,945 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 11:26:59,946 : DEBUG : Reading Data spectra ... -2023-05-08 11:26:59,947 : DEBUG : Reading CRC32 ... -2023-05-08 11:26:59,963 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 11:26:59,963 : DEBUG : Reading Data spectra ... -2023-05-08 11:26:59,964 : DEBUG : Reading CRC32 ... -2023-05-08 11:26:59,982 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 11:26:59,982 : DEBUG : Reading Data spectra ... -2023-05-08 11:26:59,984 : DEBUG : Reading CRC32 ... -2023-05-08 11:27:00,000 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 11:27:00,000 : DEBUG : Reading Data spectra ... -2023-05-08 11:27:00,002 : DEBUG : Reading CRC32 ... -2023-05-08 11:27:00,020 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 11:27:00,020 : DEBUG : Reading Data spectra ... -2023-05-08 11:27:00,021 : DEBUG : Reading CRC32 ... -2023-05-08 11:27:00,038 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 11:27:00,038 : DEBUG : Reading Data spectra ... -2023-05-08 11:27:00,039 : DEBUG : Reading CRC32 ... -2023-05-08 11:27:00,060 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 11:27:00,060 : DEBUG : Reading Data spectra ... -2023-05-08 11:27:00,061 : DEBUG : Reading CRC32 ... -2023-05-08 11:27:00,078 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 11:27:00,079 : DEBUG : Reading Data spectra ... -2023-05-08 11:27:00,081 : DEBUG : Reading CRC32 ... -2023-05-08 11:27:00,118 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 11:27:00,118 : DEBUG : Reading Data spectra ... -2023-05-08 11:27:00,121 : DEBUG : Reading CRC32 ... -2023-05-08 11:27:00,171 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 11:27:00,171 : DEBUG : Reading Data spectra ... -2023-05-08 11:27:00,174 : DEBUG : Reading CRC32 ... -2023-05-08 11:27:00,211 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 11:27:00,212 : DEBUG : Reading Data spectra ... -2023-05-08 11:27:00,213 : DEBUG : Reading CRC32 ... -2023-05-08 11:27:00,363 : INFO : Done -2023-05-08 11:27:00,363 : INFO : Processing to L1a... -2023-05-08 11:27:01,348 : DEBUG : preprocessing done -2023-05-08 11:27:27,539 : DEBUG : preprocessing done -2023-05-08 11:27:48,242 : INFO : Done -2023-05-08 11:27:48,242 : INFO : Processing to L1b radiance... -2023-05-08 11:27:55,787 : INFO : Done -2023-05-08 11:27:55,788 : INFO : Processing to L1b irradiance... -2023-05-08 11:28:01,272 : INFO : Done -2023-05-08 11:28:01,272 : INFO : Processing to L1c... -2023-05-08 11:28:01,671 : DEBUG : WaterNetworkProtocol -2023-05-08 11:28:01,747 : DEBUG : 1681137226 -2023-05-08 11:28:01,752 : DEBUG : 1681137226 -2023-05-08 11:28:01,757 : DEBUG : 1681137226 -2023-05-08 11:28:01,764 : DEBUG : 1681137226 -2023-05-08 11:28:01,771 : DEBUG : 1681137226 -2023-05-08 11:28:01,779 : DEBUG : 1681137226 -2023-05-08 11:28:09,795 : INFO : interpolate sky radiance -2023-05-08 11:28:31,700 : INFO : interpolate irradiances -2023-05-08 12:04:41,526 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/Test/SEQ20230410T143155 -2023-05-08 12:04:41,681 : INFO : Reading raw data... -2023-05-08 12:04:41,683 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 12:04:41,683 : DEBUG : {'Total Dataset Length': 4131, 'Spectrum Type Information': 136, 'acquisition_time': 1681137276013, 'integration_time': 1024, 'temperature': 16.44601821899414, 'Pixel Count': 2048, 'acceleration_x_mean': -16327, 'acceleration_x_std': 91, 'acceleration_y_mean': -396, 'acceleration_y_std': 243, 'acceleration_z_mean': 116, 'acceleration_z_std': 453} -2023-05-08 12:04:41,685 : DEBUG : Wavelength range: 163.96437072753906 -1145.4518535429638 -2023-05-08 12:04:41,716 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 12:04:41,716 : DEBUG : Reading Data spectra ... -2023-05-08 12:04:41,717 : DEBUG : Reading CRC32 ... -2023-05-08 12:04:41,722 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 12:04:41,722 : DEBUG : Reading Data spectra ... -2023-05-08 12:04:41,723 : DEBUG : Reading CRC32 ... -2023-05-08 12:04:41,731 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 12:04:41,731 : DEBUG : Reading Data spectra ... -2023-05-08 12:04:41,732 : DEBUG : Reading CRC32 ... -2023-05-08 12:04:41,738 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 12:04:41,738 : DEBUG : Reading Data spectra ... -2023-05-08 12:04:41,739 : DEBUG : Reading CRC32 ... -2023-05-08 12:04:41,744 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 12:04:41,744 : DEBUG : Reading Data spectra ... -2023-05-08 12:04:41,745 : DEBUG : Reading CRC32 ... -2023-05-08 12:04:41,750 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 12:04:41,750 : DEBUG : Reading Data spectra ... -2023-05-08 12:04:41,751 : DEBUG : Reading CRC32 ... -2023-05-08 12:04:41,806 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 12:04:41,806 : DEBUG : {'Total Dataset Length': 4131, 'Spectrum Type Information': 144, 'acquisition_time': 1681137227706, 'integration_time': 256, 'temperature': 15.911444664001465, 'Pixel Count': 2048, 'acceleration_x_mean': 12620, 'acceleration_x_std': 638, 'acceleration_y_mean': -10200, 'acceleration_y_std': 784, 'acceleration_z_mean': -2932, 'acceleration_z_std': 499} -2023-05-08 12:04:41,808 : DEBUG : Wavelength range: 163.85150146484375 -1145.61306723725 -2023-05-08 12:04:41,834 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 12:04:41,834 : DEBUG : Reading Data spectra ... -2023-05-08 12:04:41,835 : DEBUG : Reading CRC32 ... -2023-05-08 12:04:41,840 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 12:04:41,841 : DEBUG : Reading Data spectra ... -2023-05-08 12:04:41,841 : DEBUG : Reading CRC32 ... -2023-05-08 12:04:41,846 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 12:04:41,846 : DEBUG : Reading Data spectra ... -2023-05-08 12:04:41,847 : DEBUG : Reading CRC32 ... -2023-05-08 12:04:41,854 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 12:04:41,854 : DEBUG : Reading Data spectra ... -2023-05-08 12:04:41,856 : DEBUG : Reading CRC32 ... -2023-05-08 12:04:41,863 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 12:04:41,863 : DEBUG : Reading Data spectra ... -2023-05-08 12:04:41,864 : DEBUG : Reading CRC32 ... -2023-05-08 12:04:41,868 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 12:04:41,869 : DEBUG : Reading Data spectra ... -2023-05-08 12:04:41,869 : DEBUG : Reading CRC32 ... -2023-05-08 12:04:41,874 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 12:04:41,874 : DEBUG : Reading Data spectra ... -2023-05-08 12:04:41,875 : DEBUG : Reading CRC32 ... -2023-05-08 12:04:41,880 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 12:04:41,880 : DEBUG : Reading Data spectra ... -2023-05-08 12:04:41,881 : DEBUG : Reading CRC32 ... -2023-05-08 12:04:41,887 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 12:04:41,887 : DEBUG : Reading Data spectra ... -2023-05-08 12:04:41,887 : DEBUG : Reading CRC32 ... -2023-05-08 12:04:41,892 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 12:04:41,892 : DEBUG : Reading Data spectra ... -2023-05-08 12:04:41,895 : DEBUG : Reading CRC32 ... -2023-05-08 12:04:41,900 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 12:04:41,900 : DEBUG : Reading Data spectra ... -2023-05-08 12:04:41,901 : DEBUG : Reading CRC32 ... -2023-05-08 12:04:41,906 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 12:04:41,906 : DEBUG : Reading Data spectra ... -2023-05-08 12:04:41,907 : DEBUG : Reading CRC32 ... -2023-05-08 12:04:41,967 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 12:04:41,967 : DEBUG : {'Total Dataset Length': 4131, 'Spectrum Type Information': 128, 'acquisition_time': 1681137201860, 'integration_time': 32, 'temperature': 15.547460556030273, 'Pixel Count': 2048, 'acceleration_x_mean': -12608, 'acceleration_x_std': 0, 'acceleration_y_mean': -10176, 'acceleration_y_std': 0, 'acceleration_z_mean': -2752, 'acceleration_z_std': 0} -2023-05-08 12:04:41,971 : DEBUG : Wavelength range: 163.85150146484375 -1145.61306723725 -2023-05-08 12:04:42,013 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 12:04:42,014 : DEBUG : Reading Data spectra ... -2023-05-08 12:04:42,015 : DEBUG : Reading CRC32 ... -2023-05-08 12:04:42,021 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 12:04:42,021 : DEBUG : Reading Data spectra ... -2023-05-08 12:04:42,022 : DEBUG : Reading CRC32 ... -2023-05-08 12:04:42,030 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 12:04:42,030 : DEBUG : Reading Data spectra ... -2023-05-08 12:04:42,031 : DEBUG : Reading CRC32 ... -2023-05-08 12:04:42,037 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 12:04:42,038 : DEBUG : Reading Data spectra ... -2023-05-08 12:04:42,039 : DEBUG : Reading CRC32 ... -2023-05-08 12:04:42,046 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 12:04:42,046 : DEBUG : Reading Data spectra ... -2023-05-08 12:04:42,047 : DEBUG : Reading CRC32 ... -2023-05-08 12:04:42,052 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 12:04:42,052 : DEBUG : Reading Data spectra ... -2023-05-08 12:04:42,053 : DEBUG : Reading CRC32 ... -2023-05-08 12:04:42,059 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 12:04:42,059 : DEBUG : Reading Data spectra ... -2023-05-08 12:04:42,059 : DEBUG : Reading CRC32 ... -2023-05-08 12:04:42,066 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 12:04:42,066 : DEBUG : Reading Data spectra ... -2023-05-08 12:04:42,067 : DEBUG : Reading CRC32 ... -2023-05-08 12:04:42,072 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 12:04:42,072 : DEBUG : Reading Data spectra ... -2023-05-08 12:04:42,072 : DEBUG : Reading CRC32 ... -2023-05-08 12:04:42,078 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 12:04:42,078 : DEBUG : Reading Data spectra ... -2023-05-08 12:04:42,079 : DEBUG : Reading CRC32 ... -2023-05-08 12:04:42,084 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 12:04:42,084 : DEBUG : Reading Data spectra ... -2023-05-08 12:04:42,085 : DEBUG : Reading CRC32 ... -2023-05-08 12:04:42,089 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 12:04:42,090 : DEBUG : Reading Data spectra ... -2023-05-08 12:04:42,090 : DEBUG : Reading CRC32 ... -2023-05-08 12:04:42,096 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 12:04:42,096 : DEBUG : Reading Data spectra ... -2023-05-08 12:04:42,097 : DEBUG : Reading CRC32 ... -2023-05-08 12:04:42,102 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 12:04:42,102 : DEBUG : Reading Data spectra ... -2023-05-08 12:04:42,102 : DEBUG : Reading CRC32 ... -2023-05-08 12:04:42,107 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 12:04:42,108 : DEBUG : Reading Data spectra ... -2023-05-08 12:04:42,108 : DEBUG : Reading CRC32 ... -2023-05-08 12:04:42,170 : INFO : Done -2023-05-08 12:04:42,171 : INFO : Processing to L1a... -2023-05-08 12:04:42,405 : DEBUG : preprocessing done -2023-05-08 12:05:09,106 : DEBUG : preprocessing done -2023-05-08 12:05:30,843 : INFO : Done -2023-05-08 12:05:30,843 : INFO : Processing to L1b radiance... -2023-05-08 12:05:32,834 : INFO : Done -2023-05-08 12:05:32,834 : INFO : Processing to L1b irradiance... -2023-05-08 12:05:34,257 : INFO : Done -2023-05-08 12:05:34,257 : INFO : Processing to L1c... -2023-05-08 12:05:34,354 : DEBUG : WaterNetworkProtocol -2023-05-08 12:05:34,368 : DEBUG : 1681137226 -2023-05-08 12:05:34,370 : DEBUG : 1681137226 -2023-05-08 12:05:34,371 : DEBUG : 1681137226 -2023-05-08 12:05:34,372 : DEBUG : 1681137226 -2023-05-08 12:05:34,373 : DEBUG : 1681137226 -2023-05-08 12:05:34,374 : DEBUG : 1681137226 -2023-05-08 12:05:36,185 : INFO : interpolate sky radiance -2023-05-08 12:05:56,134 : INFO : interpolate irradiances -2023-05-08 12:09:32,820 : INFO : Processing Anomalies: ['x'] -2023-05-08 12:09:32,820 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/Test/SEQ20230410T143155Failed: UnboundLocalError("local variable 'wind' referenced before assignment") -2023-05-08 12:09:32,822 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 142, in process_sequence - L1c = surf.process_l1c(L1c_int, L1b_irr) - File "/home/cgoyens/hypernets_processor/hypernets_processor/surface_reflectance/surface_reflectance.py", line 51, in process_l1c - dataset_l1c = self.rh.get_wind(dataset_l1c) - File "/home/cgoyens/hypernets_processor/hypernets_processor/rhymer/rhymer/hypstar/rhymer_hypstar.py", line 300, in get_wind - anc_wind = ts_wind(isotime, l1b.attrs['site_id'], ancillary_source, met_dir) - File "/home/cgoyens/hypernets_processor/hypernets_processor/rhymer/rhymer/ancillary/rhymer_ancillary.py", line 48, in ts_wind - return (wind) -UnboundLocalError: local variable 'wind' referenced before assignment - -2023-05-08 12:09:32,823 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/Test/SEQ20230410T140203 -2023-05-08 12:09:32,931 : INFO : Reading raw data... -2023-05-08 12:09:32,933 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 12:09:32,933 : DEBUG : {'Total Dataset Length': 4131, 'Spectrum Type Information': 136, 'acquisition_time': 1681135485382, 'integration_time': 1024, 'temperature': 16.488752365112305, 'Pixel Count': 2048, 'acceleration_x_mean': -16303, 'acceleration_x_std': 131, 'acceleration_y_mean': -429, 'acceleration_y_std': 595, 'acceleration_z_mean': -29, 'acceleration_z_std': 643} -2023-05-08 12:09:32,935 : DEBUG : Wavelength range: 163.96437072753906 -1145.4518535429638 -2023-05-08 12:09:32,960 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 12:09:32,960 : DEBUG : Reading Data spectra ... -2023-05-08 12:09:32,961 : DEBUG : Reading CRC32 ... -2023-05-08 12:09:32,966 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 12:09:32,966 : DEBUG : Reading Data spectra ... -2023-05-08 12:09:32,967 : DEBUG : Reading CRC32 ... -2023-05-08 12:09:32,972 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 12:09:32,972 : DEBUG : Reading Data spectra ... -2023-05-08 12:09:32,973 : DEBUG : Reading CRC32 ... -2023-05-08 12:09:32,978 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 12:09:32,978 : DEBUG : Reading Data spectra ... -2023-05-08 12:09:32,979 : DEBUG : Reading CRC32 ... -2023-05-08 12:09:32,984 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 12:09:32,984 : DEBUG : Reading Data spectra ... -2023-05-08 12:09:32,985 : DEBUG : Reading CRC32 ... -2023-05-08 12:09:32,990 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 12:09:32,990 : DEBUG : Reading Data spectra ... -2023-05-08 12:09:32,991 : DEBUG : Reading CRC32 ... -2023-05-08 12:09:33,051 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 12:09:33,051 : DEBUG : {'Total Dataset Length': 4131, 'Spectrum Type Information': 144, 'acquisition_time': 1681135438327, 'integration_time': 256, 'temperature': 16.039813995361328, 'Pixel Count': 2048, 'acceleration_x_mean': 12473, 'acceleration_x_std': 262, 'acceleration_y_mean': -10256, 'acceleration_y_std': 292, 'acceleration_z_mean': -2960, 'acceleration_z_std': 469} -2023-05-08 12:09:33,053 : DEBUG : Wavelength range: 163.85150146484375 -1145.61306723725 -2023-05-08 12:09:33,100 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 12:09:33,100 : DEBUG : Reading Data spectra ... -2023-05-08 12:09:33,102 : DEBUG : Reading CRC32 ... -2023-05-08 12:09:33,116 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 12:09:33,116 : DEBUG : Reading Data spectra ... -2023-05-08 12:09:33,119 : DEBUG : Reading CRC32 ... -2023-05-08 12:09:33,133 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 12:09:33,133 : DEBUG : Reading Data spectra ... -2023-05-08 12:09:33,136 : DEBUG : Reading CRC32 ... -2023-05-08 12:09:33,150 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 12:09:33,150 : DEBUG : Reading Data spectra ... -2023-05-08 12:09:33,154 : DEBUG : Reading CRC32 ... -2023-05-08 12:09:33,162 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 12:09:33,162 : DEBUG : Reading Data spectra ... -2023-05-08 12:09:33,163 : DEBUG : Reading CRC32 ... -2023-05-08 12:09:33,172 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 12:09:33,172 : DEBUG : Reading Data spectra ... -2023-05-08 12:09:33,173 : DEBUG : Reading CRC32 ... -2023-05-08 12:09:33,178 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 12:09:33,178 : DEBUG : Reading Data spectra ... -2023-05-08 12:09:33,180 : DEBUG : Reading CRC32 ... -2023-05-08 12:09:33,185 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 12:09:33,185 : DEBUG : Reading Data spectra ... -2023-05-08 12:09:33,186 : DEBUG : Reading CRC32 ... -2023-05-08 12:09:33,191 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 12:09:33,191 : DEBUG : Reading Data spectra ... -2023-05-08 12:09:33,192 : DEBUG : Reading CRC32 ... -2023-05-08 12:09:33,197 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 12:09:33,197 : DEBUG : Reading Data spectra ... -2023-05-08 12:09:33,198 : DEBUG : Reading CRC32 ... -2023-05-08 12:09:33,204 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 12:09:33,204 : DEBUG : Reading Data spectra ... -2023-05-08 12:09:33,205 : DEBUG : Reading CRC32 ... -2023-05-08 12:09:33,210 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 12:09:33,210 : DEBUG : Reading Data spectra ... -2023-05-08 12:09:33,211 : DEBUG : Reading CRC32 ... -2023-05-08 12:09:33,272 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 12:09:33,272 : DEBUG : {'Total Dataset Length': 4131, 'Spectrum Type Information': 128, 'acquisition_time': 1681135412063, 'integration_time': 32, 'temperature': 15.654559135437012, 'Pixel Count': 2048, 'acceleration_x_mean': -12992, 'acceleration_x_std': 0, 'acceleration_y_mean': -9856, 'acceleration_y_std': 0, 'acceleration_z_mean': -2048, 'acceleration_z_std': 0} -2023-05-08 12:09:33,274 : DEBUG : Wavelength range: 163.85150146484375 -1145.61306723725 -2023-05-08 12:09:33,302 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 12:09:33,303 : DEBUG : Reading Data spectra ... -2023-05-08 12:09:33,303 : DEBUG : Reading CRC32 ... -2023-05-08 12:09:33,311 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 12:09:33,311 : DEBUG : Reading Data spectra ... -2023-05-08 12:09:33,312 : DEBUG : Reading CRC32 ... -2023-05-08 12:09:33,317 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 12:09:33,317 : DEBUG : Reading Data spectra ... -2023-05-08 12:09:33,318 : DEBUG : Reading CRC32 ... -2023-05-08 12:09:33,322 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 12:09:33,322 : DEBUG : Reading Data spectra ... -2023-05-08 12:09:33,323 : DEBUG : Reading CRC32 ... -2023-05-08 12:09:33,328 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 12:09:33,328 : DEBUG : Reading Data spectra ... -2023-05-08 12:09:33,329 : DEBUG : Reading CRC32 ... -2023-05-08 12:09:33,334 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 12:09:33,334 : DEBUG : Reading Data spectra ... -2023-05-08 12:09:33,335 : DEBUG : Reading CRC32 ... -2023-05-08 12:09:33,340 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 12:09:33,340 : DEBUG : Reading Data spectra ... -2023-05-08 12:09:33,341 : DEBUG : Reading CRC32 ... -2023-05-08 12:09:33,346 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 12:09:33,346 : DEBUG : Reading Data spectra ... -2023-05-08 12:09:33,347 : DEBUG : Reading CRC32 ... -2023-05-08 12:09:33,352 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 12:09:33,352 : DEBUG : Reading Data spectra ... -2023-05-08 12:09:33,352 : DEBUG : Reading CRC32 ... -2023-05-08 12:09:33,358 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 12:09:33,358 : DEBUG : Reading Data spectra ... -2023-05-08 12:09:33,361 : DEBUG : Reading CRC32 ... -2023-05-08 12:09:33,375 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 12:09:33,375 : DEBUG : Reading Data spectra ... -2023-05-08 12:09:33,376 : DEBUG : Reading CRC32 ... -2023-05-08 12:09:33,382 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 12:09:33,382 : DEBUG : Reading Data spectra ... -2023-05-08 12:09:33,383 : DEBUG : Reading CRC32 ... -2023-05-08 12:09:33,388 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 12:09:33,388 : DEBUG : Reading Data spectra ... -2023-05-08 12:09:33,389 : DEBUG : Reading CRC32 ... -2023-05-08 12:09:33,394 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 12:09:33,394 : DEBUG : Reading Data spectra ... -2023-05-08 12:09:33,395 : DEBUG : Reading CRC32 ... -2023-05-08 12:09:33,400 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 12:09:33,400 : DEBUG : Reading Data spectra ... -2023-05-08 12:09:33,401 : DEBUG : Reading CRC32 ... -2023-05-08 12:09:33,459 : INFO : Done -2023-05-08 12:09:33,459 : INFO : Processing to L1a... -2023-05-08 12:09:33,694 : DEBUG : preprocessing done -2023-05-08 12:09:57,048 : DEBUG : preprocessing done -2023-05-08 12:10:18,931 : INFO : Done -2023-05-08 12:10:18,931 : INFO : Processing to L1b radiance... -2023-05-08 12:10:20,913 : INFO : Done -2023-05-08 12:10:20,913 : INFO : Processing to L1b irradiance... -2023-05-08 12:10:22,311 : INFO : Done -2023-05-08 12:10:22,311 : INFO : Processing to L1c... -2023-05-08 12:10:22,407 : DEBUG : WaterNetworkProtocol -2023-05-08 12:10:22,419 : DEBUG : 1681135437 -2023-05-08 12:10:22,420 : DEBUG : 1681135437 -2023-05-08 12:10:22,421 : DEBUG : 1681135437 -2023-05-08 12:10:22,422 : DEBUG : 1681135437 -2023-05-08 12:10:22,423 : DEBUG : 1681135437 -2023-05-08 12:10:22,424 : DEBUG : 1681135437 -2023-05-08 12:10:24,232 : INFO : interpolate sky radiance -2023-05-08 12:10:45,113 : INFO : interpolate irradiances -2023-05-08 12:13:56,466 : INFO : Processing Anomalies: ['x', 'x'] -2023-05-08 12:13:56,466 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/Test/SEQ20230410T140203Failed: UnboundLocalError("local variable 'wind' referenced before assignment") -2023-05-08 12:13:56,466 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 142, in process_sequence - L1c = surf.process_l1c(L1c_int, L1b_irr) - File "/home/cgoyens/hypernets_processor/hypernets_processor/surface_reflectance/surface_reflectance.py", line 51, in process_l1c - dataset_l1c = self.rh.get_wind(dataset_l1c) - File "/home/cgoyens/hypernets_processor/hypernets_processor/rhymer/rhymer/hypstar/rhymer_hypstar.py", line 300, in get_wind - anc_wind = ts_wind(isotime, l1b.attrs['site_id'], ancillary_source, met_dir) - File "/home/cgoyens/hypernets_processor/hypernets_processor/rhymer/rhymer/ancillary/rhymer_ancillary.py", line 48, in ts_wind - return (wind) -UnboundLocalError: local variable 'wind' referenced before assignment - -2023-05-08 12:53:33,538 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/Test/SEQ20230410T143155 -2023-05-08 12:53:33,926 : INFO : Reading raw data... -2023-05-08 12:53:33,941 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 12:53:33,941 : DEBUG : {'Total Dataset Length': 4131, 'Spectrum Type Information': 136, 'acquisition_time': 1681137276013, 'integration_time': 1024, 'temperature': 16.44601821899414, 'Pixel Count': 2048, 'acceleration_x_mean': -16327, 'acceleration_x_std': 91, 'acceleration_y_mean': -396, 'acceleration_y_std': 243, 'acceleration_z_mean': 116, 'acceleration_z_std': 453} -2023-05-08 12:53:33,948 : DEBUG : Wavelength range: 163.96437072753906 -1145.4518535429638 -2023-05-08 12:53:34,083 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 12:53:34,083 : DEBUG : Reading Data spectra ... -2023-05-08 12:53:34,085 : DEBUG : Reading CRC32 ... -2023-05-08 12:53:34,104 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 12:53:34,104 : DEBUG : Reading Data spectra ... -2023-05-08 12:53:34,105 : DEBUG : Reading CRC32 ... -2023-05-08 12:53:34,121 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 12:53:34,121 : DEBUG : Reading Data spectra ... -2023-05-08 12:53:34,123 : DEBUG : Reading CRC32 ... -2023-05-08 12:53:34,139 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 12:53:34,139 : DEBUG : Reading Data spectra ... -2023-05-08 12:53:34,141 : DEBUG : Reading CRC32 ... -2023-05-08 12:53:34,158 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 12:53:34,158 : DEBUG : Reading Data spectra ... -2023-05-08 12:53:34,159 : DEBUG : Reading CRC32 ... -2023-05-08 12:53:34,174 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 12:53:34,175 : DEBUG : Reading Data spectra ... -2023-05-08 12:53:34,176 : DEBUG : Reading CRC32 ... -2023-05-08 12:53:34,330 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 12:53:34,330 : DEBUG : {'Total Dataset Length': 4131, 'Spectrum Type Information': 144, 'acquisition_time': 1681137227706, 'integration_time': 256, 'temperature': 15.911444664001465, 'Pixel Count': 2048, 'acceleration_x_mean': 12620, 'acceleration_x_std': 638, 'acceleration_y_mean': -10200, 'acceleration_y_std': 784, 'acceleration_z_mean': -2932, 'acceleration_z_std': 499} -2023-05-08 12:53:34,335 : DEBUG : Wavelength range: 163.85150146484375 -1145.61306723725 -2023-05-08 12:53:34,454 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 12:53:34,454 : DEBUG : Reading Data spectra ... -2023-05-08 12:53:34,456 : DEBUG : Reading CRC32 ... -2023-05-08 12:53:34,472 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 12:53:34,472 : DEBUG : Reading Data spectra ... -2023-05-08 12:53:34,474 : DEBUG : Reading CRC32 ... -2023-05-08 12:53:34,490 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 12:53:34,491 : DEBUG : Reading Data spectra ... -2023-05-08 12:53:34,492 : DEBUG : Reading CRC32 ... -2023-05-08 12:53:34,509 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 12:53:34,510 : DEBUG : Reading Data spectra ... -2023-05-08 12:53:34,511 : DEBUG : Reading CRC32 ... -2023-05-08 12:53:34,528 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 12:53:34,528 : DEBUG : Reading Data spectra ... -2023-05-08 12:53:34,529 : DEBUG : Reading CRC32 ... -2023-05-08 12:53:34,545 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 12:53:34,545 : DEBUG : Reading Data spectra ... -2023-05-08 12:53:34,547 : DEBUG : Reading CRC32 ... -2023-05-08 12:53:34,563 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 12:53:34,563 : DEBUG : Reading Data spectra ... -2023-05-08 12:53:34,564 : DEBUG : Reading CRC32 ... -2023-05-08 12:53:34,582 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 12:53:34,583 : DEBUG : Reading Data spectra ... -2023-05-08 12:53:34,584 : DEBUG : Reading CRC32 ... -2023-05-08 12:53:34,601 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 12:53:34,601 : DEBUG : Reading Data spectra ... -2023-05-08 12:53:34,603 : DEBUG : Reading CRC32 ... -2023-05-08 12:53:34,621 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 12:53:34,621 : DEBUG : Reading Data spectra ... -2023-05-08 12:53:34,626 : DEBUG : Reading CRC32 ... -2023-05-08 12:53:34,647 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 12:53:34,647 : DEBUG : Reading Data spectra ... -2023-05-08 12:53:34,648 : DEBUG : Reading CRC32 ... -2023-05-08 12:53:34,666 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 12:53:34,667 : DEBUG : Reading Data spectra ... -2023-05-08 12:53:34,668 : DEBUG : Reading CRC32 ... -2023-05-08 12:53:34,812 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 12:53:34,812 : DEBUG : {'Total Dataset Length': 4131, 'Spectrum Type Information': 128, 'acquisition_time': 1681137201860, 'integration_time': 32, 'temperature': 15.547460556030273, 'Pixel Count': 2048, 'acceleration_x_mean': -12608, 'acceleration_x_std': 0, 'acceleration_y_mean': -10176, 'acceleration_y_std': 0, 'acceleration_z_mean': -2752, 'acceleration_z_std': 0} -2023-05-08 12:53:34,816 : DEBUG : Wavelength range: 163.85150146484375 -1145.61306723725 -2023-05-08 12:53:34,954 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 12:53:34,954 : DEBUG : Reading Data spectra ... -2023-05-08 12:53:34,956 : DEBUG : Reading CRC32 ... -2023-05-08 12:53:34,976 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 12:53:34,977 : DEBUG : Reading Data spectra ... -2023-05-08 12:53:34,978 : DEBUG : Reading CRC32 ... -2023-05-08 12:53:35,003 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 12:53:35,003 : DEBUG : Reading Data spectra ... -2023-05-08 12:53:35,005 : DEBUG : Reading CRC32 ... -2023-05-08 12:53:35,029 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 12:53:35,029 : DEBUG : Reading Data spectra ... -2023-05-08 12:53:35,032 : DEBUG : Reading CRC32 ... -2023-05-08 12:53:35,052 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 12:53:35,052 : DEBUG : Reading Data spectra ... -2023-05-08 12:53:35,054 : DEBUG : Reading CRC32 ... -2023-05-08 12:53:35,078 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 12:53:35,079 : DEBUG : Reading Data spectra ... -2023-05-08 12:53:35,082 : DEBUG : Reading CRC32 ... -2023-05-08 12:53:35,106 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 12:53:35,107 : DEBUG : Reading Data spectra ... -2023-05-08 12:53:35,110 : DEBUG : Reading CRC32 ... -2023-05-08 12:53:35,134 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 12:53:35,135 : DEBUG : Reading Data spectra ... -2023-05-08 12:53:35,137 : DEBUG : Reading CRC32 ... -2023-05-08 12:53:35,157 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 12:53:35,157 : DEBUG : Reading Data spectra ... -2023-05-08 12:53:35,158 : DEBUG : Reading CRC32 ... -2023-05-08 12:53:35,183 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 12:53:35,184 : DEBUG : Reading Data spectra ... -2023-05-08 12:53:35,186 : DEBUG : Reading CRC32 ... -2023-05-08 12:53:35,212 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 12:53:35,212 : DEBUG : Reading Data spectra ... -2023-05-08 12:53:35,215 : DEBUG : Reading CRC32 ... -2023-05-08 12:53:35,242 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 12:53:35,242 : DEBUG : Reading Data spectra ... -2023-05-08 12:53:35,244 : DEBUG : Reading CRC32 ... -2023-05-08 12:53:35,269 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 12:53:35,269 : DEBUG : Reading Data spectra ... -2023-05-08 12:53:35,271 : DEBUG : Reading CRC32 ... -2023-05-08 12:53:35,292 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 12:53:35,293 : DEBUG : Reading Data spectra ... -2023-05-08 12:53:35,294 : DEBUG : Reading CRC32 ... -2023-05-08 12:53:35,317 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 12:53:35,318 : DEBUG : Reading Data spectra ... -2023-05-08 12:53:35,319 : DEBUG : Reading CRC32 ... -2023-05-08 12:53:35,464 : INFO : Done -2023-05-08 12:53:35,465 : INFO : Processing to L1a... -2023-05-08 12:53:36,558 : DEBUG : preprocessing done -2023-05-08 12:54:23,664 : DEBUG : preprocessing done -2023-05-08 12:55:13,834 : INFO : Done -2023-05-08 12:55:13,834 : INFO : Processing to L1b radiance... -2023-05-08 12:55:22,894 : INFO : Done -2023-05-08 12:55:22,894 : INFO : Processing to L1b irradiance... -2023-05-08 12:55:28,049 : INFO : Done -2023-05-08 12:55:28,049 : INFO : Processing to L1c... -2023-05-08 12:55:28,432 : DEBUG : WaterNetworkProtocol -2023-05-08 12:55:28,519 : DEBUG : 1681137226 -2023-05-08 12:55:28,524 : DEBUG : 1681137226 -2023-05-08 12:55:28,529 : DEBUG : 1681137226 -2023-05-08 12:55:28,537 : DEBUG : 1681137226 -2023-05-08 12:55:28,543 : DEBUG : 1681137226 -2023-05-08 12:55:28,548 : DEBUG : 1681137226 -2023-05-08 12:55:38,946 : INFO : interpolate sky radiance -2023-05-08 12:56:03,771 : INFO : interpolate irradiances -2023-05-08 13:52:50,587 : INFO : Processing Anomalies: ['x'] -2023-05-08 13:52:50,589 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/Test/SEQ20230410T143155Failed: UnboundLocalError("local variable 'wind' referenced before assignment") -2023-05-08 13:52:50,600 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 142, in process_sequence - L1c = surf.process_l1c(L1c_int, L1b_irr) - File "/home/cgoyens/hypernets_processor/hypernets_processor/surface_reflectance/surface_reflectance.py", line 51, in process_l1c - dataset_l1c = self.rh.get_wind(dataset_l1c) - File "/home/cgoyens/hypernets_processor/hypernets_processor/rhymer/rhymer/hypstar/rhymer_hypstar.py", line 300, in get_wind - anc_wind = ts_wind(isotime, l1b.attrs['site_id'], ancillary_source, met_dir) - File "/home/cgoyens/hypernets_processor/hypernets_processor/rhymer/rhymer/ancillary/rhymer_ancillary.py", line 48, in ts_wind -UnboundLocalError: local variable 'wind' referenced before assignment - -2023-05-08 13:52:50,601 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/Test/SEQ20230410T140203 -2023-05-08 13:52:50,947 : INFO : Reading raw data... -2023-05-08 13:52:50,953 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 13:52:50,954 : DEBUG : {'Total Dataset Length': 4131, 'Spectrum Type Information': 136, 'acquisition_time': 1681135485382, 'integration_time': 1024, 'temperature': 16.488752365112305, 'Pixel Count': 2048, 'acceleration_x_mean': -16303, 'acceleration_x_std': 131, 'acceleration_y_mean': -429, 'acceleration_y_std': 595, 'acceleration_z_mean': -29, 'acceleration_z_std': 643} -2023-05-08 13:52:50,959 : DEBUG : Wavelength range: 163.96437072753906 -1145.4518535429638 -2023-05-08 13:52:51,073 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 13:52:51,073 : DEBUG : Reading Data spectra ... -2023-05-08 13:52:51,075 : DEBUG : Reading CRC32 ... -2023-05-08 13:52:51,098 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 13:52:51,099 : DEBUG : Reading Data spectra ... -2023-05-08 13:52:51,101 : DEBUG : Reading CRC32 ... -2023-05-08 13:52:51,121 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 13:52:51,121 : DEBUG : Reading Data spectra ... -2023-05-08 13:52:51,122 : DEBUG : Reading CRC32 ... -2023-05-08 13:52:51,139 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 13:52:51,139 : DEBUG : Reading Data spectra ... -2023-05-08 13:52:51,141 : DEBUG : Reading CRC32 ... -2023-05-08 13:52:51,166 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 13:52:51,166 : DEBUG : Reading Data spectra ... -2023-05-08 13:52:51,170 : DEBUG : Reading CRC32 ... -2023-05-08 13:52:51,188 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 13:52:51,189 : DEBUG : Reading Data spectra ... -2023-05-08 13:52:51,190 : DEBUG : Reading CRC32 ... -2023-05-08 13:52:51,343 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 13:52:51,343 : DEBUG : {'Total Dataset Length': 4131, 'Spectrum Type Information': 144, 'acquisition_time': 1681135438327, 'integration_time': 256, 'temperature': 16.039813995361328, 'Pixel Count': 2048, 'acceleration_x_mean': 12473, 'acceleration_x_std': 262, 'acceleration_y_mean': -10256, 'acceleration_y_std': 292, 'acceleration_z_mean': -2960, 'acceleration_z_std': 469} -2023-05-08 13:52:51,348 : DEBUG : Wavelength range: 163.85150146484375 -1145.61306723725 -2023-05-08 13:52:51,460 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 13:52:51,461 : DEBUG : Reading Data spectra ... -2023-05-08 13:52:51,465 : DEBUG : Reading CRC32 ... -2023-05-08 13:52:51,487 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 13:52:51,487 : DEBUG : Reading Data spectra ... -2023-05-08 13:52:51,488 : DEBUG : Reading CRC32 ... -2023-05-08 13:52:51,504 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 13:52:51,505 : DEBUG : Reading Data spectra ... -2023-05-08 13:52:51,506 : DEBUG : Reading CRC32 ... -2023-05-08 13:52:51,522 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 13:52:51,522 : DEBUG : Reading Data spectra ... -2023-05-08 13:52:51,523 : DEBUG : Reading CRC32 ... -2023-05-08 13:52:51,538 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 13:52:51,539 : DEBUG : Reading Data spectra ... -2023-05-08 13:52:51,540 : DEBUG : Reading CRC32 ... -2023-05-08 13:52:51,555 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 13:52:51,555 : DEBUG : Reading Data spectra ... -2023-05-08 13:52:51,556 : DEBUG : Reading CRC32 ... -2023-05-08 13:52:51,571 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 13:52:51,571 : DEBUG : Reading Data spectra ... -2023-05-08 13:52:51,572 : DEBUG : Reading CRC32 ... -2023-05-08 13:52:51,587 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 13:52:51,587 : DEBUG : Reading Data spectra ... -2023-05-08 13:52:51,589 : DEBUG : Reading CRC32 ... -2023-05-08 13:52:51,607 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 13:52:51,608 : DEBUG : Reading Data spectra ... -2023-05-08 13:52:51,609 : DEBUG : Reading CRC32 ... -2023-05-08 13:52:51,625 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 13:52:51,626 : DEBUG : Reading Data spectra ... -2023-05-08 13:52:51,630 : DEBUG : Reading CRC32 ... -2023-05-08 13:52:51,654 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 13:52:51,654 : DEBUG : Reading Data spectra ... -2023-05-08 13:52:51,655 : DEBUG : Reading CRC32 ... -2023-05-08 13:52:51,670 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 13:52:51,670 : DEBUG : Reading Data spectra ... -2023-05-08 13:52:51,672 : DEBUG : Reading CRC32 ... -2023-05-08 13:52:51,809 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 13:52:51,809 : DEBUG : {'Total Dataset Length': 4131, 'Spectrum Type Information': 128, 'acquisition_time': 1681135412063, 'integration_time': 32, 'temperature': 15.654559135437012, 'Pixel Count': 2048, 'acceleration_x_mean': -12992, 'acceleration_x_std': 0, 'acceleration_y_mean': -9856, 'acceleration_y_std': 0, 'acceleration_z_mean': -2048, 'acceleration_z_std': 0} -2023-05-08 13:52:51,813 : DEBUG : Wavelength range: 163.85150146484375 -1145.61306723725 -2023-05-08 13:52:51,926 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 13:52:51,927 : DEBUG : Reading Data spectra ... -2023-05-08 13:52:51,929 : DEBUG : Reading CRC32 ... -2023-05-08 13:52:51,943 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 13:52:51,943 : DEBUG : Reading Data spectra ... -2023-05-08 13:52:51,945 : DEBUG : Reading CRC32 ... -2023-05-08 13:52:51,960 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 13:52:51,960 : DEBUG : Reading Data spectra ... -2023-05-08 13:52:51,961 : DEBUG : Reading CRC32 ... -2023-05-08 13:52:51,977 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 13:52:51,977 : DEBUG : Reading Data spectra ... -2023-05-08 13:52:51,978 : DEBUG : Reading CRC32 ... -2023-05-08 13:52:51,994 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 13:52:51,994 : DEBUG : Reading Data spectra ... -2023-05-08 13:52:51,995 : DEBUG : Reading CRC32 ... -2023-05-08 13:52:52,012 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 13:52:52,012 : DEBUG : Reading Data spectra ... -2023-05-08 13:52:52,014 : DEBUG : Reading CRC32 ... -2023-05-08 13:52:52,031 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 13:52:52,031 : DEBUG : Reading Data spectra ... -2023-05-08 13:52:52,032 : DEBUG : Reading CRC32 ... -2023-05-08 13:52:52,048 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 13:52:52,048 : DEBUG : Reading Data spectra ... -2023-05-08 13:52:52,050 : DEBUG : Reading CRC32 ... -2023-05-08 13:52:52,066 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 13:52:52,066 : DEBUG : Reading Data spectra ... -2023-05-08 13:52:52,067 : DEBUG : Reading CRC32 ... -2023-05-08 13:52:52,084 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 13:52:52,084 : DEBUG : Reading Data spectra ... -2023-05-08 13:52:52,086 : DEBUG : Reading CRC32 ... -2023-05-08 13:52:52,110 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 13:52:52,110 : DEBUG : Reading Data spectra ... -2023-05-08 13:52:52,111 : DEBUG : Reading CRC32 ... -2023-05-08 13:52:52,130 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 13:52:52,130 : DEBUG : Reading Data spectra ... -2023-05-08 13:52:52,131 : DEBUG : Reading CRC32 ... -2023-05-08 13:52:52,152 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 13:52:52,152 : DEBUG : Reading Data spectra ... -2023-05-08 13:52:52,154 : DEBUG : Reading CRC32 ... -2023-05-08 13:52:52,176 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 13:52:52,176 : DEBUG : Reading Data spectra ... -2023-05-08 13:52:52,177 : DEBUG : Reading CRC32 ... -2023-05-08 13:52:52,195 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 13:52:52,195 : DEBUG : Reading Data spectra ... -2023-05-08 13:52:52,197 : DEBUG : Reading CRC32 ... -2023-05-08 13:52:52,328 : INFO : Done -2023-05-08 13:52:52,328 : INFO : Processing to L1a... -2023-05-08 13:52:53,262 : DEBUG : preprocessing done -2023-05-08 13:53:09,741 : INFO : Processing Anomalies: ['x', 'x'] -2023-05-08 13:53:09,742 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/Test/SEQ20230410T140203Failed: ValueError() -2023-05-08 13:53:09,751 : INFO : Traceback (most recent call last): - File "/usr/local/lib/python3.8/dist-packages/comet_maths/linear_algebra/matrix_calculation.py", line 210, in nearestPD_cholesky - chol = np.linalg.cholesky(A3) - File "<__array_function__ internals>", line 5, in cholesky - File "/usr/local/lib/python3.8/dist-packages/numpy/linalg/linalg.py", line 764, in cholesky - r = gufunc(a, signature=signature, extobj=extobj) - File "/usr/local/lib/python3.8/dist-packages/numpy/linalg/linalg.py", line 91, in _raise_linalgerror_nonposdef - raise LinAlgError("Matrix is not positive definite") -numpy.linalg.LinAlgError: Matrix is not positive definite - -During handling of the above exception, another exception occurred: - -Traceback (most recent call last): - File "/usr/local/lib/python3.8/dist-packages/comet_maths/linear_algebra/matrix_calculation.py", line 224, in nearestPD_cholesky - mineig = np.min(np.real(np.linalg.eigvals(A3))) - File "<__array_function__ internals>", line 5, in eigvals - File "/usr/local/lib/python3.8/dist-packages/numpy/linalg/linalg.py", line 1069, in eigvals - w = _umath_linalg.eigvals(a, signature=signature, extobj=extobj) -KeyboardInterrupt - -During handling of the above exception, another exception occurred: - -Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 95, in process_sequence - L1a_rad, l0_rad_masked, l0_rad_bla_masked = cal.calibrate_l1a( - File "/home/cgoyens/hypernets_processor/hypernets_processor/calibration/calibrate.py", line 83, in calibrate_l1a - dataset_l1a = calibrate_function.propagate_ds_specific( - File "/usr/local/lib/python3.8/dist-packages/punpy/digital_effects_table/measurement_function.py", line 627, in propagate_ds_specific - u_comp_y, corr_comp_y = self.propagate_specific( - File "/usr/local/lib/python3.8/dist-packages/punpy/digital_effects_table/measurement_function.py", line 1263, in propagate_specific - return self.prop.propagate_standard( - File "/usr/local/lib/python3.8/dist-packages/punpy/mc/mc_propagation.py", line 515, in propagate_standard - outs = self.finish_repeated_outs( - File "/usr/local/lib/python3.8/dist-packages/punpy/mc/mc_propagation.py", line 1369, in finish_repeated_outs - outs[1] = cm.nearestPD_cholesky( - File "/usr/local/lib/python3.8/dist-packages/comet_maths/linear_algebra/matrix_calculation.py", line 229, in nearestPD_cholesky - raise ValueError -ValueError - -2023-05-08 13:55:14,674 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/Test/SEQ20230410T143155 -2023-05-08 13:55:34,271 : DEBUG : WaterNetworkProtocol -2023-05-08 13:55:34,297 : DEBUG : 1681137226 -2023-05-08 13:55:34,298 : DEBUG : 1681137226 -2023-05-08 13:55:34,300 : DEBUG : 1681137226 -2023-05-08 13:55:34,302 : DEBUG : 1681137226 -2023-05-08 13:55:34,304 : DEBUG : 1681137226 -2023-05-08 13:55:34,307 : DEBUG : 1681137226 -2023-05-08 13:55:37,043 : INFO : interpolate sky radiance -2023-05-08 13:56:20,857 : DEBUG : WaterNetworkProtocol -2023-05-08 13:56:20,880 : DEBUG : 1681137226 -2023-05-08 13:56:20,883 : DEBUG : 1681137226 -2023-05-08 13:56:20,886 : DEBUG : 1681137226 -2023-05-08 13:56:20,888 : DEBUG : 1681137226 -2023-05-08 13:56:20,890 : DEBUG : 1681137226 -2023-05-08 13:56:20,892 : DEBUG : 1681137226 -2023-05-08 13:56:23,526 : INFO : interpolate sky radiance -2023-05-08 13:57:31,474 : DEBUG : WaterNetworkProtocol -2023-05-08 13:57:31,495 : DEBUG : 1681137226 -2023-05-08 13:57:31,497 : DEBUG : 1681137226 -2023-05-08 13:57:31,500 : DEBUG : 1681137226 -2023-05-08 13:57:31,501 : DEBUG : 1681137226 -2023-05-08 13:57:31,503 : DEBUG : 1681137226 -2023-05-08 13:57:31,505 : DEBUG : 1681137226 -2023-05-08 13:57:33,973 : INFO : interpolate sky radiance -2023-05-08 13:58:02,179 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/Test/SEQ20230410T143155 -2023-05-08 13:58:12,908 : DEBUG : WaterNetworkProtocol -2023-05-08 13:58:12,934 : DEBUG : 1681137226 -2023-05-08 13:58:12,937 : DEBUG : 1681137226 -2023-05-08 13:58:12,939 : DEBUG : 1681137226 -2023-05-08 13:58:12,940 : DEBUG : 1681137226 -2023-05-08 13:58:12,942 : DEBUG : 1681137226 -2023-05-08 13:58:12,943 : DEBUG : 1681137226 -2023-05-08 13:58:15,567 : INFO : interpolate sky radiance -2023-05-08 13:58:15,567 : INFO : interpolate irradiances -2023-05-08 13:59:02,078 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/Test/SEQ20230410T143155 -2023-05-08 13:59:26,312 : DEBUG : WaterNetworkProtocol -2023-05-08 13:59:26,349 : DEBUG : 1681137226 -2023-05-08 13:59:26,353 : DEBUG : 1681137226 -2023-05-08 13:59:26,356 : DEBUG : 1681137226 -2023-05-08 13:59:26,358 : DEBUG : 1681137226 -2023-05-08 13:59:26,361 : DEBUG : 1681137226 -2023-05-08 13:59:26,363 : DEBUG : 1681137226 -2023-05-08 13:59:29,460 : INFO : interpolate sky radiance -2023-05-08 13:59:29,461 : INFO : interpolate irradiances -2023-05-08 14:00:12,099 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/Test/SEQ20230410T143155 -2023-05-08 14:00:36,450 : DEBUG : WaterNetworkProtocol -2023-05-08 14:00:36,472 : DEBUG : 1681137226 -2023-05-08 14:00:36,475 : DEBUG : 1681137226 -2023-05-08 14:00:36,477 : DEBUG : 1681137226 -2023-05-08 14:00:36,479 : DEBUG : 1681137226 -2023-05-08 14:00:36,481 : DEBUG : 1681137226 -2023-05-08 14:00:36,482 : DEBUG : 1681137226 -2023-05-08 14:00:39,274 : INFO : interpolate sky radiance -2023-05-08 14:00:39,275 : INFO : interpolate irradiances -2023-05-08 14:01:02,149 : DEBUG : WaterNetworkProtocol -2023-05-08 14:01:02,171 : DEBUG : 1681137226 -2023-05-08 14:01:02,172 : DEBUG : 1681137226 -2023-05-08 14:01:02,174 : DEBUG : 1681137226 -2023-05-08 14:01:02,176 : DEBUG : 1681137226 -2023-05-08 14:01:02,178 : DEBUG : 1681137226 -2023-05-08 14:01:02,179 : DEBUG : 1681137226 -2023-05-08 14:01:04,678 : INFO : interpolate sky radiance -2023-05-08 14:01:04,679 : INFO : interpolate irradiances -2023-05-08 14:08:02,385 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/Test/SEQ20230410T143155 -2023-05-08 14:08:17,513 : DEBUG : WaterNetworkProtocol -2023-05-08 14:08:17,537 : DEBUG : 1681137226 -2023-05-08 14:08:17,540 : DEBUG : 1681137226 -2023-05-08 14:08:17,542 : DEBUG : 1681137226 -2023-05-08 14:08:17,544 : DEBUG : 1681137226 -2023-05-08 14:08:17,546 : DEBUG : 1681137226 -2023-05-08 14:08:17,547 : DEBUG : 1681137226 -2023-05-08 14:08:20,343 : INFO : interpolate sky radiance -2023-05-08 14:08:20,344 : INFO : interpolate irradiances -2023-05-08 14:08:26,061 : INFO : Apply Mobley 1999 -2023-05-08 14:09:20,114 : INFO : Processing to L1c... -2023-05-08 14:09:20,757 : DEBUG : WaterNetworkProtocol -2023-05-08 14:09:20,871 : DEBUG : 1681137226 -2023-05-08 14:09:20,878 : DEBUG : 1681137226 -2023-05-08 14:09:20,885 : DEBUG : 1681137226 -2023-05-08 14:09:20,894 : DEBUG : 1681137226 -2023-05-08 14:09:20,903 : DEBUG : 1681137226 -2023-05-08 14:09:20,911 : DEBUG : 1681137226 -2023-05-08 14:09:30,136 : INFO : interpolate sky radiance -2023-05-08 14:09:30,136 : INFO : interpolate irradiances -2023-05-08 14:09:46,899 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/Test/SEQ20230410T143155 -2023-05-08 14:09:47,352 : INFO : Processing to L1c... -2023-05-08 14:09:47,911 : DEBUG : WaterNetworkProtocol -2023-05-08 14:09:48,018 : DEBUG : 1681137226 -2023-05-08 14:09:48,025 : DEBUG : 1681137226 -2023-05-08 14:09:48,033 : DEBUG : 1681137226 -2023-05-08 14:09:48,040 : DEBUG : 1681137226 -2023-05-08 14:09:48,049 : DEBUG : 1681137226 -2023-05-08 14:09:48,057 : DEBUG : 1681137226 -2023-05-08 14:09:56,171 : INFO : interpolate sky radiance -2023-05-08 14:09:56,173 : INFO : interpolate irradiances -2023-05-08 14:10:56,970 : DEBUG : WaterNetworkProtocol -2023-05-08 14:10:56,994 : DEBUG : 1681137226 -2023-05-08 14:10:56,996 : DEBUG : 1681137226 -2023-05-08 14:10:56,997 : DEBUG : 1681137226 -2023-05-08 14:10:57,000 : DEBUG : 1681137226 -2023-05-08 14:10:57,002 : DEBUG : 1681137226 -2023-05-08 14:10:57,004 : DEBUG : 1681137226 -2023-05-08 14:10:59,480 : INFO : interpolate sky radiance -2023-05-08 14:10:59,480 : INFO : interpolate irradiances -2023-05-08 14:11:09,775 : INFO : Apply Mobley 1999 -2023-05-08 14:12:06,889 : INFO : Apply Mobley 1999 -2023-05-08 14:12:24,468 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/Test/SEQ20230410T143155 -2023-05-08 14:12:24,867 : INFO : Processing to L1c... -2023-05-08 14:12:25,371 : DEBUG : WaterNetworkProtocol -2023-05-08 14:12:25,484 : DEBUG : 1681137226 -2023-05-08 14:12:25,492 : DEBUG : 1681137226 -2023-05-08 14:12:25,498 : DEBUG : 1681137226 -2023-05-08 14:12:25,506 : DEBUG : 1681137226 -2023-05-08 14:12:25,513 : DEBUG : 1681137226 -2023-05-08 14:12:25,521 : DEBUG : 1681137226 -2023-05-08 14:12:33,714 : INFO : interpolate sky radiance -2023-05-08 14:12:33,715 : INFO : interpolate irradiances -2023-05-08 14:13:37,745 : INFO : Apply Mobley 1999 -2023-05-08 14:23:53,260 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/Test/SEQ20230410T143155 -2023-05-08 14:23:53,683 : INFO : Processing to L1c... -2023-05-08 14:23:54,229 : DEBUG : WaterNetworkProtocol -2023-05-08 14:23:54,345 : DEBUG : 1681137226 -2023-05-08 14:23:54,356 : DEBUG : 1681137226 -2023-05-08 14:23:54,366 : DEBUG : 1681137226 -2023-05-08 14:23:54,374 : DEBUG : 1681137226 -2023-05-08 14:23:54,382 : DEBUG : 1681137226 -2023-05-08 14:23:54,391 : DEBUG : 1681137226 -2023-05-08 14:24:03,709 : INFO : interpolate sky radiance -2023-05-08 14:24:03,710 : INFO : interpolate irradiances -2023-05-08 14:24:54,449 : INFO : Apply Mobley 1999 -2023-05-08 14:25:35,906 : INFO : Apply Mobley 1999 -2023-05-08 14:29:22,975 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/Test/SEQ20230410T143155 -2023-05-08 14:29:23,370 : INFO : Processing to L1c... -2023-05-08 14:29:23,941 : DEBUG : WaterNetworkProtocol -2023-05-08 14:29:24,048 : DEBUG : 1681137226 -2023-05-08 14:29:24,056 : DEBUG : 1681137226 -2023-05-08 14:29:24,063 : DEBUG : 1681137226 -2023-05-08 14:29:24,070 : DEBUG : 1681137226 -2023-05-08 14:29:24,079 : DEBUG : 1681137226 -2023-05-08 14:29:24,094 : DEBUG : 1681137226 -2023-05-08 14:29:32,476 : INFO : interpolate sky radiance -2023-05-08 14:29:32,477 : INFO : interpolate irradiances -2023-05-08 14:29:40,905 : INFO : Apply Mobley 1999 -2023-05-08 14:29:41,429 : INFO : Processing Anomalies: ['x'] -2023-05-08 14:29:41,429 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/Test/SEQ20230410T143155Failed: TypeError("get_measurement_function() missing 1 required positional argument: 'name'") -2023-05-08 14:29:41,430 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 142, in process_sequence - L1c = surf.process_l1c(L1c_int, L1a_irr) - File "/home/cgoyens/hypernets_processor/hypernets_processor/surface_reflectance/surface_reflectance.py", line 55, in process_l1c - l1ctol1b_function = self._measurement_function_factory.get_measurement_function( -TypeError: get_measurement_function() missing 1 required positional argument: 'name' - -2023-05-08 14:29:41,445 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/Test/SEQ20230410T140203 -2023-05-08 14:29:41,899 : INFO : Processing to L1c... -2023-05-08 14:29:43,196 : DEBUG : WaterNetworkProtocol -2023-05-08 14:29:43,428 : DEBUG : 1681135437 -2023-05-08 14:29:43,443 : DEBUG : 1681135437 -2023-05-08 14:29:43,455 : DEBUG : 1681135437 -2023-05-08 14:29:43,467 : DEBUG : 1681135437 -2023-05-08 14:29:43,480 : DEBUG : 1681135437 -2023-05-08 14:29:43,492 : DEBUG : 1681135437 -2023-05-08 14:29:52,594 : INFO : interpolate sky radiance -2023-05-08 14:29:52,595 : INFO : interpolate irradiances -2023-05-08 14:30:01,592 : INFO : Apply Mobley 1999 -2023-05-08 14:30:02,144 : INFO : Processing Anomalies: ['x', 'x'] -2023-05-08 14:30:02,145 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/Test/SEQ20230410T140203Failed: TypeError("get_measurement_function() missing 1 required positional argument: 'name'") -2023-05-08 14:30:02,145 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 142, in process_sequence - L1c = surf.process_l1c(L1c_int, L1a_irr) - File "/home/cgoyens/hypernets_processor/hypernets_processor/surface_reflectance/surface_reflectance.py", line 55, in process_l1c - l1ctol1b_function = self._measurement_function_factory.get_measurement_function( -TypeError: get_measurement_function() missing 1 required positional argument: 'name' - -2023-05-08 14:31:11,841 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/Test/SEQ20230410T143155 -2023-05-08 14:31:12,220 : INFO : Processing to L1c... -2023-05-08 14:31:12,759 : DEBUG : WaterNetworkProtocol -2023-05-08 14:31:12,881 : DEBUG : 1681137226 -2023-05-08 14:31:12,889 : DEBUG : 1681137226 -2023-05-08 14:31:12,896 : DEBUG : 1681137226 -2023-05-08 14:31:12,903 : DEBUG : 1681137226 -2023-05-08 14:31:12,915 : DEBUG : 1681137226 -2023-05-08 14:31:12,924 : DEBUG : 1681137226 -2023-05-08 14:31:21,271 : INFO : interpolate sky radiance -2023-05-08 14:31:21,272 : INFO : interpolate irradiances -2023-05-08 14:31:29,678 : INFO : Apply Mobley 1999 -2023-05-08 14:31:30,150 : INFO : Processing Anomalies: ['x'] -2023-05-08 14:31:30,150 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/Test/SEQ20230410T143155Failed: TypeError("get_measurement_function() missing 1 required positional argument: 'name'") -2023-05-08 14:31:30,151 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 142, in process_sequence - L1c = surf.process_l1c(L1c_int, L1a_irr) - File "/home/cgoyens/hypernets_processor/hypernets_processor/surface_reflectance/surface_reflectance.py", line 55, in process_l1c - l1ctol1b_function = self._measurement_function_factory.get_measurement_function( -TypeError: get_measurement_function() missing 1 required positional argument: 'name' - -2023-05-08 14:31:30,160 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/Test/SEQ20230410T140203 -2023-05-08 14:31:30,486 : INFO : Processing to L1c... -2023-05-08 14:31:31,000 : DEBUG : WaterNetworkProtocol -2023-05-08 14:31:31,104 : DEBUG : 1681135437 -2023-05-08 14:31:31,112 : DEBUG : 1681135437 -2023-05-08 14:31:31,121 : DEBUG : 1681135437 -2023-05-08 14:31:31,129 : DEBUG : 1681135437 -2023-05-08 14:31:31,136 : DEBUG : 1681135437 -2023-05-08 14:31:31,143 : DEBUG : 1681135437 -2023-05-08 14:31:39,838 : INFO : interpolate sky radiance -2023-05-08 14:31:39,839 : INFO : interpolate irradiances -2023-05-08 14:31:49,231 : INFO : Apply Mobley 1999 -2023-05-08 14:31:49,946 : INFO : Processing Anomalies: ['x', 'x'] -2023-05-08 14:31:49,946 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/Test/SEQ20230410T140203Failed: TypeError("get_measurement_function() missing 1 required positional argument: 'name'") -2023-05-08 14:31:49,947 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 142, in process_sequence - L1c = surf.process_l1c(L1c_int, L1a_irr) - File "/home/cgoyens/hypernets_processor/hypernets_processor/surface_reflectance/surface_reflectance.py", line 55, in process_l1c - l1ctol1b_function = self._measurement_function_factory.get_measurement_function( -TypeError: get_measurement_function() missing 1 required positional argument: 'name' - -2023-05-08 14:32:58,394 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/Test/SEQ20230410T143155 -2023-05-08 14:32:58,779 : INFO : Processing to L1c... -2023-05-08 14:32:59,300 : DEBUG : WaterNetworkProtocol -2023-05-08 14:32:59,402 : DEBUG : 1681137226 -2023-05-08 14:32:59,410 : DEBUG : 1681137226 -2023-05-08 14:32:59,417 : DEBUG : 1681137226 -2023-05-08 14:32:59,424 : DEBUG : 1681137226 -2023-05-08 14:32:59,432 : DEBUG : 1681137226 -2023-05-08 14:32:59,438 : DEBUG : 1681137226 -2023-05-08 14:33:07,668 : INFO : interpolate sky radiance -2023-05-08 14:33:07,669 : INFO : interpolate irradiances -2023-05-08 14:33:15,896 : INFO : Apply Mobley 1999 -2023-05-08 14:33:16,389 : INFO : Processing Anomalies: ['x'] -2023-05-08 14:33:16,389 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/Test/SEQ20230410T143155Failed: TypeError("get_measurement_function() missing 1 required positional argument: 'name'") -2023-05-08 14:33:16,390 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 139, in process_sequence - L1c = surf.process_l1c(L1c_int, L1a_irr) - File "/home/cgoyens/hypernets_processor/hypernets_processor/surface_reflectance/surface_reflectance.py", line 55, in process_l1c - l1ctol1b_function = self._measurement_function_factory.get_measurement_function( -TypeError: get_measurement_function() missing 1 required positional argument: 'name' - -2023-05-08 14:33:16,400 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/Test/SEQ20230410T140203 -2023-05-08 14:33:16,750 : INFO : Processing to L1c... -2023-05-08 14:33:17,217 : DEBUG : WaterNetworkProtocol -2023-05-08 14:33:17,319 : DEBUG : 1681135437 -2023-05-08 14:33:17,327 : DEBUG : 1681135437 -2023-05-08 14:33:17,334 : DEBUG : 1681135437 -2023-05-08 14:33:17,341 : DEBUG : 1681135437 -2023-05-08 14:33:17,349 : DEBUG : 1681135437 -2023-05-08 14:33:17,356 : DEBUG : 1681135437 -2023-05-08 14:33:25,356 : INFO : interpolate sky radiance -2023-05-08 14:33:25,357 : INFO : interpolate irradiances -2023-05-08 14:33:33,452 : INFO : Apply Mobley 1999 -2023-05-08 14:33:33,942 : INFO : Processing Anomalies: ['x', 'x'] -2023-05-08 14:33:33,942 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/Test/SEQ20230410T140203Failed: TypeError("get_measurement_function() missing 1 required positional argument: 'name'") -2023-05-08 14:33:33,942 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 139, in process_sequence - L1c = surf.process_l1c(L1c_int, L1a_irr) - File "/home/cgoyens/hypernets_processor/hypernets_processor/surface_reflectance/surface_reflectance.py", line 55, in process_l1c - l1ctol1b_function = self._measurement_function_factory.get_measurement_function( -TypeError: get_measurement_function() missing 1 required positional argument: 'name' - -2023-05-08 14:34:23,798 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/Test/SEQ20230410T143155 -2023-05-08 14:34:24,169 : INFO : Processing to L1c... -2023-05-08 14:34:24,745 : DEBUG : WaterNetworkProtocol -2023-05-08 14:34:24,849 : DEBUG : 1681137226 -2023-05-08 14:34:24,856 : DEBUG : 1681137226 -2023-05-08 14:34:24,864 : DEBUG : 1681137226 -2023-05-08 14:34:24,871 : DEBUG : 1681137226 -2023-05-08 14:34:24,878 : DEBUG : 1681137226 -2023-05-08 14:34:24,885 : DEBUG : 1681137226 -2023-05-08 14:34:33,010 : INFO : interpolate sky radiance -2023-05-08 14:34:33,011 : INFO : interpolate irradiances -2023-05-08 14:34:41,936 : INFO : Apply Mobley 1999 -2023-05-08 14:34:42,517 : INFO : Processing Anomalies: ['x'] -2023-05-08 14:34:42,517 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/Test/SEQ20230410T143155Failed: TypeError("get_measurement_function() missing 1 required positional argument: 'name'") -2023-05-08 14:34:42,519 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 139, in process_sequence - L1c = surf.process_l1c(L1c_int, L1a_irr) - File "/home/cgoyens/hypernets_processor/hypernets_processor/surface_reflectance/surface_reflectance.py", line 55, in process_l1c - l1ctol1b_function = self._measurement_function_factory.get_measurement_function( -TypeError: get_measurement_function() missing 1 required positional argument: 'name' - -2023-05-08 14:34:42,540 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/Test/SEQ20230410T140203 -2023-05-08 14:34:42,971 : INFO : Processing to L1c... -2023-05-08 14:34:43,525 : DEBUG : WaterNetworkProtocol -2023-05-08 14:34:43,661 : DEBUG : 1681135437 -2023-05-08 14:34:43,669 : DEBUG : 1681135437 -2023-05-08 14:34:43,677 : DEBUG : 1681135437 -2023-05-08 14:34:43,685 : DEBUG : 1681135437 -2023-05-08 14:34:43,693 : DEBUG : 1681135437 -2023-05-08 14:34:43,707 : DEBUG : 1681135437 -2023-05-08 14:34:53,397 : INFO : interpolate sky radiance -2023-05-08 14:34:53,397 : INFO : interpolate irradiances -2023-05-08 14:35:02,890 : INFO : Apply Mobley 1999 -2023-05-08 14:35:03,411 : INFO : Processing Anomalies: ['x', 'x'] -2023-05-08 14:35:03,411 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/Test/SEQ20230410T140203Failed: TypeError("get_measurement_function() missing 1 required positional argument: 'name'") -2023-05-08 14:35:03,411 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 139, in process_sequence - L1c = surf.process_l1c(L1c_int, L1a_irr) - File "/home/cgoyens/hypernets_processor/hypernets_processor/surface_reflectance/surface_reflectance.py", line 55, in process_l1c - l1ctol1b_function = self._measurement_function_factory.get_measurement_function( -TypeError: get_measurement_function() missing 1 required positional argument: 'name' - -2023-05-08 19:20:20,172 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/Test/SEQ20230410T143155 -2023-05-08 19:20:20,535 : INFO : Reading raw data... -2023-05-08 19:20:20,555 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 19:20:20,555 : DEBUG : {'Total Dataset Length': 4131, 'Spectrum Type Information': 136, 'acquisition_time': 1681137276013, 'integration_time': 1024, 'temperature': 16.44601821899414, 'Pixel Count': 2048, 'acceleration_x_mean': -16327, 'acceleration_x_std': 91, 'acceleration_y_mean': -396, 'acceleration_y_std': 243, 'acceleration_z_mean': 116, 'acceleration_z_std': 453} -2023-05-08 19:20:20,559 : DEBUG : Wavelength range: 163.96437072753906 -1145.4518535429638 -2023-05-08 19:20:20,686 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 19:20:20,687 : DEBUG : Reading Data spectra ... -2023-05-08 19:20:20,688 : DEBUG : Reading CRC32 ... -2023-05-08 19:20:20,704 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 19:20:20,704 : DEBUG : Reading Data spectra ... -2023-05-08 19:20:20,706 : DEBUG : Reading CRC32 ... -2023-05-08 19:20:20,722 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 19:20:20,722 : DEBUG : Reading Data spectra ... -2023-05-08 19:20:20,724 : DEBUG : Reading CRC32 ... -2023-05-08 19:20:20,738 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 19:20:20,738 : DEBUG : Reading Data spectra ... -2023-05-08 19:20:20,739 : DEBUG : Reading CRC32 ... -2023-05-08 19:20:20,753 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 19:20:20,753 : DEBUG : Reading Data spectra ... -2023-05-08 19:20:20,755 : DEBUG : Reading CRC32 ... -2023-05-08 19:20:20,768 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 19:20:20,769 : DEBUG : Reading Data spectra ... -2023-05-08 19:20:20,770 : DEBUG : Reading CRC32 ... -2023-05-08 19:20:20,872 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 19:20:20,872 : DEBUG : {'Total Dataset Length': 4131, 'Spectrum Type Information': 144, 'acquisition_time': 1681137227706, 'integration_time': 256, 'temperature': 15.911444664001465, 'Pixel Count': 2048, 'acceleration_x_mean': 12620, 'acceleration_x_std': 638, 'acceleration_y_mean': -10200, 'acceleration_y_std': 784, 'acceleration_z_mean': -2932, 'acceleration_z_std': 499} -2023-05-08 19:20:20,876 : DEBUG : Wavelength range: 163.85150146484375 -1145.61306723725 -2023-05-08 19:20:20,993 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 19:20:20,993 : DEBUG : Reading Data spectra ... -2023-05-08 19:20:20,994 : DEBUG : Reading CRC32 ... -2023-05-08 19:20:21,008 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 19:20:21,008 : DEBUG : Reading Data spectra ... -2023-05-08 19:20:21,009 : DEBUG : Reading CRC32 ... -2023-05-08 19:20:21,023 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 19:20:21,024 : DEBUG : Reading Data spectra ... -2023-05-08 19:20:21,025 : DEBUG : Reading CRC32 ... -2023-05-08 19:20:21,040 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 19:20:21,040 : DEBUG : Reading Data spectra ... -2023-05-08 19:20:21,041 : DEBUG : Reading CRC32 ... -2023-05-08 19:20:21,055 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 19:20:21,055 : DEBUG : Reading Data spectra ... -2023-05-08 19:20:21,056 : DEBUG : Reading CRC32 ... -2023-05-08 19:20:21,072 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 19:20:21,072 : DEBUG : Reading Data spectra ... -2023-05-08 19:20:21,075 : DEBUG : Reading CRC32 ... -2023-05-08 19:20:21,096 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 19:20:21,096 : DEBUG : Reading Data spectra ... -2023-05-08 19:20:21,098 : DEBUG : Reading CRC32 ... -2023-05-08 19:20:21,113 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 19:20:21,113 : DEBUG : Reading Data spectra ... -2023-05-08 19:20:21,114 : DEBUG : Reading CRC32 ... -2023-05-08 19:20:21,129 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 19:20:21,129 : DEBUG : Reading Data spectra ... -2023-05-08 19:20:21,131 : DEBUG : Reading CRC32 ... -2023-05-08 19:20:21,146 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 19:20:21,146 : DEBUG : Reading Data spectra ... -2023-05-08 19:20:21,147 : DEBUG : Reading CRC32 ... -2023-05-08 19:20:21,161 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 19:20:21,161 : DEBUG : Reading Data spectra ... -2023-05-08 19:20:21,163 : DEBUG : Reading CRC32 ... -2023-05-08 19:20:21,176 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 19:20:21,176 : DEBUG : Reading Data spectra ... -2023-05-08 19:20:21,178 : DEBUG : Reading CRC32 ... -2023-05-08 19:20:21,354 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 19:20:21,354 : DEBUG : {'Total Dataset Length': 4131, 'Spectrum Type Information': 128, 'acquisition_time': 1681137201860, 'integration_time': 32, 'temperature': 15.547460556030273, 'Pixel Count': 2048, 'acceleration_x_mean': -12608, 'acceleration_x_std': 0, 'acceleration_y_mean': -10176, 'acceleration_y_std': 0, 'acceleration_z_mean': -2752, 'acceleration_z_std': 0} -2023-05-08 19:20:21,360 : DEBUG : Wavelength range: 163.85150146484375 -1145.61306723725 -2023-05-08 19:20:21,460 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 19:20:21,461 : DEBUG : Reading Data spectra ... -2023-05-08 19:20:21,462 : DEBUG : Reading CRC32 ... -2023-05-08 19:20:21,476 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 19:20:21,476 : DEBUG : Reading Data spectra ... -2023-05-08 19:20:21,477 : DEBUG : Reading CRC32 ... -2023-05-08 19:20:21,491 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 19:20:21,491 : DEBUG : Reading Data spectra ... -2023-05-08 19:20:21,492 : DEBUG : Reading CRC32 ... -2023-05-08 19:20:21,506 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 19:20:21,506 : DEBUG : Reading Data spectra ... -2023-05-08 19:20:21,507 : DEBUG : Reading CRC32 ... -2023-05-08 19:20:21,521 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 19:20:21,522 : DEBUG : Reading Data spectra ... -2023-05-08 19:20:21,523 : DEBUG : Reading CRC32 ... -2023-05-08 19:20:21,536 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 19:20:21,537 : DEBUG : Reading Data spectra ... -2023-05-08 19:20:21,538 : DEBUG : Reading CRC32 ... -2023-05-08 19:20:21,552 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 19:20:21,553 : DEBUG : Reading Data spectra ... -2023-05-08 19:20:21,554 : DEBUG : Reading CRC32 ... -2023-05-08 19:20:21,567 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 19:20:21,568 : DEBUG : Reading Data spectra ... -2023-05-08 19:20:21,569 : DEBUG : Reading CRC32 ... -2023-05-08 19:20:21,587 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 19:20:21,587 : DEBUG : Reading Data spectra ... -2023-05-08 19:20:21,588 : DEBUG : Reading CRC32 ... -2023-05-08 19:20:21,603 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 19:20:21,603 : DEBUG : Reading Data spectra ... -2023-05-08 19:20:21,604 : DEBUG : Reading CRC32 ... -2023-05-08 19:20:21,618 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 19:20:21,618 : DEBUG : Reading Data spectra ... -2023-05-08 19:20:21,619 : DEBUG : Reading CRC32 ... -2023-05-08 19:20:21,633 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 19:20:21,633 : DEBUG : Reading Data spectra ... -2023-05-08 19:20:21,635 : DEBUG : Reading CRC32 ... -2023-05-08 19:20:21,649 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 19:20:21,649 : DEBUG : Reading Data spectra ... -2023-05-08 19:20:21,650 : DEBUG : Reading CRC32 ... -2023-05-08 19:20:21,664 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 19:20:21,664 : DEBUG : Reading Data spectra ... -2023-05-08 19:20:21,666 : DEBUG : Reading CRC32 ... -2023-05-08 19:20:21,686 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 19:20:21,686 : DEBUG : Reading Data spectra ... -2023-05-08 19:20:21,689 : DEBUG : Reading CRC32 ... -2023-05-08 19:20:21,831 : INFO : Done -2023-05-08 19:20:21,832 : INFO : Processing to L1a... -2023-05-08 19:20:22,639 : DEBUG : preprocessing done -2023-05-08 19:20:29,819 : DEBUG : preprocessing done -2023-05-08 19:20:33,182 : INFO : Done -2023-05-08 19:20:33,183 : INFO : Processing to L1b radiance... -2023-05-08 19:20:39,799 : INFO : Done -2023-05-08 19:20:39,800 : INFO : Processing to L1b irradiance... -2023-05-08 19:20:44,248 : INFO : Done -2023-05-08 19:20:44,555 : INFO : Processing to L1c... -2023-05-08 19:20:44,968 : DEBUG : WaterNetworkProtocol -2023-05-08 19:20:45,060 : DEBUG : 1681137226 -2023-05-08 19:20:45,067 : DEBUG : 1681137226 -2023-05-08 19:20:45,073 : DEBUG : 1681137226 -2023-05-08 19:20:45,081 : DEBUG : 1681137226 -2023-05-08 19:20:45,092 : DEBUG : 1681137226 -2023-05-08 19:20:45,099 : DEBUG : 1681137226 -2023-05-08 19:20:52,301 : INFO : interpolate sky radiance -2023-05-08 19:20:52,302 : INFO : interpolate irradiances -2023-05-08 19:20:59,708 : INFO : Apply Mobley 1999 -2023-05-08 19:28:22,650 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/Test/SEQ20230410T143155 -2023-05-08 19:28:22,787 : INFO : Reading raw data... -2023-05-08 19:28:22,788 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 19:28:22,788 : DEBUG : {'Total Dataset Length': 4131, 'Spectrum Type Information': 136, 'acquisition_time': 1681137276013, 'integration_time': 1024, 'temperature': 16.44601821899414, 'Pixel Count': 2048, 'acceleration_x_mean': -16327, 'acceleration_x_std': 91, 'acceleration_y_mean': -396, 'acceleration_y_std': 243, 'acceleration_z_mean': 116, 'acceleration_z_std': 453} -2023-05-08 19:28:22,790 : DEBUG : Wavelength range: 163.96437072753906 -1145.4518535429638 -2023-05-08 19:28:22,815 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 19:28:22,815 : DEBUG : Reading Data spectra ... -2023-05-08 19:28:22,816 : DEBUG : Reading CRC32 ... -2023-05-08 19:28:22,821 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 19:28:22,821 : DEBUG : Reading Data spectra ... -2023-05-08 19:28:22,822 : DEBUG : Reading CRC32 ... -2023-05-08 19:28:22,826 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 19:28:22,826 : DEBUG : Reading Data spectra ... -2023-05-08 19:28:22,827 : DEBUG : Reading CRC32 ... -2023-05-08 19:28:22,832 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 19:28:22,832 : DEBUG : Reading Data spectra ... -2023-05-08 19:28:22,832 : DEBUG : Reading CRC32 ... -2023-05-08 19:28:22,837 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 19:28:22,837 : DEBUG : Reading Data spectra ... -2023-05-08 19:28:22,838 : DEBUG : Reading CRC32 ... -2023-05-08 19:28:22,842 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 19:28:22,842 : DEBUG : Reading Data spectra ... -2023-05-08 19:28:22,843 : DEBUG : Reading CRC32 ... -2023-05-08 19:28:22,900 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 19:28:22,900 : DEBUG : {'Total Dataset Length': 4131, 'Spectrum Type Information': 144, 'acquisition_time': 1681137227706, 'integration_time': 256, 'temperature': 15.911444664001465, 'Pixel Count': 2048, 'acceleration_x_mean': 12620, 'acceleration_x_std': 638, 'acceleration_y_mean': -10200, 'acceleration_y_std': 784, 'acceleration_z_mean': -2932, 'acceleration_z_std': 499} -2023-05-08 19:28:22,902 : DEBUG : Wavelength range: 163.85150146484375 -1145.61306723725 -2023-05-08 19:28:22,925 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 19:28:22,925 : DEBUG : Reading Data spectra ... -2023-05-08 19:28:22,926 : DEBUG : Reading CRC32 ... -2023-05-08 19:28:22,931 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 19:28:22,931 : DEBUG : Reading Data spectra ... -2023-05-08 19:28:22,931 : DEBUG : Reading CRC32 ... -2023-05-08 19:28:22,936 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 19:28:22,936 : DEBUG : Reading Data spectra ... -2023-05-08 19:28:22,937 : DEBUG : Reading CRC32 ... -2023-05-08 19:28:22,941 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 19:28:22,941 : DEBUG : Reading Data spectra ... -2023-05-08 19:28:22,942 : DEBUG : Reading CRC32 ... -2023-05-08 19:28:22,946 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 19:28:22,947 : DEBUG : Reading Data spectra ... -2023-05-08 19:28:22,947 : DEBUG : Reading CRC32 ... -2023-05-08 19:28:22,952 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 19:28:22,952 : DEBUG : Reading Data spectra ... -2023-05-08 19:28:22,953 : DEBUG : Reading CRC32 ... -2023-05-08 19:28:22,957 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 19:28:22,957 : DEBUG : Reading Data spectra ... -2023-05-08 19:28:22,958 : DEBUG : Reading CRC32 ... -2023-05-08 19:28:22,962 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 19:28:22,962 : DEBUG : Reading Data spectra ... -2023-05-08 19:28:22,963 : DEBUG : Reading CRC32 ... -2023-05-08 19:28:22,967 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 19:28:22,967 : DEBUG : Reading Data spectra ... -2023-05-08 19:28:22,968 : DEBUG : Reading CRC32 ... -2023-05-08 19:28:22,973 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 19:28:22,973 : DEBUG : Reading Data spectra ... -2023-05-08 19:28:22,974 : DEBUG : Reading CRC32 ... -2023-05-08 19:28:22,978 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 19:28:22,978 : DEBUG : Reading Data spectra ... -2023-05-08 19:28:22,979 : DEBUG : Reading CRC32 ... -2023-05-08 19:28:22,983 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 19:28:22,983 : DEBUG : Reading Data spectra ... -2023-05-08 19:28:22,984 : DEBUG : Reading CRC32 ... -2023-05-08 19:28:23,035 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 19:28:23,035 : DEBUG : {'Total Dataset Length': 4131, 'Spectrum Type Information': 128, 'acquisition_time': 1681137201860, 'integration_time': 32, 'temperature': 15.547460556030273, 'Pixel Count': 2048, 'acceleration_x_mean': -12608, 'acceleration_x_std': 0, 'acceleration_y_mean': -10176, 'acceleration_y_std': 0, 'acceleration_z_mean': -2752, 'acceleration_z_std': 0} -2023-05-08 19:28:23,037 : DEBUG : Wavelength range: 163.85150146484375 -1145.61306723725 -2023-05-08 19:28:23,059 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 19:28:23,059 : DEBUG : Reading Data spectra ... -2023-05-08 19:28:23,060 : DEBUG : Reading CRC32 ... -2023-05-08 19:28:23,065 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 19:28:23,065 : DEBUG : Reading Data spectra ... -2023-05-08 19:28:23,065 : DEBUG : Reading CRC32 ... -2023-05-08 19:28:23,070 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 19:28:23,070 : DEBUG : Reading Data spectra ... -2023-05-08 19:28:23,071 : DEBUG : Reading CRC32 ... -2023-05-08 19:28:23,075 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 19:28:23,075 : DEBUG : Reading Data spectra ... -2023-05-08 19:28:23,076 : DEBUG : Reading CRC32 ... -2023-05-08 19:28:23,081 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 19:28:23,081 : DEBUG : Reading Data spectra ... -2023-05-08 19:28:23,081 : DEBUG : Reading CRC32 ... -2023-05-08 19:28:23,086 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 19:28:23,086 : DEBUG : Reading Data spectra ... -2023-05-08 19:28:23,087 : DEBUG : Reading CRC32 ... -2023-05-08 19:28:23,091 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 19:28:23,091 : DEBUG : Reading Data spectra ... -2023-05-08 19:28:23,092 : DEBUG : Reading CRC32 ... -2023-05-08 19:28:23,096 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 19:28:23,097 : DEBUG : Reading Data spectra ... -2023-05-08 19:28:23,097 : DEBUG : Reading CRC32 ... -2023-05-08 19:28:23,102 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 19:28:23,102 : DEBUG : Reading Data spectra ... -2023-05-08 19:28:23,102 : DEBUG : Reading CRC32 ... -2023-05-08 19:28:23,107 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 19:28:23,107 : DEBUG : Reading Data spectra ... -2023-05-08 19:28:23,108 : DEBUG : Reading CRC32 ... -2023-05-08 19:28:23,112 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 19:28:23,112 : DEBUG : Reading Data spectra ... -2023-05-08 19:28:23,113 : DEBUG : Reading CRC32 ... -2023-05-08 19:28:23,117 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 19:28:23,117 : DEBUG : Reading Data spectra ... -2023-05-08 19:28:23,118 : DEBUG : Reading CRC32 ... -2023-05-08 19:28:23,123 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 19:28:23,123 : DEBUG : Reading Data spectra ... -2023-05-08 19:28:23,123 : DEBUG : Reading CRC32 ... -2023-05-08 19:28:23,128 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 19:28:23,128 : DEBUG : Reading Data spectra ... -2023-05-08 19:28:23,129 : DEBUG : Reading CRC32 ... -2023-05-08 19:28:23,133 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 19:28:23,133 : DEBUG : Reading Data spectra ... -2023-05-08 19:28:23,135 : DEBUG : Reading CRC32 ... -2023-05-08 19:28:23,244 : INFO : Done -2023-05-08 19:28:23,244 : INFO : Processing to L1a... -2023-05-08 19:28:23,435 : DEBUG : preprocessing done -2023-05-08 19:28:29,270 : DEBUG : preprocessing done -2023-05-08 19:28:32,419 : INFO : Done -2023-05-08 19:28:32,419 : INFO : Processing to L1b radiance... -2023-05-08 19:28:34,002 : INFO : Done -2023-05-08 19:28:34,002 : INFO : Processing to L1b irradiance... -2023-05-08 19:28:35,067 : INFO : Done -2023-05-08 19:28:35,158 : INFO : Processing to L1c... -2023-05-08 19:28:35,283 : DEBUG : WaterNetworkProtocol -2023-05-08 19:28:35,302 : DEBUG : 1681137226 -2023-05-08 19:28:35,304 : DEBUG : 1681137226 -2023-05-08 19:28:35,305 : DEBUG : 1681137226 -2023-05-08 19:28:35,307 : DEBUG : 1681137226 -2023-05-08 19:28:35,308 : DEBUG : 1681137226 -2023-05-08 19:28:35,310 : DEBUG : 1681137226 -2023-05-08 19:28:37,147 : INFO : interpolate sky radiance -2023-05-08 19:28:37,147 : INFO : interpolate irradiances -2023-05-08 19:28:38,843 : INFO : Apply Mobley 1999 -2023-05-08 19:28:39,136 : INFO : Processing Anomalies: ['x'] -2023-05-08 19:28:39,136 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/Test/SEQ20230410T143155Failed: AttributeError("'WaterNetworkProtocol' object has no attribute 'context'") -2023-05-08 19:28:39,152 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 139, in process_sequence - L1c = surf.process_l1c(L1c_int, L1a_irr) - File "/home/cgoyens/hypernets_processor/hypernets_processor/surface_reflectance/surface_reflectance.py", line 65, in process_l1c - dataset_l1c = l1ctol1b_function.propagate_ds_specific( - File "/usr/local/lib/python3.8/dist-packages/punpy/digital_effects_table/measurement_function.py", line 581, in propagate_ds_specific - y = self.check_sizes_and_run(*args, expand=expand, ds_out_pre=ds_out_pre) - File "/usr/local/lib/python3.8/dist-packages/punpy/digital_effects_table/measurement_function.py", line 821, in check_sizes_and_run - y = self.run(*args, expand=expand) - File "/usr/local/lib/python3.8/dist-packages/punpy/digital_effects_table/measurement_function.py", line 778, in run - return np.array(self.meas_function(*input_qty),dtype=self.prop.dtype) - File "/home/cgoyens/hypernets_processor/hypernets_processor/surface_reflectance/measurement_functions/water_network_protocol.py", line 43, in meas_function - w1 = self.context.get_config_value("similarity_w1") -AttributeError: 'WaterNetworkProtocol' object has no attribute 'context' - -2023-05-08 19:28:39,157 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/Test/SEQ20230410T140203 -2023-05-08 19:28:39,208 : INFO : Reading raw data... -2023-05-08 19:28:39,210 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 19:28:39,210 : DEBUG : {'Total Dataset Length': 4131, 'Spectrum Type Information': 136, 'acquisition_time': 1681135485382, 'integration_time': 1024, 'temperature': 16.488752365112305, 'Pixel Count': 2048, 'acceleration_x_mean': -16303, 'acceleration_x_std': 131, 'acceleration_y_mean': -429, 'acceleration_y_std': 595, 'acceleration_z_mean': -29, 'acceleration_z_std': 643} -2023-05-08 19:28:39,211 : DEBUG : Wavelength range: 163.96437072753906 -1145.4518535429638 -2023-05-08 19:28:39,234 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 19:28:39,234 : DEBUG : Reading Data spectra ... -2023-05-08 19:28:39,235 : DEBUG : Reading CRC32 ... -2023-05-08 19:28:39,240 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 19:28:39,240 : DEBUG : Reading Data spectra ... -2023-05-08 19:28:39,241 : DEBUG : Reading CRC32 ... -2023-05-08 19:28:39,245 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 19:28:39,245 : DEBUG : Reading Data spectra ... -2023-05-08 19:28:39,246 : DEBUG : Reading CRC32 ... -2023-05-08 19:28:39,250 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 19:28:39,250 : DEBUG : Reading Data spectra ... -2023-05-08 19:28:39,251 : DEBUG : Reading CRC32 ... -2023-05-08 19:28:39,256 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 19:28:39,256 : DEBUG : Reading Data spectra ... -2023-05-08 19:28:39,256 : DEBUG : Reading CRC32 ... -2023-05-08 19:28:39,261 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 19:28:39,261 : DEBUG : Reading Data spectra ... -2023-05-08 19:28:39,262 : DEBUG : Reading CRC32 ... -2023-05-08 19:28:39,313 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 19:28:39,313 : DEBUG : {'Total Dataset Length': 4131, 'Spectrum Type Information': 144, 'acquisition_time': 1681135438327, 'integration_time': 256, 'temperature': 16.039813995361328, 'Pixel Count': 2048, 'acceleration_x_mean': 12473, 'acceleration_x_std': 262, 'acceleration_y_mean': -10256, 'acceleration_y_std': 292, 'acceleration_z_mean': -2960, 'acceleration_z_std': 469} -2023-05-08 19:28:39,314 : DEBUG : Wavelength range: 163.85150146484375 -1145.61306723725 -2023-05-08 19:28:39,350 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 19:28:39,350 : DEBUG : Reading Data spectra ... -2023-05-08 19:28:39,351 : DEBUG : Reading CRC32 ... -2023-05-08 19:28:39,362 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 19:28:39,362 : DEBUG : Reading Data spectra ... -2023-05-08 19:28:39,364 : DEBUG : Reading CRC32 ... -2023-05-08 19:28:39,373 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 19:28:39,374 : DEBUG : Reading Data spectra ... -2023-05-08 19:28:39,376 : DEBUG : Reading CRC32 ... -2023-05-08 19:28:39,387 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 19:28:39,387 : DEBUG : Reading Data spectra ... -2023-05-08 19:28:39,389 : DEBUG : Reading CRC32 ... -2023-05-08 19:28:39,403 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 19:28:39,403 : DEBUG : Reading Data spectra ... -2023-05-08 19:28:39,404 : DEBUG : Reading CRC32 ... -2023-05-08 19:28:39,414 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 19:28:39,414 : DEBUG : Reading Data spectra ... -2023-05-08 19:28:39,415 : DEBUG : Reading CRC32 ... -2023-05-08 19:28:39,426 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 19:28:39,427 : DEBUG : Reading Data spectra ... -2023-05-08 19:28:39,428 : DEBUG : Reading CRC32 ... -2023-05-08 19:28:39,436 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 19:28:39,436 : DEBUG : Reading Data spectra ... -2023-05-08 19:28:39,438 : DEBUG : Reading CRC32 ... -2023-05-08 19:28:39,443 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 19:28:39,443 : DEBUG : Reading Data spectra ... -2023-05-08 19:28:39,444 : DEBUG : Reading CRC32 ... -2023-05-08 19:28:39,449 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 19:28:39,449 : DEBUG : Reading Data spectra ... -2023-05-08 19:28:39,449 : DEBUG : Reading CRC32 ... -2023-05-08 19:28:39,454 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 19:28:39,454 : DEBUG : Reading Data spectra ... -2023-05-08 19:28:39,455 : DEBUG : Reading CRC32 ... -2023-05-08 19:28:39,459 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 19:28:39,459 : DEBUG : Reading Data spectra ... -2023-05-08 19:28:39,460 : DEBUG : Reading CRC32 ... -2023-05-08 19:28:39,508 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 19:28:39,508 : DEBUG : {'Total Dataset Length': 4131, 'Spectrum Type Information': 128, 'acquisition_time': 1681135412063, 'integration_time': 32, 'temperature': 15.654559135437012, 'Pixel Count': 2048, 'acceleration_x_mean': -12992, 'acceleration_x_std': 0, 'acceleration_y_mean': -9856, 'acceleration_y_std': 0, 'acceleration_z_mean': -2048, 'acceleration_z_std': 0} -2023-05-08 19:28:39,510 : DEBUG : Wavelength range: 163.85150146484375 -1145.61306723725 -2023-05-08 19:28:39,536 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 19:28:39,536 : DEBUG : Reading Data spectra ... -2023-05-08 19:28:39,537 : DEBUG : Reading CRC32 ... -2023-05-08 19:28:39,541 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 19:28:39,541 : DEBUG : Reading Data spectra ... -2023-05-08 19:28:39,542 : DEBUG : Reading CRC32 ... -2023-05-08 19:28:39,547 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 19:28:39,547 : DEBUG : Reading Data spectra ... -2023-05-08 19:28:39,547 : DEBUG : Reading CRC32 ... -2023-05-08 19:28:39,552 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 19:28:39,552 : DEBUG : Reading Data spectra ... -2023-05-08 19:28:39,553 : DEBUG : Reading CRC32 ... -2023-05-08 19:28:39,557 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 19:28:39,557 : DEBUG : Reading Data spectra ... -2023-05-08 19:28:39,558 : DEBUG : Reading CRC32 ... -2023-05-08 19:28:39,562 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 19:28:39,562 : DEBUG : Reading Data spectra ... -2023-05-08 19:28:39,563 : DEBUG : Reading CRC32 ... -2023-05-08 19:28:39,572 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 19:28:39,572 : DEBUG : Reading Data spectra ... -2023-05-08 19:28:39,573 : DEBUG : Reading CRC32 ... -2023-05-08 19:28:39,582 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 19:28:39,583 : DEBUG : Reading Data spectra ... -2023-05-08 19:28:39,584 : DEBUG : Reading CRC32 ... -2023-05-08 19:28:39,595 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 19:28:39,595 : DEBUG : Reading Data spectra ... -2023-05-08 19:28:39,596 : DEBUG : Reading CRC32 ... -2023-05-08 19:28:39,609 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 19:28:39,609 : DEBUG : Reading Data spectra ... -2023-05-08 19:28:39,610 : DEBUG : Reading CRC32 ... -2023-05-08 19:28:39,620 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 19:28:39,620 : DEBUG : Reading Data spectra ... -2023-05-08 19:28:39,621 : DEBUG : Reading CRC32 ... -2023-05-08 19:28:39,629 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 19:28:39,630 : DEBUG : Reading Data spectra ... -2023-05-08 19:28:39,631 : DEBUG : Reading CRC32 ... -2023-05-08 19:28:39,640 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 19:28:39,640 : DEBUG : Reading Data spectra ... -2023-05-08 19:28:39,642 : DEBUG : Reading CRC32 ... -2023-05-08 19:28:39,653 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 19:28:39,653 : DEBUG : Reading Data spectra ... -2023-05-08 19:28:39,654 : DEBUG : Reading CRC32 ... -2023-05-08 19:28:39,662 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 19:28:39,662 : DEBUG : Reading Data spectra ... -2023-05-08 19:28:39,663 : DEBUG : Reading CRC32 ... -2023-05-08 19:28:39,727 : INFO : Done -2023-05-08 19:28:39,727 : INFO : Processing to L1a... -2023-05-08 19:28:39,965 : DEBUG : preprocessing done -2023-05-08 19:28:45,610 : DEBUG : preprocessing done -2023-05-08 19:28:48,629 : INFO : Done -2023-05-08 19:28:48,629 : INFO : Processing to L1b radiance... -2023-05-08 19:28:50,215 : INFO : Done -2023-05-08 19:28:50,215 : INFO : Processing to L1b irradiance... -2023-05-08 19:28:51,281 : INFO : Done -2023-05-08 19:28:51,376 : INFO : Processing to L1c... -2023-05-08 19:28:51,494 : DEBUG : WaterNetworkProtocol -2023-05-08 19:28:51,514 : DEBUG : 1681135437 -2023-05-08 19:28:51,515 : DEBUG : 1681135437 -2023-05-08 19:28:51,516 : DEBUG : 1681135437 -2023-05-08 19:28:51,518 : DEBUG : 1681135437 -2023-05-08 19:28:51,519 : DEBUG : 1681135437 -2023-05-08 19:28:51,521 : DEBUG : 1681135437 -2023-05-08 19:28:53,357 : INFO : interpolate sky radiance -2023-05-08 19:28:53,358 : INFO : interpolate irradiances -2023-05-08 19:28:55,090 : INFO : Apply Mobley 1999 -2023-05-08 19:28:55,367 : INFO : Processing Anomalies: ['x', 'x'] -2023-05-08 19:28:55,367 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/Test/SEQ20230410T140203Failed: AttributeError("'WaterNetworkProtocol' object has no attribute 'context'") -2023-05-08 19:28:55,367 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 139, in process_sequence - L1c = surf.process_l1c(L1c_int, L1a_irr) - File "/home/cgoyens/hypernets_processor/hypernets_processor/surface_reflectance/surface_reflectance.py", line 65, in process_l1c - dataset_l1c = l1ctol1b_function.propagate_ds_specific( - File "/usr/local/lib/python3.8/dist-packages/punpy/digital_effects_table/measurement_function.py", line 581, in propagate_ds_specific - y = self.check_sizes_and_run(*args, expand=expand, ds_out_pre=ds_out_pre) - File "/usr/local/lib/python3.8/dist-packages/punpy/digital_effects_table/measurement_function.py", line 821, in check_sizes_and_run - y = self.run(*args, expand=expand) - File "/usr/local/lib/python3.8/dist-packages/punpy/digital_effects_table/measurement_function.py", line 778, in run - return np.array(self.meas_function(*input_qty),dtype=self.prop.dtype) - File "/home/cgoyens/hypernets_processor/hypernets_processor/surface_reflectance/measurement_functions/water_network_protocol.py", line 43, in meas_function - w1 = self.context.get_config_value("similarity_w1") -AttributeError: 'WaterNetworkProtocol' object has no attribute 'context' - -2023-05-08 19:30:02,066 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/Test/SEQ20230410T143155 -2023-05-08 19:30:02,341 : INFO : Reading raw data... -2023-05-08 19:30:02,346 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 19:30:02,346 : DEBUG : {'Total Dataset Length': 4131, 'Spectrum Type Information': 136, 'acquisition_time': 1681137276013, 'integration_time': 1024, 'temperature': 16.44601821899414, 'Pixel Count': 2048, 'acceleration_x_mean': -16327, 'acceleration_x_std': 91, 'acceleration_y_mean': -396, 'acceleration_y_std': 243, 'acceleration_z_mean': 116, 'acceleration_z_std': 453} -2023-05-08 19:30:02,350 : DEBUG : Wavelength range: 163.96437072753906 -1145.4518535429638 -2023-05-08 19:30:02,452 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 19:30:02,452 : DEBUG : Reading Data spectra ... -2023-05-08 19:30:02,453 : DEBUG : Reading CRC32 ... -2023-05-08 19:30:02,468 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 19:30:02,469 : DEBUG : Reading Data spectra ... -2023-05-08 19:30:02,470 : DEBUG : Reading CRC32 ... -2023-05-08 19:30:02,484 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 19:30:02,484 : DEBUG : Reading Data spectra ... -2023-05-08 19:30:02,485 : DEBUG : Reading CRC32 ... -2023-05-08 19:30:02,499 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 19:30:02,500 : DEBUG : Reading Data spectra ... -2023-05-08 19:30:02,501 : DEBUG : Reading CRC32 ... -2023-05-08 19:30:02,515 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 19:30:02,515 : DEBUG : Reading Data spectra ... -2023-05-08 19:30:02,516 : DEBUG : Reading CRC32 ... -2023-05-08 19:30:02,530 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 19:30:02,530 : DEBUG : Reading Data spectra ... -2023-05-08 19:30:02,532 : DEBUG : Reading CRC32 ... -2023-05-08 19:30:02,637 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 19:30:02,637 : DEBUG : {'Total Dataset Length': 4131, 'Spectrum Type Information': 144, 'acquisition_time': 1681137227706, 'integration_time': 256, 'temperature': 15.911444664001465, 'Pixel Count': 2048, 'acceleration_x_mean': 12620, 'acceleration_x_std': 638, 'acceleration_y_mean': -10200, 'acceleration_y_std': 784, 'acceleration_z_mean': -2932, 'acceleration_z_std': 499} -2023-05-08 19:30:02,640 : DEBUG : Wavelength range: 163.85150146484375 -1145.61306723725 -2023-05-08 19:30:02,800 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 19:30:02,800 : DEBUG : Reading Data spectra ... -2023-05-08 19:30:02,801 : DEBUG : Reading CRC32 ... -2023-05-08 19:30:02,816 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 19:30:02,817 : DEBUG : Reading Data spectra ... -2023-05-08 19:30:02,818 : DEBUG : Reading CRC32 ... -2023-05-08 19:30:02,833 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 19:30:02,833 : DEBUG : Reading Data spectra ... -2023-05-08 19:30:02,835 : DEBUG : Reading CRC32 ... -2023-05-08 19:30:02,850 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 19:30:02,851 : DEBUG : Reading Data spectra ... -2023-05-08 19:30:02,852 : DEBUG : Reading CRC32 ... -2023-05-08 19:30:02,867 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 19:30:02,867 : DEBUG : Reading Data spectra ... -2023-05-08 19:30:02,868 : DEBUG : Reading CRC32 ... -2023-05-08 19:30:02,884 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 19:30:02,884 : DEBUG : Reading Data spectra ... -2023-05-08 19:30:02,885 : DEBUG : Reading CRC32 ... -2023-05-08 19:30:02,901 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 19:30:02,901 : DEBUG : Reading Data spectra ... -2023-05-08 19:30:02,902 : DEBUG : Reading CRC32 ... -2023-05-08 19:30:02,917 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 19:30:02,917 : DEBUG : Reading Data spectra ... -2023-05-08 19:30:02,919 : DEBUG : Reading CRC32 ... -2023-05-08 19:30:02,934 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 19:30:02,934 : DEBUG : Reading Data spectra ... -2023-05-08 19:30:02,935 : DEBUG : Reading CRC32 ... -2023-05-08 19:30:02,951 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 19:30:02,951 : DEBUG : Reading Data spectra ... -2023-05-08 19:30:02,952 : DEBUG : Reading CRC32 ... -2023-05-08 19:30:02,967 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 19:30:02,967 : DEBUG : Reading Data spectra ... -2023-05-08 19:30:02,968 : DEBUG : Reading CRC32 ... -2023-05-08 19:30:02,983 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 19:30:02,983 : DEBUG : Reading Data spectra ... -2023-05-08 19:30:02,984 : DEBUG : Reading CRC32 ... -2023-05-08 19:30:03,147 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 19:30:03,147 : DEBUG : {'Total Dataset Length': 4131, 'Spectrum Type Information': 128, 'acquisition_time': 1681137201860, 'integration_time': 32, 'temperature': 15.547460556030273, 'Pixel Count': 2048, 'acceleration_x_mean': -12608, 'acceleration_x_std': 0, 'acceleration_y_mean': -10176, 'acceleration_y_std': 0, 'acceleration_z_mean': -2752, 'acceleration_z_std': 0} -2023-05-08 19:30:03,150 : DEBUG : Wavelength range: 163.85150146484375 -1145.61306723725 -2023-05-08 19:30:03,258 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 19:30:03,258 : DEBUG : Reading Data spectra ... -2023-05-08 19:30:03,259 : DEBUG : Reading CRC32 ... -2023-05-08 19:30:03,273 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 19:30:03,273 : DEBUG : Reading Data spectra ... -2023-05-08 19:30:03,274 : DEBUG : Reading CRC32 ... -2023-05-08 19:30:03,288 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 19:30:03,289 : DEBUG : Reading Data spectra ... -2023-05-08 19:30:03,290 : DEBUG : Reading CRC32 ... -2023-05-08 19:30:03,304 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 19:30:03,304 : DEBUG : Reading Data spectra ... -2023-05-08 19:30:03,305 : DEBUG : Reading CRC32 ... -2023-05-08 19:30:03,319 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 19:30:03,319 : DEBUG : Reading Data spectra ... -2023-05-08 19:30:03,321 : DEBUG : Reading CRC32 ... -2023-05-08 19:30:03,335 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 19:30:03,335 : DEBUG : Reading Data spectra ... -2023-05-08 19:30:03,336 : DEBUG : Reading CRC32 ... -2023-05-08 19:30:03,352 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 19:30:03,352 : DEBUG : Reading Data spectra ... -2023-05-08 19:30:03,355 : DEBUG : Reading CRC32 ... -2023-05-08 19:30:03,370 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 19:30:03,371 : DEBUG : Reading Data spectra ... -2023-05-08 19:30:03,372 : DEBUG : Reading CRC32 ... -2023-05-08 19:30:03,386 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 19:30:03,386 : DEBUG : Reading Data spectra ... -2023-05-08 19:30:03,387 : DEBUG : Reading CRC32 ... -2023-05-08 19:30:03,401 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 19:30:03,401 : DEBUG : Reading Data spectra ... -2023-05-08 19:30:03,403 : DEBUG : Reading CRC32 ... -2023-05-08 19:30:03,416 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 19:30:03,417 : DEBUG : Reading Data spectra ... -2023-05-08 19:30:03,418 : DEBUG : Reading CRC32 ... -2023-05-08 19:30:03,431 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 19:30:03,432 : DEBUG : Reading Data spectra ... -2023-05-08 19:30:03,433 : DEBUG : Reading CRC32 ... -2023-05-08 19:30:03,450 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 19:30:03,450 : DEBUG : Reading Data spectra ... -2023-05-08 19:30:03,451 : DEBUG : Reading CRC32 ... -2023-05-08 19:30:03,465 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 19:30:03,465 : DEBUG : Reading Data spectra ... -2023-05-08 19:30:03,466 : DEBUG : Reading CRC32 ... -2023-05-08 19:30:03,491 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 19:30:03,492 : DEBUG : Reading Data spectra ... -2023-05-08 19:30:03,494 : DEBUG : Reading CRC32 ... -2023-05-08 19:30:03,636 : INFO : Done -2023-05-08 19:30:03,636 : INFO : Processing to L1a... -2023-05-08 19:30:04,407 : DEBUG : preprocessing done -2023-05-08 19:30:11,043 : DEBUG : preprocessing done -2023-05-08 19:30:14,405 : INFO : Done -2023-05-08 19:30:14,405 : INFO : Processing to L1b radiance... -2023-05-08 19:30:20,939 : INFO : Done -2023-05-08 19:30:20,939 : INFO : Processing to L1b irradiance... -2023-05-08 19:30:25,394 : INFO : Done -2023-05-08 19:30:25,706 : INFO : Processing to L1c... -2023-05-08 19:30:26,108 : DEBUG : WaterNetworkProtocol -2023-05-08 19:30:26,200 : DEBUG : 1681137226 -2023-05-08 19:30:26,207 : DEBUG : 1681137226 -2023-05-08 19:30:26,214 : DEBUG : 1681137226 -2023-05-08 19:30:26,221 : DEBUG : 1681137226 -2023-05-08 19:30:26,227 : DEBUG : 1681137226 -2023-05-08 19:30:26,234 : DEBUG : 1681137226 -2023-05-08 19:30:33,459 : INFO : interpolate sky radiance -2023-05-08 19:30:33,460 : INFO : interpolate irradiances -2023-05-08 19:30:40,871 : INFO : Apply Mobley 1999 -2023-05-08 19:30:41,343 : INFO : Processing Anomalies: ['x'] -2023-05-08 19:30:41,344 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/Test/SEQ20230410T143155Failed: AttributeError("'WaterNetworkProtocol' object has no attribute 'context'") -2023-05-08 19:30:41,345 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 139, in process_sequence - L1c = surf.process_l1c(L1c_int, L1a_irr) - File "/home/cgoyens/hypernets_processor/hypernets_processor/surface_reflectance/surface_reflectance.py", line 65, in process_l1c - dataset_l1c = l1ctol1b_function.propagate_ds_specific( - File "/usr/local/lib/python3.8/dist-packages/punpy/digital_effects_table/measurement_function.py", line 581, in propagate_ds_specific - y = self.check_sizes_and_run(*args, expand=expand, ds_out_pre=ds_out_pre) - File "/usr/local/lib/python3.8/dist-packages/punpy/digital_effects_table/measurement_function.py", line 821, in check_sizes_and_run - y = self.run(*args, expand=expand) - File "/usr/local/lib/python3.8/dist-packages/punpy/digital_effects_table/measurement_function.py", line 778, in run - return np.array(self.meas_function(*input_qty),dtype=self.prop.dtype) - File "/home/cgoyens/hypernets_processor/hypernets_processor/surface_reflectance/measurement_functions/water_network_protocol.py", line 43, in meas_function - w1 = self.context.get_config_value("similarity_w1") -AttributeError: 'WaterNetworkProtocol' object has no attribute 'context' - -2023-05-08 19:30:41,350 : INFO : Processing sequence: /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/Test/SEQ20230410T140203 -2023-05-08 19:30:41,541 : INFO : Reading raw data... -2023-05-08 19:30:41,550 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 19:30:41,550 : DEBUG : {'Total Dataset Length': 4131, 'Spectrum Type Information': 136, 'acquisition_time': 1681135485382, 'integration_time': 1024, 'temperature': 16.488752365112305, 'Pixel Count': 2048, 'acceleration_x_mean': -16303, 'acceleration_x_std': 131, 'acceleration_y_mean': -429, 'acceleration_y_std': 595, 'acceleration_z_mean': -29, 'acceleration_z_std': 643} -2023-05-08 19:30:41,555 : DEBUG : Wavelength range: 163.96437072753906 -1145.4518535429638 -2023-05-08 19:30:41,672 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 19:30:41,673 : DEBUG : Reading Data spectra ... -2023-05-08 19:30:41,674 : DEBUG : Reading CRC32 ... -2023-05-08 19:30:41,688 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 19:30:41,688 : DEBUG : Reading Data spectra ... -2023-05-08 19:30:41,690 : DEBUG : Reading CRC32 ... -2023-05-08 19:30:41,703 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 19:30:41,704 : DEBUG : Reading Data spectra ... -2023-05-08 19:30:41,705 : DEBUG : Reading CRC32 ... -2023-05-08 19:30:41,719 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 19:30:41,719 : DEBUG : Reading Data spectra ... -2023-05-08 19:30:41,720 : DEBUG : Reading CRC32 ... -2023-05-08 19:30:41,734 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 19:30:41,734 : DEBUG : Reading Data spectra ... -2023-05-08 19:30:41,736 : DEBUG : Reading CRC32 ... -2023-05-08 19:30:41,749 : DEBUG : Spectrum Type Info : VIS Irr -2023-05-08 19:30:41,750 : DEBUG : Reading Data spectra ... -2023-05-08 19:30:41,751 : DEBUG : Reading CRC32 ... -2023-05-08 19:30:41,852 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 19:30:41,852 : DEBUG : {'Total Dataset Length': 4131, 'Spectrum Type Information': 144, 'acquisition_time': 1681135438327, 'integration_time': 256, 'temperature': 16.039813995361328, 'Pixel Count': 2048, 'acceleration_x_mean': 12473, 'acceleration_x_std': 262, 'acceleration_y_mean': -10256, 'acceleration_y_std': 292, 'acceleration_z_mean': -2960, 'acceleration_z_std': 469} -2023-05-08 19:30:41,855 : DEBUG : Wavelength range: 163.85150146484375 -1145.61306723725 -2023-05-08 19:30:41,988 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 19:30:41,988 : DEBUG : Reading Data spectra ... -2023-05-08 19:30:41,990 : DEBUG : Reading CRC32 ... -2023-05-08 19:30:42,004 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 19:30:42,004 : DEBUG : Reading Data spectra ... -2023-05-08 19:30:42,005 : DEBUG : Reading CRC32 ... -2023-05-08 19:30:42,019 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 19:30:42,019 : DEBUG : Reading Data spectra ... -2023-05-08 19:30:42,020 : DEBUG : Reading CRC32 ... -2023-05-08 19:30:42,034 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 19:30:42,035 : DEBUG : Reading Data spectra ... -2023-05-08 19:30:42,036 : DEBUG : Reading CRC32 ... -2023-05-08 19:30:42,050 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 19:30:42,050 : DEBUG : Reading Data spectra ... -2023-05-08 19:30:42,051 : DEBUG : Reading CRC32 ... -2023-05-08 19:30:42,065 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 19:30:42,065 : DEBUG : Reading Data spectra ... -2023-05-08 19:30:42,066 : DEBUG : Reading CRC32 ... -2023-05-08 19:30:42,080 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 19:30:42,080 : DEBUG : Reading Data spectra ... -2023-05-08 19:30:42,082 : DEBUG : Reading CRC32 ... -2023-05-08 19:30:42,096 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 19:30:42,096 : DEBUG : Reading Data spectra ... -2023-05-08 19:30:42,097 : DEBUG : Reading CRC32 ... -2023-05-08 19:30:42,119 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 19:30:42,120 : DEBUG : Reading Data spectra ... -2023-05-08 19:30:42,121 : DEBUG : Reading CRC32 ... -2023-05-08 19:30:42,136 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 19:30:42,136 : DEBUG : Reading Data spectra ... -2023-05-08 19:30:42,137 : DEBUG : Reading CRC32 ... -2023-05-08 19:30:42,151 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 19:30:42,152 : DEBUG : Reading Data spectra ... -2023-05-08 19:30:42,153 : DEBUG : Reading CRC32 ... -2023-05-08 19:30:42,167 : DEBUG : Spectrum Type Info : VIS Rad -2023-05-08 19:30:42,167 : DEBUG : Reading Data spectra ... -2023-05-08 19:30:42,168 : DEBUG : Reading CRC32 ... -2023-05-08 19:30:42,324 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 19:30:42,324 : DEBUG : {'Total Dataset Length': 4131, 'Spectrum Type Information': 128, 'acquisition_time': 1681135412063, 'integration_time': 32, 'temperature': 15.654559135437012, 'Pixel Count': 2048, 'acceleration_x_mean': -12992, 'acceleration_x_std': 0, 'acceleration_y_mean': -9856, 'acceleration_y_std': 0, 'acceleration_z_mean': -2048, 'acceleration_z_std': 0} -2023-05-08 19:30:42,328 : DEBUG : Wavelength range: 163.85150146484375 -1145.61306723725 -2023-05-08 19:30:42,426 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 19:30:42,426 : DEBUG : Reading Data spectra ... -2023-05-08 19:30:42,427 : DEBUG : Reading CRC32 ... -2023-05-08 19:30:42,441 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 19:30:42,442 : DEBUG : Reading Data spectra ... -2023-05-08 19:30:42,443 : DEBUG : Reading CRC32 ... -2023-05-08 19:30:42,456 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 19:30:42,457 : DEBUG : Reading Data spectra ... -2023-05-08 19:30:42,458 : DEBUG : Reading CRC32 ... -2023-05-08 19:30:42,472 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 19:30:42,472 : DEBUG : Reading Data spectra ... -2023-05-08 19:30:42,473 : DEBUG : Reading CRC32 ... -2023-05-08 19:30:42,487 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 19:30:42,487 : DEBUG : Reading Data spectra ... -2023-05-08 19:30:42,489 : DEBUG : Reading CRC32 ... -2023-05-08 19:30:42,502 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 19:30:42,503 : DEBUG : Reading Data spectra ... -2023-05-08 19:30:42,504 : DEBUG : Reading CRC32 ... -2023-05-08 19:30:42,518 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 19:30:42,518 : DEBUG : Reading Data spectra ... -2023-05-08 19:30:42,519 : DEBUG : Reading CRC32 ... -2023-05-08 19:30:42,533 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 19:30:42,533 : DEBUG : Reading Data spectra ... -2023-05-08 19:30:42,534 : DEBUG : Reading CRC32 ... -2023-05-08 19:30:42,548 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 19:30:42,548 : DEBUG : Reading Data spectra ... -2023-05-08 19:30:42,549 : DEBUG : Reading CRC32 ... -2023-05-08 19:30:42,563 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 19:30:42,564 : DEBUG : Reading Data spectra ... -2023-05-08 19:30:42,565 : DEBUG : Reading CRC32 ... -2023-05-08 19:30:42,579 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 19:30:42,579 : DEBUG : Reading Data spectra ... -2023-05-08 19:30:42,580 : DEBUG : Reading CRC32 ... -2023-05-08 19:30:42,594 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 19:30:42,594 : DEBUG : Reading Data spectra ... -2023-05-08 19:30:42,595 : DEBUG : Reading CRC32 ... -2023-05-08 19:30:42,617 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 19:30:42,617 : DEBUG : Reading Data spectra ... -2023-05-08 19:30:42,620 : DEBUG : Reading CRC32 ... -2023-05-08 19:30:42,635 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 19:30:42,635 : DEBUG : Reading Data spectra ... -2023-05-08 19:30:42,637 : DEBUG : Reading CRC32 ... -2023-05-08 19:30:42,674 : DEBUG : Spectrum Type Info : VIS Dark -2023-05-08 19:30:42,675 : DEBUG : Reading Data spectra ... -2023-05-08 19:30:42,678 : DEBUG : Reading CRC32 ... -2023-05-08 19:30:42,867 : INFO : Done -2023-05-08 19:30:42,867 : INFO : Processing to L1a... -2023-05-08 19:30:43,650 : DEBUG : preprocessing done -2023-05-08 19:30:50,367 : DEBUG : preprocessing done -2023-05-08 19:30:53,782 : INFO : Done -2023-05-08 19:30:53,782 : INFO : Processing to L1b radiance... -2023-05-08 19:31:00,525 : INFO : Done -2023-05-08 19:31:00,525 : INFO : Processing to L1b irradiance... -2023-05-08 19:31:05,003 : INFO : Done -2023-05-08 19:31:05,321 : INFO : Processing to L1c... -2023-05-08 19:31:05,724 : DEBUG : WaterNetworkProtocol -2023-05-08 19:31:05,820 : DEBUG : 1681135437 -2023-05-08 19:31:05,827 : DEBUG : 1681135437 -2023-05-08 19:31:05,833 : DEBUG : 1681135437 -2023-05-08 19:31:05,840 : DEBUG : 1681135437 -2023-05-08 19:31:05,847 : DEBUG : 1681135437 -2023-05-08 19:31:05,853 : DEBUG : 1681135437 -2023-05-08 19:31:13,120 : INFO : interpolate sky radiance -2023-05-08 19:31:13,121 : INFO : interpolate irradiances -2023-05-08 19:31:20,576 : INFO : Apply Mobley 1999 -2023-05-08 19:31:21,063 : INFO : Processing Anomalies: ['x', 'x'] -2023-05-08 19:31:21,063 : ERROR : /home/cgoyens/waterhypernet/HYPSTAR/Raw/M1BE/Test/SEQ20230410T140203Failed: AttributeError("'WaterNetworkProtocol' object has no attribute 'context'") -2023-05-08 19:31:21,064 : INFO : Traceback (most recent call last): - File "/home/cgoyens/hypernets_processor/hypernets_processor/main/sequence_processor_main.py", line 91, in run_sequence - sp.process_sequence(target_sequence) - File "/home/cgoyens/hypernets_processor/hypernets_processor/sequence_processor.py", line 139, in process_sequence - L1c = surf.process_l1c(L1c_int, L1a_irr) - File "/home/cgoyens/hypernets_processor/hypernets_processor/surface_reflectance/surface_reflectance.py", line 65, in process_l1c - dataset_l1c = l1ctol1b_function.propagate_ds_specific( - File "/usr/local/lib/python3.8/dist-packages/punpy/digital_effects_table/measurement_function.py", line 581, in propagate_ds_specific - y = self.check_sizes_and_run(*args, expand=expand, ds_out_pre=ds_out_pre) - File "/usr/local/lib/python3.8/dist-packages/punpy/digital_effects_table/measurement_function.py", line 821, in check_sizes_and_run - y = self.run(*args, expand=expand) - File "/usr/local/lib/python3.8/dist-packages/punpy/digital_effects_table/measurement_function.py", line 778, in run - return np.array(self.meas_function(*input_qty),dtype=self.prop.dtype) - File "/home/cgoyens/hypernets_processor/hypernets_processor/surface_reflectance/measurement_functions/water_network_protocol.py", line 43, in meas_function - w1 = self.context.get_config_value("similarity_w1") -AttributeError: 'WaterNetworkProtocol' object has no attribute 'context' - diff --git a/hypernets_processor/test/test_functions.py b/hypernets_processor/test/test_functions.py index 46577bde..12a6c807 100755 --- a/hypernets_processor/test/test_functions.py +++ b/hypernets_processor/test/test_functions.py @@ -1,6 +1,7 @@ """ test_functions module - module with functions to help with testing """ + from hypernets_processor.version import __version__ from hypernets_processor.data_io.hypernets_db_builder import HypernetsDBBuilder from hypernets_processor.data_io.hypernets_ds_builder import HypernetsDSBuilder @@ -10,6 +11,7 @@ from hypernets_processor.utils.config import ( PROCESSOR_CONFIG_PATH, JOB_CONFIG_TEMPLATE_PATH, + PROCESSOR_LAND_DEFAULTS_CONFIG_PATH, ) from hypernets_processor.utils.paths import relative_path import datetime @@ -132,7 +134,7 @@ def setup_test_processor_config( :rtype: configparser.RawConfigParser """ - processor_config = read_config_file(PROCESSOR_CONFIG_PATH) + processor_config = read_config_file(PROCESSOR_LAND_DEFAULTS_CONFIG_PATH) processor_config["Processor"]["version"] = "0.0" diff --git a/hypernets_processor/utils/config.py b/hypernets_processor/utils/config.py index f2bfb748..406fd47e 100755 --- a/hypernets_processor/utils/config.py +++ b/hypernets_processor/utils/config.py @@ -1,6 +1,9 @@ """ Module of helper functions for configfile operations """ + +import warnings + import numpy as np from hypernets_processor.version import __version__ from hypernets_processor.utils.paths import relative_path @@ -16,6 +19,9 @@ __email__ = "sam.hunt@npl.co.uk" __status__ = "Development" +PROCESSOR_CONFIG_PATH = None +SCHEDULER_CONFIG_PATH = None +JOBS_FILE_PATH = None this_directory = os.path.dirname(__file__) etc_directory = os.path.join(os.path.dirname(this_directory), "etc") @@ -25,16 +31,16 @@ str(np.genfromtxt(WORKING_DIRECTORY_FILE_PATH, dtype=str)) ) - PROCESSOR_CONFIG_PATH = os.path.join(working_directory, "processor.config") - SCHEDULER_CONFIG_PATH = os.path.join(working_directory, "scheduler.config") - JOBS_FILE_PATH = os.path.join(working_directory, "jobs.txt") + if os.path.exists(working_directory): + PROCESSOR_CONFIG_PATH = os.path.join(working_directory, "processor.config") + SCHEDULER_CONFIG_PATH = os.path.join(working_directory, "scheduler.config") + JOBS_FILE_PATH = os.path.join(working_directory, "jobs.txt") else: - print( - "working_directory.txt does not exist so some of the paths for automated processing could not be correctly set up." + warnings.warn( + "working_directory.txt does not exist so some of the paths for automated processing could not " + "be correctly set up. You can ignore this warning if only processing data ad-hoc using hypernets_sequence_processor command." ) - PROCESSOR_CONFIG_PATH = None - SCHEDULER_CONFIG_PATH = None - JOBS_FILE_PATH = None + PROCESSOR_DEFAULT_CONFIG_PATH = os.path.join(etc_directory, "processor.config") PROCESSOR_LAND_DEFAULTS_CONFIG_PATH = os.path.join( diff --git a/hypernets_processor/utils/paths.py b/hypernets_processor/utils/paths.py index 39edc166..f1757385 100755 --- a/hypernets_processor/utils/paths.py +++ b/hypernets_processor/utils/paths.py @@ -1,6 +1,7 @@ """ Module of helper functions for file path operations """ + import warnings from hypernets_processor.version import __version__ @@ -42,7 +43,7 @@ def relative_path(path, directory): return full_path -def parse_sequence_path(path): +def parse_sequence_path(path, context=None): """ Unpacks info from raw sequence path, e.g. some/directory/SEQ20200312T135926 @@ -60,7 +61,16 @@ def parse_sequence_path(path): date = dt.strptime(date_str, "%Y%m%dT%H%M%S") return {"datetime": date} except: - warnings.warn("The following path does not match the expected format and will not be processed (%s)"%(path)) + if context is not None: + context.logger.error( + "The following path does not match the expected format and will not be processed (%s)" + % (path) + ) + else: + warnings.warn( + "The following path does not match the expected format and will not be processed (%s)" + % (path) + ) return None diff --git a/hypernets_processor/utils/utils.py b/hypernets_processor/utils/utils.py new file mode 100644 index 00000000..581443e0 --- /dev/null +++ b/hypernets_processor/utils/utils.py @@ -0,0 +1,48 @@ +import datetime +import numpy as np +from typing import Optional, Union, List, Any +from dateutil.parser import parse + + +def convert_datetime( + date_time: Union[datetime.datetime, datetime.date, str, float, int, np.ndarray], +) -> datetime.datetime: + """ + Convert input datetimes to a datetime object + + :param date_time: date time to convert to a datetime object + :return: datetime object corresponding to input date_time + """ + if isinstance(date_time, np.ndarray): + return np.array([convert_datetime(date_time_i) for date_time_i in date_time]) + elif isinstance(date_time, datetime.datetime): + return date_time + elif isinstance(date_time, datetime.date): + return datetime.datetime.combine( + date_time, datetime.time(), tzinfo=datetime.timezone.utc + ) + elif isinstance(date_time, np.datetime64): + unix_epoch = np.datetime64(0, "s") + one_second = np.timedelta64(1, "s") + seconds_since_epoch = (date_time - unix_epoch) / one_second + return datetime.datetime.fromtimestamp( + seconds_since_epoch, datetime.timezone.utc + ) + elif isinstance(date_time, (float, int, np.integer)): + return datetime.datetime.fromtimestamp(date_time, datetime.timezone.utc) + elif isinstance(date_time, str) and date_time.lower() == "present": + return datetime.datetime.now(datetime.timezone.utc) + else: + if isinstance(date_time, str) and date_time[-1] == "Z": + date_time = date_time[:-1] + try: + dt = parse(date_time, fuzzy=False) + if dt.tzinfo is None: + dt = dt.replace(tzinfo=datetime.timezone.utc) + return dt + except ValueError: + raise ValueError( + "Unable to discern datetime requested: '{}' ({})".format( + date_time, type(date_time) + ) + ) diff --git a/hypernets_processor/version.py b/hypernets_processor/version.py index 159d48b8..55e47090 100755 --- a/hypernets_processor/version.py +++ b/hypernets_processor/version.py @@ -1 +1 @@ -__version__ = "2.0.1" +__version__ = "2.3.0" diff --git a/setup.py b/setup.py index 66d2eaf1..5ae7baba 100755 --- a/setup.py +++ b/setup.py @@ -102,12 +102,13 @@ def run(self): os.path.join("etc", "job_template.config"), os.path.join("etc", "processor_land_defaults.config"), os.path.join("etc", "processor_water_defaults.config"), - "calibration/calibration_files/*/*/*/*"], + "calibration/calibration_files/*/*/*/*", + ], }, - setup_requires=['numpy'], + setup_requires=["numpy"], install_requires=[ "alembic==1.10.2", - "numpy", + "numpy>=2.3.1", "netCDF4", "schedule", "matplotlib", @@ -115,34 +116,33 @@ def run(self): "dataset", "sqlalchemy==1.3.20", "sqlalchemy-utils", - "punpy>=1.0.0", + "punpy>=1.0.7", "matheo", - "comet_maths>=1.0.0", - "obsarray>=1.0.0", "freezegun", "importlib-metadata>=4.0.1", "pyhdf", - ], + "cftime>=1.6.4", + ], extras_require={ - "docs": [ - "sphinx", - "sphinx_design", - "sphinx_book_theme", - "ipython", - "sphinx_autosummary_accessors", - "sphinx-rtd-theme" + "docs": [ + "sphinx", + "sphinx_design", + "sphinx_book_theme", + "ipython", + "sphinx_autosummary_accessors", + "sphinx-rtd-theme", ], "dev": [ - "pytest >=5.4.1", - "pytest-cov >=2.8.1", - ] + "pytest >=5.4.1", + "pytest-cov >=2.8.1", + ], }, entry_points={ "console_scripts": [ "hypernets_sequence_processor = hypernets_processor.cli.sequence_processor_cli:cli", "hypernets_scheduler = hypernets_processor.cli.scheduler_cli:cli", "hypernets_processor_setup = hypernets_processor.cli.setup_processor_cli:cli", - "hypernets_processor_job_init = hypernets_processor.cli.init_job_cli:cli" + "hypernets_processor_job_init = hypernets_processor.cli.init_job_cli:cli", ], }, )