From cea7f8e105054b3608d76bd4b80c35131445b61c Mon Sep 17 00:00:00 2001 From: Robert Jackson Date: Thu, 23 Apr 2026 14:11:27 -0500 Subject: [PATCH] ADD: Integration tests for radar preprocessing with baseline image comparison - Fix sweep_mode parsing to handle CF/Radial 'azimuth_surveillance' mode (null-byte-delimited string) alongside 'ppi' and 'sector' - Fix ref_min/ref_max computation to use np.nanmin/nanmax and a local variable instead of reassigning the min_ref parameter (which caused NaN) - Add numpy import to radar_preprocessing.py - Add open-radar-data to dev dependencies - Add tests/integration/ with conftest that removes unit-test mocks so real xradar/cmweather are used - Add baseline PNGs in tests/data/baseline/preprocessing/ for the three GUC XPRECIP CMAC PPI files from open-radar-data Co-Authored-By: Claude Sonnet 4.6 --- lars/preprocessing/radar_preprocessing.py | 21 ++- pyproject.toml | 2 +- ...recipradarcmacppiS2.c1.20220314.021559.png | Bin 0 -> 2543 bytes ...recipradarcmacppiS2.c1.20220314.024239.png | Bin 0 -> 3186 bytes ...recipradarcmacppiS2.c1.20220314.025840.png | Bin 0 -> 2659 bytes tests/integration/conftest.py | 25 +++ tests/integration/test_radar_preprocessing.py | 144 ++++++++++++++++++ 7 files changed, 183 insertions(+), 9 deletions(-) create mode 100644 tests/data/baseline/preprocessing/gucxprecipradarcmacppiS2.c1.20220314.021559.png create mode 100644 tests/data/baseline/preprocessing/gucxprecipradarcmacppiS2.c1.20220314.024239.png create mode 100644 tests/data/baseline/preprocessing/gucxprecipradarcmacppiS2.c1.20220314.025840.png create mode 100644 tests/integration/conftest.py create mode 100644 tests/integration/test_radar_preprocessing.py diff --git a/lars/preprocessing/radar_preprocessing.py b/lars/preprocessing/radar_preprocessing.py index 5e3e5b8..af167a5 100644 --- a/lars/preprocessing/radar_preprocessing.py +++ b/lars/preprocessing/radar_preprocessing.py @@ -1,9 +1,10 @@ import xradar as xd import matplotlib.pyplot as plt import glob +import numpy as np import os import pandas as pd -import cmweather # noqa +import cmweather # noqa def preprocess_radar_data(file_path, output_path, date=None, @@ -66,7 +67,8 @@ def preprocess_radar_data(file_path, output_path, date=None, radar = radar.xradar.georeference() if 'sweep_0' in radar: sweep = radar['sweep_0'] - if sweep["sweep_mode"] == 'ppi' or sweep["sweep_mode"] == 'sector': + sweep_mode = str(sweep["sweep_mode"].values).split('\x00')[0].strip() + if sweep_mode in ('ppi', 'sector', 'azimuth_surveillance'): fig = plt.figure(figsize=(size_px/dpi, size_px/dpi)) ax = plt.axes() sweep["corrected_reflectivity"].where( @@ -74,11 +76,14 @@ def preprocess_radar_data(file_path, output_path, date=None, ax=ax, add_colorbar=False, **kwargs) - min_ref = sweep["corrected_reflectivity"].where( - sweep["corrected_reflectivity"] > min_ref).values.min() - max_ref = sweep["corrected_reflectivity"].where( - sweep["corrected_reflectivity"] > min_ref).values.max() - + masked = sweep["corrected_reflectivity"].where( + sweep["corrected_reflectivity"] > min_ref).values + ref_min = np.nanmin(masked) + ref_max = np.nanmax(masked) + ax.axis('off') + ax.set_title('') + ax.set_ylabel('') + ax.set_xlabel('') ax.set_xlim(x_bounds) ax.set_ylim(y_bounds) @@ -91,7 +96,7 @@ def preprocess_radar_data(file_path, output_path, date=None, os.path.basename(file).replace('.nc', '.png')), dpi=dpi, bbox_inches='tight', pad_inches=0) plt.close(fig) - out_df.loc[len(out_df)] = [file_name, time_str, label, min_ref, max_ref] + out_df.loc[len(out_df)] = [file_name, time_str, label, ref_min, ref_max] else: print(f"Sweep mode is not PPI or sector scan in {file}, skipping.") diff --git a/pyproject.toml b/pyproject.toml index be50e70..21fc9fe 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,7 +25,7 @@ dependencies = ["xradar", "scikit-learn", "python-dotenv", "aiohttp", "asksagecl [project.optional-dependencies] dev = ["pytest>=6.0", "pytest-asyncio>=0.21", "black", "flake8", "openai", "xradar", "python-dotenv", "scikit-learn", "cmweather", "torchvision", "torch", "aiohttp", "matplotlib", "pandas", - "asksageclient", "pip_system_certs", "requests"] + "asksageclient", "pip_system_certs", "requests", "open-radar-data"] [project.urls] Homepage = "https://github.com/rcjackson/lars" diff --git a/tests/data/baseline/preprocessing/gucxprecipradarcmacppiS2.c1.20220314.021559.png b/tests/data/baseline/preprocessing/gucxprecipradarcmacppiS2.c1.20220314.021559.png new file mode 100644 index 0000000000000000000000000000000000000000..1f26e195afaf88b8379bd60ad43b9b80db69dd8c GIT binary patch literal 2543 zcmd^B`#%#3AC~hXLsHaCKB(ygq(%kLhx;aNfF6m-naU5(jR#VD# zt_fLZoLaLmbKGq^o0H2XYdZSn{Ttr*^ZxKW&kx`4=lgwrd!9QW-=9&xT22N{s117<_E+iei#`NUDs?(L>*`FQ(@G-Y>;451=zRtx7PB zj$^~MM?M9wiCdAZS;q-PQayxtI5!$te858?u0U6Rz`IJG9p(7pASf3waM0%q_L#Dw z*%6<9%o|x+(z^kxGpF9F%g(Wmw7o*X;siTT2zs`ZQ~eo*qXUU8f^4Y4_=qD5wI-Zi zYfTqzNKwL61B^?puFT5sNFi0m$va2S&9HyJ`98M-MKc%g9JNT<;)937Z&%3Zzy8PftB!~NYdEGinW8Qx%>y*BDw!I>lLLjFV z+R#wJWx2eO59kQOEvg&#E@r7HZ(I9oO=Ogv>aGKxAnr}#@$b)>1Z_1CU1%Dc)cn5f zT5;xE?P>4Q!z+0eg^N3qsjmy!FAk{0zUxAi4CfZI==V*_S!uCO`qi0iDLIjY*RXcj zaYVau`zyHS3+#h|YH9j@6vDM&Y12&geD$L1ywPo0&)xb!Y|nLC>D^Tz9K3tgIlJh{ zDv?^l(nnt3lYyo#ZBW_Tl!?M8vA;Qqy))>UJ(q&9kSjU;6KsI2=TjzMwyF)^v;GM{=h2(m z?MUydso-GiuVAO5qZ~@Sz0RM0X1QkPk9lFTX+!64kuFg_r>b_`OR?>1iRjBxQf-R$ z&!-*w(UWjeGc=|1P|d~PT0T{Mfhbm!l#Qy!j36h1kS%NoR9gyktDGv^-DErPwzC#L zjO*bm^DBy(@Z*6JM_Gi^&Oi<5NV>!`SvHX6jjRs2^V z$#mYW`Z*``Licf2RPSqtXx*SXT%L(RU96F1BL^Ak=g=1!?sDTOTxvZt)5FP2Sb1J>78d0ITeNdT0yT$QrWE_d<1V zjg)>tvn4-%_rO5)-TgRPMTB>9@gRF+lrQ zZA8S-W2*Wcy!wH~mnpj)A{BX!O%HS@)ajffh1ul_J*TSW_I;)DkXwr;#U#kemc$Ng z?lN%tw>$KaqruntTsu2nFm$A&?nXV&Rt>3-oh@fYS>c@w8+lv{;71BK^s0vn=mY^ zJ`VmWyliCTbHXSH-+L`nLIZ2*u=?jFX%Y*&tWi8pbQ}EO@s7@l+{?LEg$q?%+#zXD zhKPU=q;1M0XPbi1%s*NhjbOhk7klR+zC5GT7h7l332kiFG9|uy$$19uY(0AK+;|70 z!{uf`2{p1XkAZvmrUvjFm3U=Nh{BHX{8K{)AYtjwLHSCzK88>pS!c{#FOT2t8J^cu zkw%v$gzOY7dG*_dM;X`QrqYEIMQ0|C1~9_(r^KBdG-G*qT&?=3JUq#!L9QIO+9OIg z)*8JzNNnbIjnsf3A0Ctb<~Lg7TH}{gc9sVNC09Sq#ApcCIWcSC?5&FqZRv3HYqDc9 z`hjZcWR33e(^&7@mTFiLB$V5h$#QSh-x6*L@!+?Lgv`>8W-rcDmi=ks?sz&YI9~Nq zgQ;oPNDr&*fJCFIa!JTP_Oamxi*k0^tN)lT@nqDDO@Qv^FwkZ&o4E0iKJeE&a)9 zA=Oq=fx=LhqRn@&!9asbkCNgolDPkD(_bh2`Cfxfv!C0#Vn$|2i$0e(MJ&?AQzZ>Q z(pJ9g2it2{+f}4yGO|qe!5}-?nUQtISVCn@Ny(CBkTSy<<|_M= zB~pZh5yQwjmTQKlp)rQ*{__0`-}61+5AS)O^PKmb^PHc~`_LM03=jc|@bU2hOihds zhdlZ(gai(wjG)rwA%W2_N3?B-9~$c!?#pNCi4F}4K?enTos0Ahj|dD2zO1hKr}|aZ za{*{{Xarb8LKC&8eJ}!p~ES{p(aide02ghlCOumL}{i*`gT!88`Hu; zMN7E8t=VbW;iwO?efi(E?n`DVUHDDS=D0!fl^ckFm*`Q2Qi(kLeGWmHy2Y(Aa*ZiO(|(hEMto@2ku zD|HHK9=VzJe_AqM?1n_DjpJ=DyfZ6DR8kTbh9hP$dX%xb*%ex9@FICrLr7EYhWAdAfgQYvUGF%ESU;hQYCGlS z+&x0TdRwF7uhhJ2tRtQnREhhuv}da*eePcOgT*T~d-T2;l-ih>qHPB6JwRN= zS{3!{tjwZ@TGH4Hpsh<=g{9gBOpgkfpiLL!6w3mLy6!gEvaWTJMVpjQz}&<{k-2;#}(3TWoGvu+2ztoaWS3>rCUYA$@Q1qzn-ov z`#_H?P}!|xt-9DZGi7(K(&lF6T7HV^?yfM-0zGUf=x|RXZeJC{t&K;8sMpLD?{CQC zfhzFFM7CoWY;o{QIbvZAo~;Fz9{c9{4feC4p@=)V?Ffsr)yIvvex*`>ILoS4(jBlX z^Mx_)Y-=)G$IvXP`~yY4J&+l;ql+IwrM^IFMKlKViVRW&wIDPS<%ywl2}l{34Zov` zwD)!=D2Wz);%aBH1AR#qIkj0+UNGNB(B$|Xs))L3!7s)80iQFx=B)LTqukUY0p!kO zMpAC9VyeAsyIB_kHj-boP0m!&v=1{+^K#kM)^s7*7Q6vhSCUFQ$}$Gh`%i*mkXOst z8KOsEiBi}?ET_<_20?%2-S2mi-7c4a41>@>!pHZfwmk9zxq7lp0fDjZk{SBR;TNRd>(`4(Dn_HL zukqF#KMwwl8_mBJo9Jq1+_8vG>i0Ja3n`h1y9~S9}dr zz=NiEk6i48%z5jH{aQ&{Z?0E_QeRxcMTm_1YG2Qy-A~Vw=!j%LLBie`Ru$x- zEZ@BD6H8^A98~zReA$-CKMiJw$9_faCQUX!91p~`Bn-Ljoaz<0`frZ0w$vD5v#d#7=PS9k%9qKC#`_M>LKByVq2_Ad*8-y)XPXbE zkWAn{-@rqn>iAeSzTFTw{bh~vE!sR0&p)8W-gCx~RwwqKK{49D?4(Ie68$nc!1Y4* zV&|iDfjDq30QqiZbV2T2PrK}o=_dbF#PbWm48^EFV!xg+i#_`@#l9 zO)uj8O`n^w!P6u|<6AAgM0XRpZ}qkz!Jm-SslaL!C;-f~9Udu7@jKeT`v|PX&T{sa zDqrhoWMaF&b7ogYIz>_SWjYl%t`dKl8-k}U%iWC$ixp3Ze7j2<8jGryu+5>+ zc>7O3w_Y#NWl;K~Q;`_G)lm8+OJ6T!W{$QdbK%0ssTj648{ZCvbU~heR1O| zi0m%qo`}9E(ohuBIVkkO??(X?T2cE6S{XoUa<0|uD2+p) z)ipjwC0ySdShsK-jWfg5(tBfpS=H3$CA^Ad_Cr|P%*MMmt4kq~nXTN*h4s9zE+?x| zUpH4+Q|#yDTi!dHz~A$U=ZY(vMu&8p|H{rxSqw2np@T}mg7lg6P1=6&#@Wh=Vtt&)_36z)sp)2mrYdA{*dVh|EI(l#%&Gg#lJ!TfJquEOgYfVJn_{Wx_6!>rplK?q@ z$N6iiVA>Yp$p<_mqc@(O`l`s(XMw_Z-f@I+t$`i!ap$Us2nuU%9s~{7ZBlSc$Sc`{ zyg_optc#p|0Eo2hF9s?KqJ+9w=(VlxNzy(KN-v)j8%NRGndC0oy~-`%mRJW~2gkTc z^l}W&-hzvrnS6#oyWDJ-0oE-yN6$3l_pMSES2-&XcvJjFL&$o8)hQ46t)~{m;46E^ zB|%ARo<78^!N%)&imgEQAnatuVs=xb>11vaw~wKA7uUX-206|v#wIvAw6FW&c+F|h zFp4o8Sx8%a?;2qNj519Oaa(@?+xYGIOYFOi(Y3YsDuG5HDG@q5aX{}oT&*Oy23vCJ zB$Rn?$nh4Mt_sK`Os|(#doaQz1&gsI2TVUEAeycjwhOx>VJj6qx?t=7WSvwa4Dj0n zUj0h!4r&MQocU5~iPA1nRaY1P0hO6=6)&3pm{bn4u>X#0%L~bOIG-CfF;ISQp8Fhm ztssnG+IC=!ZF=A8P&3=ge3tA4*<12LwwV<-?R(Yl4JSdeSx~yVBPn1uC8>2IM?P9; zm3;K@5#l=wA%d8LSf@|CiN`O9e`mFJklml!wltGDJ}E&;$NA9ibOL j&GGmA|4|d_v(R{l(Iqhc+1&8Z{_~l_;6^nDw;%iiTOa6! literal 0 HcmV?d00001 diff --git a/tests/data/baseline/preprocessing/gucxprecipradarcmacppiS2.c1.20220314.025840.png b/tests/data/baseline/preprocessing/gucxprecipradarcmacppiS2.c1.20220314.025840.png new file mode 100644 index 0000000000000000000000000000000000000000..45f11dc281c821113544265766663572e01e600d GIT binary patch literal 2659 zcmd^BX*U~+7NrBKT)joLYHGEHAgYv>tF&fCil|xC6!RD&hEiTp6g6C-npDM9(<@P8 zD4uz)A%-AG4J9N^gAl@X--q`X-g@hO*k^w@XPv#)Icx26UOzQ6;N=$PW@BUHH8Rw- zIOU1I!NqYJuX9Sxo{~CD&jw~0-~od_4|ZoWeGUus4S@N2xk!Y$2ZwkC_$vdI?f~!0 zOL)Ryfg$ROihlp800sm@6+7%hM^3w32sE?}VPg}z{5RO4=pL_LD|e-g_K>Zfwkx zP3q}S(viG5Ak~-Q-+xz`KbJQ<`RpJY zGF!2rhiK~hp(aT6-7t4Mbd9wYz#fZa(@8an{oUnE>Lq^Xe<0Z;yUty`A=S>M#IB!o z;m+g#tJM3VE3Nw$g>d%Hf6rpX992SiOg)o8bE})(`pL;26_IM$y z)6leQ0z-oV$GU0{9cnU(*pIIp4{hKRk#_Z4$d*+>CW}es^>04vjd^w-+!Nr#XsT!NpBjlES2@A~Mg~HPbv4@>#Ku z3oVLEpH)bcuFaRU);UC5Tk#Ux{Q8xJ8~$+kpxG1HrJ{dJ{ozzX6?}8eN6g$5@6N3^ zfM3i5>p>|8Dc*CtlW}+*Cpy|kN={(Z^z5kHZu0cJwh|M$CWxf>Qe=QtW1sMIRs!-l zi$Y-aAJ%0Wg~u7A(JvsK;9O{KQHRka67YZd5Odi0LvS4nkFB<}8YPYy}5iECZ}y#rh`n zg7D|*$X`Pj`tm+TPcqZeOjDaAE#>zZdDs*baAZ2xqq7OzZLZO9JQ?srY3;sxc`ER_ zQ0ww|bEFeKcC@HoaUL^nSVbD94r*oK4V1kn1Fdwob9c(g;W?j@vnb@aH#y1C2mFl1 z@1-i_gGDE`ebi#acUzYChS;lU+@QR4t~54udUW?_E$7pN{BH>m#jCY^t*y87>28#n zi(yISY-R~d4!{*pl(^L)X#)o_bSwpE(E#sx$|^rPzGkN`eaXXu78?U{)37FDQHtxq zMV!^CWW)K=KgH0(&6?`p%t@%7&`L;5XA&B{b+Llt?DTN+{OLlY*vWCdoA=z|{)pzV zM_EkQ;^f5ECYGEjg{y8enb_f%qV~nBoodcnm~Ldef0fY9Cu$Y#^Yp}*QSo`OqC_WD zee4hOIWbg754w$t zjrv_CwoMrsIs8=`e@z3EwrQEd1yc02zJ(lfUzIJsD^@R70R~(do75CNfuo(6t5M{1 zM2BR_AG#_>$W;tkd|1I%ykbWV=2!=7(alRxe{aa#N$iG~wa8T2{F>yt>ou&B<3G2J zn%TB>DmOi>BTQYku>V&n{GE^Bf_-&nzN;OBG+sw&%UZ8=j#uQof~>prbZ*n!SHCpU zL_qe97rjbuH~ziPLJltNMY`C*ruV|}n(}69;x3_<`lG66u}rYAc-N-JmLf zJoWv`jpH7Q{`jqrn&UCzIl2EOTf@6BuyG2bUGOAcr9<&{obmZ$=R4)KL5@urd3bkx zGDasTy(DdL!H;QC zVep|0^vhiUK%Lr=yFM@LWDU>LxM?cW5BM`Xje*#mC(grj{E_vLwnK)w--Aovq-m9V z@bg&j2!i8w^HB;fz{m|tDk(iE0!ox15Nnmh2SQjg!tYs%%ht864vEd3l+PAch*E$K zj=zTgrmdBLyGo=*dSnc!c~SEP_hGU0G3~)HN&vj5ZN~{ZH*eKmopPD5&t%Y9L4m?o zBzlkt@|Qt)i(!_1L@1^@yzAg5l(sxR@PX8iK@qP{E}%ghUzx^wYN7m=FN20uIEG^O z(z(TfL7nAQJ~;uA#^ljE0}dM71{A+1y><)1L2btb;in&QX7~;h6hpXEX;_X8nh$S+ zfowGKIG>*)P-Vw=4mdk{39(2avbXsOX|%Q01-vH+P4^&wXgM~3TK`lr`k r^1plgC8V5vpS%5!z91QIPR