diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f2c2cbd..dab920f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,24 +1,36 @@ repos: # https://pycqa.github.io/isort/docs/configuration/black_compatibility.html#integration-with-pre-commit - repo: https://github.com/pycqa/isort - rev: 5.12.0 + rev: 5.13.2 hooks: - id: isort args: ["--profile", "black", "--filter-files"] - repo: https://github.com/psf/black - rev: 23.7.0 + rev: 24.3.0 hooks: - id: black args: ["--line-length=100"] # https://black.readthedocs.io/en/stable/guides/using_black_with_other_tools.html?highlight=other%20tools#flake8 - repo: https://github.com/PyCQA/flake8 - rev: 6.1.0 + rev: 7.0.0 hooks: - id: flake8 args: ["--max-line-length=100", "--extend-ignore=E203,E712"] # https://github.com/pre-commit/pre-commit-hooks - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 + rev: v4.6.0 hooks: - id: trailing-whitespace - id: end-of-file-fixer + # codespell + - repo: https://github.com/codespell-project/codespell + rev: v2.2.6 + hooks: + - id: codespell + args: [ + "--skip=*.json,*.bib,*.txt,*.cls,scripts/environment_vtscat_save.yml", + "--ignore-words-list=livetime", + ] + additional_dependencies: + - tomli + diff --git a/analysis/CTA.DISPTRAINING.qsub_analyse.sh b/analysis/CTA.DISPTRAINING.qsub_analyse.sh index 31be991..09af75d 100755 --- a/analysis/CTA.DISPTRAINING.qsub_analyse.sh +++ b/analysis/CTA.DISPTRAINING.qsub_analyse.sh @@ -1,12 +1,10 @@ #!/bin/bash # -# train disp for CTA -# -# called from CTA.DISPTRAINING_sub_analyse.sh -# +# train dispBDTs for CTA # -ODIR=OFILE +ODIR=OODIR +DDIR=DDDIR RECID=RECONSTRUCTIONID TTYPE=TELTYPE BDT=MLPTYPE @@ -17,15 +15,48 @@ ARRAY=AAA QC="QQQQ" # set the right observatory (environmental variables) -source $EVNDISPSYS/setObservatory.sh CTA +if [ ! -n "$EVNDISP_APPTAINER" ]; then + source "${EVNDISPSYS}"/setObservatory.sh CTA +fi -# output data files are written to this directory -mkdir -p $ODIR +# temporary (scratch) directory +if [[ -n $TMPDIR ]]; then + TEMPDIR=$TMPDIR/$RUN +else + TEMPDIR="$CTA_USER_DATA_DIR/TMPDIR" +fi +echo "Scratch dir: $TEMPDIR" +mkdir -p "$TEMPDIR" +# output data files are written to this directory +# (note that $OUTPUTDIR is pointing inside the +# apptainer to a mounted directory, while $ODIR +# is not changed) +OUTPUTDIR="${ODIR}" +mkdir -p $OUTPUTDIR # delete old log files -rm -f $ODIR/${BDT}-${TTYPE}.training.log +rm -f $OUTPUTDIR/${BDT}-${TTYPE}.training.log # delete old training files -rm -f $ODIR/*${TTYPE}* +rm -f $OUTPUTDIR/*${TTYPE}* + +# data data directory to LIST +cp -v "$TLIST" "$TEMPDIR" + +if [ -n "$EVNDISP_APPTAINER" ]; then + APPTAINER_MOUNT=" --bind ${OUTPUTDIR}:/eventdisplay_datadir/output/ " + APPTAINER_MOUNT+=" --bind ${DDIR}:/eventdisplay_datadir/data/ " + APPTAINER_MOUNT+=" --bind ${TEMPDIR}:/eventdisplay_datadir/tmp/ " + echo "APPTAINER MOUNT: ${APPTAINER_MOUNT}" + APPTAINER_ENV="--env OUTPUTDIR=/eventdisplay_datadir/data/,TEMPDIR=/eventdisplay_datadir/tmp/" + EVNDISPSYS="${EVNDISPSYS/--cleanenv/--cleanenv $APPTAINER_ENV $APPTAINER_MOUNT}" + echo "APPTAINER SYS: $EVNDISPSYS" + OUTPUTDIR="/eventdisplay_datadir/output/" + DDIR="/eventdisplay_datadir/data" + sed -i "s|^|${DDIR}/|" "$TEMPDIR/$(basename $TLIST)" + TEMPDIR="/eventdisplay_datadir/tmp/" +else + sed -i "s|^|${DDIR}/|" "$TEMPDIR/$(basename $TLIST)" +fi # array layout file @@ -51,15 +82,15 @@ elif [[ $DSET == *"prod5"* ]]; then elif [[ $DSET == *"prod6"* ]]; then ADIR=$CTA_EVNDISP_AUX_DIR/DetectorGeometry/CTA.prod6${DARR}.lis else - echo "Unknown data set: $DSET" - echo "exiting..." + echo "Unknown data set: $DSET, exiting" exit fi -######################################### +echo $OUTPUTDIR + # train TMVA -$EVNDISPSYS/bin/trainTMVAforAngularReconstruction $TLIST \ - $ODIR \ +$EVNDISPSYS/bin/trainTMVAforAngularReconstruction "$TEMPDIR/$(basename $TLIST)" \ + "$OUTPUTDIR" \ 0.8 \ ${RECID} \ ${TTYPE} \ @@ -68,10 +99,9 @@ $EVNDISPSYS/bin/trainTMVAforAngularReconstruction $TLIST \ ${ADIR} \ "" \ ${QC} \ - 0 > $ODIR/${BDT}-${TTYPE}.training.log 2>&1 -######################################### + 0 > # $ODIR/${BDT}-${TTYPE}.training.log 2>&1 + -############## # cleanup # remove everything if telescope type is not found if [[ -e $ODIR/${BDT}-${TTYPE}.training.log ]]; then @@ -79,8 +109,6 @@ if [[ -e $ODIR/${BDT}-${TTYPE}.training.log ]]; then then echo "No telescopes found of type ${TTYPE}" head -n 10 ${TLIST} - # rm -f -v $ODIR/${BDT}"_"${TTYPE}.root - # rm -f $ODIR/${BDT}-${TTYPE}.training.log fi fi # move everything into root files @@ -92,12 +120,6 @@ if [[ -e $ODIR/${BDT}_MLP_${TTYPE}.weights.xml ]]; then $EVNDISPSYS/bin/logFile dispXML-MLP-${TTYPE} $ODIR/${BDT}-${TTYPE}.disptmva.root $ODIR/${BDT}_MLP_${TTYPE}.weights.xml rm -f $ODIR/${BDT}_MLP_${TTYPE}.weights.xml fi -if [[ -e $ODIR/${BDT}-${TTYPE}.training.log ]]; then +if [[ -e $ODIR/${BDT}-${TTYPE}.training.log ]] && [[ -e $ODIR/${BDT}-${TTYPE}.disptmva.root ]]; then $EVNDISPSYS/bin/logFile dispLog-${TTYPE} $ODIR/${BDT}-${TTYPE}.disptmva.root $ODIR/${BDT}-${TTYPE}.training.log -# rm -f $ODIR/${BDT}-${TTYPE}.training.log fi - -#rm -f $ODIR/${BDT}_${TTYPE}.root -#rm -f $ODIR/${BDT}_${TTYPE}.tmva.root - -exit diff --git a/analysis/CTA.DISPTRAINING.sub_analyse.sh b/analysis/CTA.DISPTRAINING.sub_analyse.sh index b9e1768..1bcfbfc 100755 --- a/analysis/CTA.DISPTRAINING.sub_analyse.sh +++ b/analysis/CTA.DISPTRAINING.sub_analyse.sh @@ -34,7 +34,7 @@ then echo " file name of list of TMVA parameter file" echo " layout scaling (e.g. 5); give 99 to ignore scaling" echo - echo " (note 1: hardwired telescope types in this script)" + echo " (note 1: hardwired telescope types in this script)" echo " (note 2: disp core training switched off)" echo exit @@ -62,8 +62,8 @@ if [ -n $8 ] then QSUBOPT="$8" fi -QSUBOPT=${QSUBOPT//_X_/ } -QSUBOPT=${QSUBOPT//_M_/-} +QSUBOPT=${QSUBOPT//_X_/ } +QSUBOPT=${QSUBOPT//_M_/-} ######################################### # TMVA options @@ -153,7 +153,7 @@ else fi ######################################### -# +# ######################################### for MLP in "${MLPLIST[@]}" do @@ -190,7 +190,8 @@ do #################### # input file list rm -f $SHELLDIR/tempList.list - find $CTA_USER_DATA_DIR/analysis/AnalysisData/$DSET/$ARRAY/${EVNDISP}/gamma_cone/ -name "*[_,.]${MCAZ}*.root" > $SHELLDIR/tempList.list + DATADIR="$CTA_USER_DATA_DIR/analysis/AnalysisData/$DSET/$ARRAY/${EVNDISP}/gamma_cone/" + find ${DATADIR} -name "*[_,.]${MCAZ}*.root" -printf "%f\n" > $SHELLDIR/tempList.list NFIL=`wc -l $SHELLDIR/tempList.list | awk '{print $1}'` echo "Total number of files available: $NFIL" # only use NN% of all evndisp files for training @@ -226,7 +227,8 @@ do FNAM="$SHELLDIR/EDISP-$ARRAY-$SCALING-$MCAZ-$TELTYPE-$MLP-$NSTEP" cp $FSCRIPT.sh $FNAM.sh - sed -i -e "s|OFILE|$TDIR|" \ + sed -i -e "s|OODIR|$TDIR|" \ + -e "s|DDDIR|$DATADIR|" \ -e "s|TELTYPE|$TELTYPE|" \ -e "s|MLPTYPE|$MLP|" \ -e "s|RECONSTRUCTIONID|$RECID|" \ diff --git a/analysis/CTA.EFFAREA.qsub_analyse_list.sh b/analysis/CTA.EFFAREA.qsub_analyse_list.sh index 0e811b1..9526a08 100644 --- a/analysis/CTA.EFFAREA.qsub_analyse_list.sh +++ b/analysis/CTA.EFFAREA.qsub_analyse_list.sh @@ -2,16 +2,16 @@ # # calculate effective areas and instrument response functions for CTA # -# -# -############################################################################## echo echo "calculating effective areas for CTA: create run scripts" echo "------------------------------------------------" echo -source ${EVNDISPSYS}/setObservatory.sh CTA +# set the right observatory (environmental variables) +if [ ! -n "$EVNDISP_APPTAINER" ]; then + source "${EVNDISPSYS}"/setObservatory.sh CTA +fi ###################################################################### # input variables @@ -45,7 +45,7 @@ echo "PROCESSING PARTICLE $PART" # read analysis values from parameter file if [ ! -e $ANAPAR ] then - echo "error: analysis parameter file not found: $ANAPAR" + echo "error: analysis parameter file not found: $ANAPAR" exit fi cp -f $ANAPAR $TMPDIR @@ -54,7 +54,7 @@ ANAPAR="${TMPDIR}/${ANAPARF}" # check again than runparameter file is available if [ ! -e $ANAPAR ] then - echo "error: analysis parameter file not found in tmp directory: $ANAPAR" + echo "error: analysis parameter file not found in tmp directory: $ANAPAR" exit fi echo "reading analysis parameter from $ANAPAR" @@ -74,7 +74,7 @@ then fi NIMAGESMIN=`grep NIMAGESMIN $ANAPAR | awk {'print $2'}` -# get telescope type dependent cuts +# get telescope type dependent cuts NCUTLST=`grep NLST $ANAPAR | awk {'print $2'}` NCUTMST=`grep NMST $ANAPAR | awk {'print $2'}` NCUTSST=`grep NSST $ANAPAR | awk {'print $2'}` @@ -100,7 +100,7 @@ else fi THETA2MINENEERGY=`grep THETA2MINENEERGY $ANAPAR | awk {'print $2'}` -for T in LST MST SST SCMST +for T in LST MST SST SCMST do NCUT="NCUT${T}" if [ -z "${!NCUT}" ] @@ -140,7 +140,7 @@ echo " Analysis parameters: $NIMAGESMIN $ANADIR $TMVACUT $EFFAREABASEDIR $OBSTI if [ -z "$ANADIR" ] || [ -z "$NIMAGESMIN" ] || [ -z "$TMVACUT" ] || [ -z "$EFFAREABASEDIR" ] || [ -z "$OBSTIME" ] then - echo "error: analysis parameter file not correct: $ANAPAR" + echo "error: analysis parameter file not correct: $ANAPAR" echo " one variable missing" exit fi @@ -259,10 +259,10 @@ then THETA2MIN=( 0. ) THETA2MAX=( 1. ) OFFMEA=( 0.5 ) - fi + fi ISOTROPY="1" DIRECTIONCUT="0" -fi +fi if [ $PART = "proton" ] || [ $PART = "proton_onSource" ] then if [[ ${DSET:0:2} == "GR" ]] @@ -270,7 +270,7 @@ then MSCFILE=$DDIR/proton*"deg$MCAZ"*mscw.root else MSCFILE=$DDIR/proton*."$ARRAY"_ID"$RECID$MCAZ"*.mscw.root - fi + fi if [ $ARRAY = "V5" ] then MSCFILE=$DDIR/proton."$ARRAY"_ID"$RECID$MCAZ".mscw.root @@ -279,7 +279,7 @@ then OFFMIN=( 0. ) OFFMAX=( 100000. ) # NOTE: this is theta and not theta2 - if [ $PART = "proton" ] + if [ $PART = "proton" ] then OFIL=proton."$ARRAY"_ID"$RECID".eff if [ $BFINEBINNING = "TRUE" ] @@ -389,8 +389,8 @@ do fi ############################################################################### # create cut file - iCBFILE=`basename $CFIL` - if [ $PART = "gamma_onSource" ] || [ $PART = "gamma_cone" ] + iCBFILE=`basename $CFIL` + if [ $PART = "gamma_onSource" ] || [ $PART = "gamma_cone" ] then CFILP="${CFIL}.gamma.dat" else @@ -407,14 +407,14 @@ do cp -f $CFILP $iCFIL # wobble offset - if [ $PART = "gamma_onSource" ] || [ $PART = "gamma_cone" ] + if [ $PART = "gamma_onSource" ] || [ $PART = "gamma_cone" ] then WOBBLEOFFSET=${OFFMEA[$i]} else WOBBLEOFFSET=${OFFMEA[$j]} fi # angular resolution file - if [ $PART = "gamma_onSource" ] + if [ $PART = "gamma_onSource" ] then ANGRESFILE=${PRODBASEDIR}/EffectiveAreas/${EFFMCAZDIR}/AngularResolution/gamma_onSource."$ARRAY"_ID"$RECID".eff-0.root else @@ -522,9 +522,9 @@ do echo "* ENERGYSPECTRUMINDEX 1 2.5 0.1" >> $MSCF echo "* ESPECTRUM_FOR_WEIGHTING $CTA_EVNDISP_AUX_DIR/AstroData/TeV_data/EnergySpectrum_literatureValues_CR.dat 0" >> $MSCF if [ $GETXOFFYOFFAFTERCUTS = "yes" ] - then + then echo "* GETXOFFYOFFAFTERCUTS 1" >> $MSCF - fi + fi fi if [ $PART = "electron" ] || [ $PART = "electron_onSource" ] @@ -587,7 +587,7 @@ do # run effective area code ${EVNDISPSYS}/bin/makeEffectiveArea $MSCF $OFIX.root > $OLOG.log - # cross check if run was successfull + # cross check if run was successful # (expect simply > 800k) DS=$(du -k $OFIX.root | cut -f 1) if [[ ${DS} -le $minimumsize ]]; then @@ -626,6 +626,3 @@ do done exit - - -exit diff --git a/analysis/CTA.MSCW_ENERGY.qsub_make_tables.sh b/analysis/CTA.MSCW_ENERGY.qsub_make_tables.sh index 5b06c3c..67adf27 100755 --- a/analysis/CTA.MSCW_ENERGY.qsub_make_tables.sh +++ b/analysis/CTA.MSCW_ENERGY.qsub_make_tables.sh @@ -2,7 +2,6 @@ # # fill tables for CTA # -# TFIL=TABLEFILE RECID=RECONSTRUCTIONID @@ -13,7 +12,9 @@ MCAZ="AZIMUTH" MINTEL="MMTEL" # set the right observatory (environmental variables) -source ${EVNDISPSYS}/setObservatory.sh CTA +if [ ! -n "$EVNDISP_APPTAINER" ]; then + source "${EVNDISPSYS}"/setObservatory.sh CTA +fi # output data files are written to this directory ODIR=${CTA_USER_DATA_DIR}"/analysis/AnalysisData/$DSET/"$ARRAY"/Tables/" @@ -71,7 +72,7 @@ then LISFILE=$CTA_EVNDISP_AUX_DIR/DetectorGeometry/CTA.prod6${DARR}.lis elif [[ $DSET == *"prod3"* ]] then - if [[ $DSET == *"paranal"* ]] && [[ $DSET != *"prod3b"* ]] + if [[ $DSET == *"paranal"* ]] && [[ $DSET != *"prod3b"* ]] then DARR=${ARRAY%??} LISFILE=$CTA_EVNDISP_AUX_DIR/DetectorGeometry/CTA.prod3${DARR}.lis @@ -100,7 +101,7 @@ fi if [[ $DSET == *"prod3b"* ]] && [[ $DSET != *"SCT"* ]] then MOPT="$MOPT -teltypeweightfile $CTA_EVNDISP_AUX_DIR/DetectorGeometry/CTA.prod3b.TelescopeWeights.dat" -fi +fi echo $MOPT ######################################### @@ -119,8 +120,3 @@ then ${EVNDISPSYS}/bin/logFile makeTableFileList $ODIR/$TFIL-$ARRAY.root ${TMPLIST} rm -f ${TMPLIST} fi - -# sleep -sleep 2 - -exit diff --git a/analysis/CTA.TMVA.qsub_train.sh b/analysis/CTA.TMVA.qsub_train.sh index 239248c..6e14e02 100755 --- a/analysis/CTA.TMVA.qsub_train.sh +++ b/analysis/CTA.TMVA.qsub_train.sh @@ -2,8 +2,6 @@ # # script to train cuts/MVAs with TMVA # -# -# RPARA=RUNPARA NENE=NBINSNBINS @@ -11,7 +9,9 @@ NENE=NBINSNBINS ulimit -n 2056 # set the right observatory (environmental variables) -source "${EVNDISPSYS}"/setObservatory.sh CTA +if [ ! -n "$EVNDISP_APPTAINER" ]; then + source "${EVNDISPSYS}"/setObservatory.sh CTA +fi for ((EBIN=0; EBIN < $NENE; EBIN++)) do @@ -49,5 +49,3 @@ do rm -f $CDIR/BDT_${EBIN}_BDT_0.weights.xml fi done - -exit diff --git a/analysis/CTA.WPPhysWriter.qsub.sh b/analysis/CTA.WPPhysWriter.qsub.sh index 18058a9..9d6a3f7 100755 --- a/analysis/CTA.WPPhysWriter.qsub.sh +++ b/analysis/CTA.WPPhysWriter.qsub.sh @@ -2,9 +2,6 @@ # # script to write CTA WP Phys Files # -# -# -####################################################################### AXRRAY=ARRAY DXDIR=DDIR @@ -22,7 +19,9 @@ fi echo "OFFAXIS FINE BINNING $OBBIN" # set the right observatory (environmental variables) -source $EVNDISPSYS/setObservatory.sh CTA +if [ ! -n "$EVNDISP_APPTAINER" ]; then + source "${EVNDISPSYS}"/setObservatory.sh CTA +fi rm -f $OXUTNAME.$AXRRAY.$OXBSTIME.log @@ -32,7 +31,7 @@ $EVNDISPSYS/bin/writeCTAWPPhysSensitivityFiles $AXRRAY $OXBSTIME $DXDIR $OXUTNAM ############################################################################ if [ -e $OXUTNAME.$AXRRAY.$OXBSTIME.log ] -then +then DE=$(grep "error filling" $OXUTNAME.$AXRRAY.$OXBSTIME.log) DF=$(grep "error, cannot find effective area tree" $OXUTNAME.$AXRRAY.$OXBSTIME.log) if [[ -z ${DE} ]] && [[ -z ${DF} ]]; then @@ -44,5 +43,3 @@ then fi fi fi - -exit diff --git a/analysis/CTA.prepareTMVA.qsub_train.sh b/analysis/CTA.prepareTMVA.qsub_train.sh index 10813ea..57d2241 100755 --- a/analysis/CTA.prepareTMVA.qsub_train.sh +++ b/analysis/CTA.prepareTMVA.qsub_train.sh @@ -2,15 +2,15 @@ # # script to prepare training events # -# -# PFIL=RUNPARA OFIL=OOOFILE ulimit -n 2056 # set the right observatory (environmental variables) -source "${EVNDISPSYS}"/setObservatory.sh CTA +if [ ! -n "$EVNDISP_APPTAINER" ]; then + source "${EVNDISPSYS}"/setObservatory.sh CTA +fi echo ${PFIL}.runparameter rm -f $PFIL.log @@ -28,5 +28,3 @@ then ${EVNDISPSYS}/bin/logFile tmvaPrepareRunparameter ${OFIL}.root ${PFIL}.runparameter rm -f ${PFIL}.runparameter fi - -exit diff --git a/setSoftwarePathsForApptainers.sh b/setSoftwarePathsForApptainers.sh new file mode 100755 index 0000000..2163df3 --- /dev/null +++ b/setSoftwarePathsForApptainers.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# +# set software paths to analysis paths +# (usage of apptainers) +# + +if [[ "${BASH_SOURCE[0]}" == "${0}" ]] +then + echo "source ./setSoftwarePathsForApptainers.sh.sh " + echo "(use source)" + exit +fi + +if [ ! -n "$1" ] +then + echo "source ./setSoftwarePathsForApptainers.sh .sh " + echo + return +fi + +TDIR=$(pwd) + +# main working directory (logs and code) +DSET="${1}" +export WORKDIR="${CTA_USER_WORK_DIR%/}/analysis/AnalysisData/${DSET}" + +# Eventdisplay settings +export EVNDISP_APPTAINER="${CTA_USER_WORK_DIR%/}/analysis/AnalysisData/APPTAINERS/eventdisplay_20240409-144727-cta-prod5.sif" +export EVNDISPSYS="apptainer exec --no-mount bind-paths --cleanenv ${EVNDISP_APPTAINER} /eventdisplay_workdir/Eventdisplay/" + +export CTA_EVNDISP_AUX_DIR=${WORKDIR}/Eventdisplay_AnalysisFiles_CTA/ +export OBS_EVNDISP_AUX_DIR=${CTA_EVNDISP_AUX_DIR} +export CTA_USER_LOG_DIR="${WORKDIR}/LOGS/"