Merge pull request #47 from Eventdisplay/prod6-tests

Script additions for Prod6 North

Author: GernotMaier

.pre-commit-config.yaml

@@ -1,24 +1,24 @@
 repos:
   # https://pycqa.github.io/isort/docs/configuration/black_compatibility.html#integration-with-pre-commit
   - repo: https://github.com/pycqa/isort
-    rev: 5.12.0
+    rev: 5.13.2
     hooks:
       - id: isort
         args: ["--profile", "black", "--filter-files"]
   - repo: https://github.com/psf/black
-    rev: 23.7.0
+    rev: 24.8.0
     hooks:
       - id: black
         args: ["--line-length=100"]
   # https://black.readthedocs.io/en/stable/guides/using_black_with_other_tools.html?highlight=other%20tools#flake8
   - repo: https://github.com/PyCQA/flake8
-    rev: 6.1.0
+    rev: 7.1.1
     hooks:
       - id: flake8
         args: ["--max-line-length=100", "--extend-ignore=E203,E712"]
   # https://github.com/pre-commit/pre-commit-hooks
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.4.0
+    rev: v4.6.0
     hooks:
       - id: trailing-whitespace
       - id: end-of-file-fixer

CTA.mainRunScriptsReduced.sh

@@ -8,7 +8,7 @@
 if [ $# -lt 2 ]; then
     echo "
 ./CTA.mainRunScripts.sh <data set> <run mode>
-    
+
     data sets:
 
        Prod3b analysis:
@@ -19,7 +19,7 @@ if [ $# -lt 2 ]; then
             prod5b-North-20deg prod5b-North-40deg prod5b-North-60deg
             prod5b-North-20deg-moon prod5b-North-40deg-moon prod5b-North-60deg-moon
        Prod6 analysis:
-            prod6-North-20deg
+            prod6-North-20deg prod6-North-40deg prod6-North-52deg prod6-North-60deg
             prod6-South-20deg
 
     run modes:
@@ -34,7 +34,7 @@ P2="$1"
 RUN="$2"
 
 SITE="South"
-if [[ $P2 == *"North"* ]]; then 
+if [[ $P2 == *"North"* ]]; then
     SITE="North"
 fi
 
@@ -43,10 +43,10 @@ RECID="0"
 # run scripts are collected here
 RUNSCRIPTDIR="${CTA_USER_LOG_DIR}/jobs/$(uuidgen)"
 mkdir -p ${RUNSCRIPTDIR}
-    
+
 if [[ ${RUN} == "MAKETABLES" ]] || [[ ${RUN} == "DISPBDT" ]] || [[ ${RUN} == "ANATABLES" ]] || [[ ${RUN} == "PREPARETMVA" ]]; then
    ./CTA.runAnalysis.sh ${P2} ${RUN} ${RECID} 2 2 2 2 ${RUNSCRIPTDIR}
-   if [[ $SITE == "South" ]]; then
+   if [[ $SITE == "South" ]] || [[ $P2 == *"prod6"* ]]; then
        ./CTA.runAnalysis.sh ${P2}-sub ${RUN} ${RECID} 2 2 2 2 ${RUNSCRIPTDIR}
    elif [[ $SITE == *"North"* ]]; then
        ./CTA.runAnalysis.sh ${P2}-LST ${RUN} ${RECID} 2 2 2 2 ${RUNSCRIPTDIR}
@@ -60,7 +60,7 @@ else
        while IFS= read -r mult
        do
            ./CTA.runAnalysis.sh ${P2}-sub ${RUN} ${RECID} $mult ${RUNSCRIPTDIR}
-       done < NIM-South-sub.dat
+       done < NIM-${SITE}-sub.dat
    elif [[ $SITE == "North" ]]; then
        ./CTA.runAnalysis.sh ${P2}-LST ${RUN} ${RECID} 2 2 2 2 ${RUNSCRIPTDIR}
    fi

CTA.runAnalysis.sh

@@ -25,7 +25,7 @@ then
          add 'moon' for NSB5x data sets
          prod3b-S20-SCT156Tel
     Prod6 analysis:
-        prod6-North-20deg
+        prod6-North-20deg prod6-North-40deg prod6-North-52deg prod6-North-60deg
         prod6-South-20deg
 
     possible run modes are EVNDISP MAKETABLES DISPBDT/DISPMLP ANATABLES PREPARETMVA TRAIN ANGRES QC CUTS PHYS
@@ -274,10 +274,12 @@ then
    if [[ $P2 == *"SV0"* ]]; then
       ARRAY=( "subArray.prod5.South-SV0.list" )
    elif [[ $P2 == *"SV"* ]]; then
-       ARRAY=( "subArray.prod5.South-Alpha-SV.list" )
+       ARRAY=( "subArray.prod5.South-Alpha-SV-2MSTs.list" )
        if [[ $P2 == *"sub"* ]]; then
            ARRAY=( "subArray.prod5.South-Alpha-SV-sub.list" )
        fi
+   elif [[ $P2 == *"MSTF"* ]]; then
+       ARRAY=( "subArray.prod5.South-MSTF.list" )
    fi
    ARRAYDIR="prod5"
    TDATE="g20230823"
@@ -317,17 +319,23 @@ then
    else
        SITE="${SITE}-dark"
    fi
-   EDM="-sq20-LL"
+   EDM="-sq30-LL"
+   if [[ $P2 == "prod6-N"* ]]; then
+       EDM="-lin30-LL"
+       EDM="-sq230-LL"
+   fi
    if [[ $P2 == *"DL2plus"* ]]; then
        EDM="-sq10-LL-DL2plus"
    fi
-#   ARRAY=( "subArray.prod6.${NS}Hyper${SCT}.list" )
    ARRAY=( "subArray.prod6.${NS}Alpha${SCT}.list" )
    if [[ $P2 == *"sub"* ]]; then
        ARRAY=( "subArray.prod6.${NS}Alpha-sub.list" )
    fi
+   if [[ $P2 == *"Hyper"* ]] || [[ $P2 == *"hyper"* ]]; then
+       ARRAY=( "subArray.prod6.NorthHyper.list" )
+   fi
    ARRAYDIR="prod6"
-   TDATE="g20220901"
+   TDATE="g20240826"
    ANADATE="${TDATE}"
    TMVADATE="${ANADATE}"
    EFFDATE="${ANADATE}"
@@ -360,11 +368,12 @@ NIMAGESMIN=$((SCMST<NIMAGESMIN ? SCMST : NIMAGESMIN))
 # (note that all steps except CUTS and PHYS are done only for 50h)
 OBSTIME=( "50h" "5h" "30m" "10m" "10h" "20h" "100h" "500h" "5m" "1m" "2h" )
 OBSTIME=( "10s" "30s" "300s" "1000s" "3000s" "10000s" "30000s" )
-OBSTIME=( "50h" "30m" )
 OBSTIME=( "50h" "5h" )
 OBSTIME=( "50h" "5h" "30m" "100s" )
 OBSTIME=( "5h" "30m" "100s" )
 OBSTIME=( "500h" "250h" "100h" "50h" "20h" "10h" "5h" "1h" "5m" "30m" "100s" )
+OBSTIME=( "50h" "30m" )
+OBSTIME=( "5h" )
 OBSTIME=( "50h" )
 
 echo "$RUN" "$SITE"
@@ -565,7 +574,7 @@ do
                   fi
                   echo "OFFAXISFINEBINNING $BFINEBINNING" >> "$PARA"
                   if [[ ${RUN} == "CUTS" ]] && [[ ${OOTIME} == "50h" ]]; then
-                     if [[ $P2 == *"DL2plus"* ]] || [[ $EDM == *"sq51"* ]] || [[ $EDM == *"lin51"* ]]; then
+                     if [[ $P2 == *"DL2plus"* ]] || [[ $EDM == *"prod6"* ]]; then
                          echo "DL2FILLING FULLTREES" >> "$PARA"
                      else
                          echo "DL2FILLING DL2" >> "$PARA"

analysis/CTA.DISPTRAINING.sub_analyse.sh

@@ -19,7 +19,7 @@
 #
 SUBC="condor"
 h_cpu="47:29:00"
-h_vmem="12000M"
+h_vmem="24000M"
 tmpdir_size="1G"
 
 if [ $# -lt 5 ]
@@ -34,7 +34,7 @@ then
    echo "  <TMVA parameters>  file name of list of TMVA parameter file"
    echo "  <scaling>          layout scaling (e.g. 5); give 99 to ignore scaling"
    echo
-   echo "  (note 1: hardwired telescope types in this script)" 
+   echo "  (note 1: hardwired telescope types in this script)"
    echo "  (note 2: disp core training switched off)"
    echo
    exit
@@ -62,8 +62,8 @@ if [ -n $8 ]
 then
    QSUBOPT="$8"
 fi
-QSUBOPT=${QSUBOPT//_X_/ } 
-QSUBOPT=${QSUBOPT//_M_/-} 
+QSUBOPT=${QSUBOPT//_X_/ }
+QSUBOPT=${QSUBOPT//_M_/-}
 
 #########################################
 # TMVA options
@@ -153,7 +153,7 @@ else
 fi
 
 #########################################
-# 
+#
 #########################################
 for MLP in "${MLPLIST[@]}"
 do

analysis/CTA.EFFAREA.qsub_analyse_list.sh

@@ -45,7 +45,7 @@ echo "PROCESSING PARTICLE $PART"
 # read analysis values from parameter file
 if [ ! -e $ANAPAR ]
 then
-  echo "error: analysis parameter file not found: $ANAPAR" 
+  echo "error: analysis parameter file not found: $ANAPAR"
   exit
 fi
 cp -f $ANAPAR $TMPDIR
@@ -54,7 +54,7 @@ ANAPAR="${TMPDIR}/${ANAPARF}"
 # check again than runparameter file is available
 if [ ! -e $ANAPAR ]
 then
-  echo "error: analysis parameter file not found in tmp directory: $ANAPAR" 
+  echo "error: analysis parameter file not found in tmp directory: $ANAPAR"
   exit
 fi
 echo "reading analysis parameter from $ANAPAR"
@@ -74,7 +74,7 @@ then
 fi
 
 NIMAGESMIN=`grep NIMAGESMIN $ANAPAR | awk {'print $2'}`
-# get telescope type dependent cuts 
+# get telescope type dependent cuts
 NCUTLST=`grep NLST $ANAPAR | awk {'print $2'}`
 NCUTMST=`grep NMST $ANAPAR | awk {'print $2'}`
 NCUTSST=`grep NSST $ANAPAR | awk {'print $2'}`
@@ -100,7 +100,7 @@ else
 fi
 THETA2MINENEERGY=`grep THETA2MINENEERGY $ANAPAR | awk {'print $2'}`
 
-for T in LST MST SST SCMST 
+for T in LST MST SST SCMST
 do
     NCUT="NCUT${T}"
     if [ -z "${!NCUT}" ]
@@ -140,7 +140,7 @@ echo "  Analysis parameters: $NIMAGESMIN $ANADIR $TMVACUT $EFFAREABASEDIR $OBSTI
 
 if [ -z "$ANADIR" ] || [ -z "$NIMAGESMIN" ] || [ -z "$TMVACUT" ] || [ -z "$EFFAREABASEDIR" ] || [ -z "$OBSTIME" ]
 then
-  echo "error: analysis parameter file not correct: $ANAPAR" 
+  echo "error: analysis parameter file not correct: $ANAPAR"
   echo " one variable missing"
   exit
 fi
@@ -259,18 +259,18 @@ then
       THETA2MIN=( 0. )
       THETA2MAX=( 1. )
       OFFMEA=( 0.5 )
-   fi   
+   fi
    ISOTROPY="1"
    DIRECTIONCUT="0"
-fi 
+fi
 if [ $PART = "proton" ] || [ $PART = "proton_onSource" ]
 then
    if [[ ${DSET:0:2} == "GR" ]]
    then
        MSCFILE=$DDIR/proton*"deg$MCAZ"*mscw.root
    else
        MSCFILE=$DDIR/proton*."$ARRAY"_ID"$RECID$MCAZ"*.mscw.root
-   fi    
+   fi
    if [ $ARRAY = "V5" ]
    then
       MSCFILE=$DDIR/proton."$ARRAY"_ID"$RECID$MCAZ".mscw.root
@@ -279,7 +279,7 @@ then
    OFFMIN=( 0. )
    OFFMAX=( 100000. )
 # NOTE: this is theta and not theta2
-   if [ $PART = "proton" ] 
+   if [ $PART = "proton" ]
    then
       OFIL=proton."$ARRAY"_ID"$RECID".eff
        if [ $BFINEBINNING = "TRUE" ]
@@ -389,8 +389,8 @@ do
           fi
     ###############################################################################
     # create cut file
-          iCBFILE=`basename $CFIL`      
-          if [ $PART = "gamma_onSource" ] || [ $PART = "gamma_cone" ] 
+          iCBFILE=`basename $CFIL`
+          if [ $PART = "gamma_onSource" ] || [ $PART = "gamma_cone" ]
           then
               CFILP="${CFIL}.gamma.dat"
           else
@@ -407,14 +407,14 @@ do
           cp -f $CFILP $iCFIL
 
     # wobble offset
-          if [ $PART = "gamma_onSource" ] || [ $PART = "gamma_cone" ] 
+          if [ $PART = "gamma_onSource" ] || [ $PART = "gamma_cone" ]
           then
              WOBBLEOFFSET=${OFFMEA[$i]}
           else
              WOBBLEOFFSET=${OFFMEA[$j]}
           fi
     # angular resolution file
-          if [ $PART = "gamma_onSource" ] 
+          if [ $PART = "gamma_onSource" ]
           then
              ANGRESFILE=${PRODBASEDIR}/EffectiveAreas/${EFFMCAZDIR}/AngularResolution/gamma_onSource."$ARRAY"_ID"$RECID".eff-0.root
           else
@@ -522,9 +522,9 @@ do
          echo "* ENERGYSPECTRUMINDEX  1 2.5 0.1" >> $MSCF
          echo "* ESPECTRUM_FOR_WEIGHTING $CTA_EVNDISP_AUX_DIR/AstroData/TeV_data/EnergySpectrum_literatureValues_CR.dat 0" >> $MSCF
          if  [ $GETXOFFYOFFAFTERCUTS = "yes" ]
-         then	
+         then
              echo "* GETXOFFYOFFAFTERCUTS 1" >> $MSCF
-         fi    
+         fi
 
       fi
       if [ $PART = "electron" ] || [ $PART = "electron_onSource" ]
@@ -624,8 +624,3 @@ do
       mv -v $OFIX.root ${ODIR}/
    done
 done
-
-exit
-
-
-exit

analysis/CTA.EVNDISP.qsub_convert_and_analyse_MC_VDST_ArrayJob.sh

@@ -26,11 +26,17 @@ fi
 # set array
 FIELD=$SUBA
 
+# TMPDIR - usually set on cluster nodes; useful for testing
+if [[ ! -n "$TMPDIR" ]]; then
+    TMPDIR="$CTA_USER_DATA_DIR/tmp/${DSET}"
+    mkdir -p "${TMPDIR}"
+fi
+
 ###################################
 # converter command line parameter
 COPT="-pe"
 # prod3(b): read effective focal lengths from external file
-if [[ $DSET == *"prod3"* ]] 
+if [[ $DSET == *"prod3"* ]]
 then
     COPT="-rfile ${CTA_EVNDISP_AUX_DIR}/DetectorGeometry/CTA.prod3b.EffectiveFocalLength.dat"
     COPT="$COPT -pe"
@@ -42,7 +48,7 @@ COPT="$COPT -c $PEDFILE"
 # OPT="-averagetzerofiducialradius=0.5 -reconstructionparameter $ACUT"
 OPT="-imagesquared -averagetzerofiducialradius=0.5 -reconstructionparameter $ACUT"
 OPT="$OPT -writeimagepixellist"
-if [[ $DSET == *"prod3"* ]] 
+if [[ $DSET == *"prod3"* ]]
 then
     # needs to be the same as used for IPR graph preparation
     OPT="$OPT -ignoredstgains"
@@ -114,7 +120,7 @@ do
 # input sim_telarray file: $SIMFIL
 # ($SIMFIL should be set then to the zero suppressed file)
 #
-# best to write to output (zero suppressed file) to the temporary disk on the 
+# best to write to output (zero suppressed file) to the temporary disk on the
 # current note: $TMPDIR/<zerosuppressed file>
 #
 # $HESSIOSYS/bin/ ....
@@ -123,7 +129,7 @@ do
 # execute converter
    SIMFIL=`ls $TMPDIR/*.simtel.${EXTE}`
    echo "TMPDIR FILES " $SIMFIL
-   if [[ $DSET == *"prod3"* ]] 
+   if [[ $DSET == *"prod3"* ]]
    then
        if [[ $DSET == *"paranal"* ]] && [[ $DSET != *"prod3b"* ]]
        then
@@ -137,13 +143,13 @@ do
        else
            DETGEO=${CTA_EVNDISP_AUX_DIR}/DetectorGeometry/CTA.prod3Sb${N:1}.lis
        fi
-   elif [[ $DSET == *"prod4"* ]] 
+   elif [[ $DSET == *"prod4"* ]]
    then
        DETGEO=${CTA_EVNDISP_AUX_DIR}/DetectorGeometry/CTA.prod4${N}.lis
-   elif [[ $DSET == *"prod5"* ]] 
+   elif [[ $DSET == *"prod5"* ]]
    then
        DETGEO=${CTA_EVNDISP_AUX_DIR}/DetectorGeometry/CTA.prod5${N}.lis
-   elif [[ $DSET == *"prod6"* ]] 
+   elif [[ $DSET == *"prod6"* ]]
    then
        DETGEO=${CTA_EVNDISP_AUX_DIR}/DetectorGeometry/CTA.prod6${N}.lis
    fi
@@ -185,7 +191,7 @@ do
       fi
       cp -v -f $TMPDIR/[0-9]*.root ${ODIR}/${RUNN}HD_${ILINE}_${MCAZ}deg.root
   else
-      echo "No root files found!" 
+      echo "No root files found!"
       if [ -e $TMPDIR/$OFIL.$N.convert.log ]; then
          cp -f -v $TMPDIR/$OFIL.$N.convert.log $ODIR/
       fi
@@ -202,7 +208,7 @@ do
       cp -v -f $TMPDIR/$OFIL.root $ODIR/VDST/
    fi
    ls -lh $TMPDIR/*.root
-# clean up 
+# clean up
    rm -f $TMPDIR/$OFIL.root
    rm -f $TMPDIR/[0-9]*.root
    echo "==================================================================="