Merge pull request #55 from Eventdisplay/v2.3-dev1

v2.3 dev1 - loss04 analysis

Author: GernotMaier

CTA.mainRunScriptsReduced.sh

@@ -24,7 +24,9 @@ if [ $# -lt 2 ]; then
             prod6-South-20deg
 
     run modes:
-        MAKETABLES DISPBDT ANATABLES PREPARETMVA TRAIN ANGRES QC CUTS PHYS
+        MAKETABLES DISPBDT ANATABLES PREPARETMVA TRAIN ANGRES QC CUTS PHYS CLEANUP
+
+    optional run modes: TRAIN_RECO_QUALITY TRAIN_RECO_METHOD
 
     "
     exit
@@ -52,12 +54,14 @@ if [[ ${RUN} == "MAKETABLES" ]] || [[ ${RUN} == "DISPBDT" ]] || [[ ${RUN} == "AN
    elif [[ $SITE == *"North"* ]]; then
        ./CTA.runAnalysis.sh ${P2}-LST ${RUN} ${RECID} 2 2 2 2 ${RUNSCRIPTDIR}
    fi
+elif [[ ${RUN} == "CLEANUP" ]]; then
+   ./CTA.runAnalysis.sh ${P2} ${RUN} ${RECID} 2 2 2 2 ${RUNSCRIPTDIR}
 else
    while IFS= read -r mult
    do
        ./CTA.runAnalysis.sh ${P2} ${RUN} ${RECID} $mult ${RUNSCRIPTDIR}
    done < NIM-${SITE}.dat
-   if [[ $SITE == "South" ]]; then
+   if [[ $SITE == "South" ]] || [[ $P2 == *"prod6"* ]]; then
        while IFS= read -r mult
        do
            ./CTA.runAnalysis.sh ${P2}-sub ${RUN} ${RECID} $mult ${RUNSCRIPTDIR}

CTA.runAllScripts.sh

@@ -0,0 +1,26 @@
+#!/bin/sh
+#
+# Run all prod6 scripts over all zenith angles and NSB levels
+#
+
+if [ $# -lt 1 ]; then
+    echo "
+    ./CTA.runAllScripts.sh <run mode>
+
+    run modes:
+        MAKETABLES DISPBDT ANATABLES PREPARETMVA TRAIN ANGRES QC CUTS PHYS CLEANUP
+
+    optional run modes: TRAIN_RECO_QUALITY TRAIN_RECO_METHOD
+
+    "
+    exit
+fi
+# run mode
+RUN="$1"
+
+for ZE in 20deg 40deg 52deg 60deg; do
+    for NSB in dark moon; do
+        dataset="prod6-North-${ZE}-${NSB}"
+        ./CTA.mainRunScriptsReduced.sh $dataset $RUN
+    done
+done

CTA.runAnalysis.sh

@@ -28,7 +28,9 @@ then
         prod6-North-20deg prod6-North-40deg prod6-North-52deg prod6-North-60deg
         prod6-South-20deg
 
-    possible run modes are EVNDISP MAKETABLES DISPBDT/DISPMLP ANATABLES PREPARETMVA TRAIN ANGRES QC CUTS PHYS
+    possible run modes are EVNDISP MAKETABLES DISPBDT/DISPMLP ANATABLES PREPARETMVA TRAIN ANGRES QC CUTS PHYS CLEANUP
+
+    optional run modes: TRAIN_RECO_QUALITY TRAIN_RECO_METHOD
 
     [recids]: 0 = all telescopes (default), 1 = LSTs, 2 = MSTs, 3 = SSTs, 4 = MSTs+SSTs, 5 = LSTs+MSTs
 
@@ -249,7 +251,7 @@ then
    elif [[ $P2 == *"moon"* ]]; then
        SITE="${SITE}-NSB5x"
    fi
-   EDM="-sq52-LL"
+   EDM="-sq70-LL"
    if [[ $P2 == *"DL2plus"* ]]; then
        EDM="-sq10-LL-DL2plus"
    fi
@@ -259,11 +261,13 @@ then
    ARRAY=( "subArray.prod5.South-Alpha.list" )
    ARRAY=( "subArray.prod5.South-SV3f-v2.list" )
    ARRAY=( "subArray.prod5.South-AlphaC8aj.list" )
+   ARRAY=( "subArray.prod5.South-Beta.list" )
    if [[ $P2 == *"sub"* ]]; then
        ARRAY=( "subArray.prod5.South-Alpha-2LSTs42SSTsBeta-sub.list")
        ARRAY=( "subArray.prod5.South-Alpha-sub.list" )
        ARRAY=( "subArray.prod5.South-SV3f-v3-sub.list" )
        ARRAY=( "subArray.prod5.South-AlphaC8aj-sub.list" )
+       ARRAY=( "subArray.prod5.South-Beta-sub.list" )
 #       ARRAY=( "subArray.prod5.South-70SSTs-sub.list" )
    fi
    if [[ $P2 == *"Hyper"* ]] || [[ $P2 == *"hyper"* ]]; then
@@ -290,9 +294,8 @@ then
        ARRAY=( "subArray.prod5.South-MSTF.list" )
    fi
    ARRAYDIR="prod5"
-   TDATE="g20230823"
+   TDATE="g20250826"
    ANADATE="${TDATE}"
-   ANADATE="g20250108"
    TMVADATE="${ANADATE}"
    EFFDATE="${ANADATE}"
    PHYSDATE="${EFFDATE}"
@@ -327,21 +330,22 @@ then
    else
        SITE="${SITE}-dark"
    fi
-   EDM="-sq40-LL"
+   EDM="-sq50-LL"
+   EDM="-sq51-LL"
    if [[ $P2 == *"DL2plus"* ]]; then
-       EDM="-sq40-LL-DL2plus"
+       EDM="-sq50-LL-DL2plus"
    fi
-   ARRAY=( "subArray.prod6.${NS}Alpha${SCT}.list" )
    ARRAY=( "subArray.prod6.${NS}ML${SCT}.list" )
+   ARRAY=( "subArray.prod6.${NS}Alpha${SCT}.list" )
    if [[ $P2 == *"sub"* ]]; then
-       ARRAY=( "subArray.prod6.${NS}Alpha-sub.list" )
        ARRAY=( "subArray.prod6.${NS}ML-sub.list" )
+       ARRAY=( "subArray.prod6.${NS}Alpha-sub.list" )
    fi
    if [[ $P2 == *"Hyper"* ]] || [[ $P2 == *"hyper"* ]]; then
        ARRAY=( "subArray.prod6.NorthHyper.list" )
    fi
    ARRAYDIR="prod6"
-   TDATE="g20241104"
+   TDATE="g20250822"
    ANADATE="${TDATE}"
    TMVADATE="${ANADATE}"
    EFFDATE="${ANADATE}"
@@ -420,6 +424,12 @@ then
    done
    continue
 fi
+# remove from PHYS directory any unreasonable files (e.g. LST4 requirement for 2 LST array)
+if [[ $RUN == "CLEANUP" ]]; then
+    PHYSDIR="${CTA_USER_DATA_DIR}/analysis/AnalysisData/${SITE}${EDM}/Phys-${PHYSDATE}"
+    ./utilities/removeUnreaseonablePhysFiles.sh ${PHYSDIR}
+    exit
+fi
 ##########################################
 # for the following: duplicate the array list adding the scaling to array names
 if [[ ! -e ${ARRAYDIR}/$ARRAY ]]; then
@@ -559,6 +569,8 @@ do
                   EFFFULLDIR="${CTA_USER_DATA_DIR}/analysis/AnalysisData/${SITE}${EDM}/EffectiveAreas/${EFFDIR}/"
                   echo "MSCWSUBDIRECTORY ${MSCWSUBDIRECTORY}" >> "$PARA"
                   echo "TMVASUBDIR BDT-${TMVAVERSION}-ID$ID$AZ-$TMVATYPF-$TMVADATE" >> "$PARA"
+                  echo "TMVA_RECO_METHOD BDT-RECO-METHOD-${TMVAVERSION}-ID$ID$AZ-$TMVATYPF-$TMVADATE" >> "$PARA"
+                  echo "TMVA_RECO_QUALITY BDT-RECO-QUALITY-${TMVAVERSION}-ID$ID$AZ-$TMVATYPF-$TMVADATE" >> "$PARA"
                   echo "EFFAREASUBDIR ${EFFDIR}" >> "$PARA"
                   EFFBDIR=EffectiveArea-50h-ID$ID$AZ-$ETYPF-$EFFDATE-$EFFVERSION
                   echo "EFFAREASUBBASEDIR $EFFBDIR" >> "$PARA"
@@ -608,11 +620,20 @@ do
 ##########################################
 # train BDTs
 # (note: BDT training does not need to be done for all observing periods)
-                  elif [[ $RUN == "TRAIN" ]] || [[ $RUN == "TMVA" ]]
+                  elif [[ $RUN == TRAIN* ]] || [[ $RUN == "TMVA" ]]
                   then
+
+                     if [ $RUN == "TRAIN_RECO_METHOD" ]; then
+                         TMVA_RUN_MODE="TrainAngularReconstructionMethod"
+                     elif [ $RUN == "TRAIN_RECO_QUALITY" ]; then
+                         TMVA_RUN_MODE="TrainReconstructionQuality"
+                     else
+                         TMVA_RUN_MODE="TrainGammaHadronSeparation"
+                     fi
                      if [ ${o} -eq 0 ] && [[ ! -z ${AZ} ]]
                      then
                          ./CTA.TMVA.sub_train.sh \
+                                "$TMVA_RUN_MODE" \
                                 "$NFILARRAY" \
                                 $OFFAXIS \
                                 ${SITE}${EDM} \

NIM-North-sub.dat

@@ -0,0 +1,3 @@
+2 2 2 2
+3 3 3 3
+4 4 4 4

analysis/CTA.DISPTRAINING.qsub_analyse.sh

@@ -15,6 +15,8 @@ TMVAO=TTT
 DSET="DATASET"
 ARRAY=AAA
 QC="QQQQ"
+# WEIGHT="MCe0*MCe0/(1.-loss)/(1.-loss)/(1.-loss)/(1.-loss)/(1.-loss)/(1.-loss)"
+WEIGHT="1."
 
 # set the right observatory (environmental variables)
 source $EVNDISPSYS/setObservatory.sh CTA
@@ -67,7 +69,8 @@ $EVNDISPSYS/bin/trainTMVAforAngularReconstruction $TLIST \
                                                   ${TMVAO} \
                                                   ${ADIR} \
                                                   "" \
-                                                  ${QC} \
+                                                  "${QC}" \
+                                                  "${WEIGHT}" \
                                                   0 > $ODIR/${BDT}-${TTYPE}.training.log 2>&1
 #########################################
 

analysis/CTA.DISPTRAINING.sub_analyse.sh

@@ -149,7 +149,7 @@ fi
 if [[ $TMVAP == *"MLP"* ]]; then
    declare -a MLPLIST=( "MLPDisp" "MLPDispEnergy" "MLPDispError" "MLPDispCore" )
 else
-   declare -a MLPLIST=( "BDTDisp" "BDTDispEnergy" "BDTDispError" "BDTDispCore" "BDTDispPhi" )
+   declare -a MLPLIST=( "BDTDisp" "BDTDispEnergy" "BDTDispError" "BDTDispCore" "BDTDispPhi" "BDTDispSign" )
 fi
 
 #########################################
@@ -168,12 +168,8 @@ do
             echo $QC
 
             let "NSTEP = $NSTEP + 1"
-            OFFDIR=${ODIR}.T${NSTEP}
-            OFFDIR=${ODIR}.E${NSTEP}
-            # removed cross and tgrad
-            OFFDIR=${ODIR}.S${NSTEP}
-            # tgrad^2 to trad
-            OFFDIR=${ODIR}.R${NSTEP}
+            # output directory (match CTA.MSCW_ENERGY.qsub_analyse_MC.sh)
+            OFFDIR=${ODIR}.G${NSTEP}
             ####################
             # output directory
             TDIR="${OFFDIR}/${MLP}/${MCAZ}/"
@@ -198,20 +194,13 @@ do
                 # South:  10%
                 # North: 20%
                 k=`expr 0.2*$NFIL | bc`
-                if [[ $DSET == *"LaPalma"* ]]
-                then
-                    k=$(echo $NFIL | awk '{printf "%d\n",$1*0.25}')
-                elif [[ $DSET == *"SCT"* ]]
-                then
-                    k=$(echo $NFIL | awk '{printf "%d\n",$1*0.30}')
-                else
-                    k=$(echo $NFIL | awk '{printf "%d\n",$1*0.10}')
-                fi
+                k=$(echo $NFIL | awk '{printf "%d\n",$1*0.20}')
                 # SV1 arrays: mix directories by hand!!
                 if [[ $ARRAY == *"SV1"* ]]
                 then
                     k=$(echo $NFIL)
                 fi
+                # experimental only
                 if [[ $ARRAY == *"DISP"* ]]
                 then
                     k=$(echo $NFIL | awk '{printf "%d\n",$1*0.50}')
@@ -223,7 +212,7 @@ do
 
                 ####################
                 # prepare run scripts
-                FNAM="$SHELLDIR/EDISP-$ARRAY-$SCALING-$MCAZ-$TELTYPE-$MLP-$NSTEP"
+                FNAM="$SHELLDIR/EDISP-$ARRAY-$DSET-$SCALING-$MCAZ-$TELTYPE-$MLP-$NSTEP"
                 cp $FSCRIPT.sh $FNAM.sh
 
                   sed -i -e "s|OFILE|$TDIR|" \

analysis/CTA.EFFAREA.sub_analyse_list.sh

@@ -9,7 +9,7 @@ set -e
 
 SUBC="condor"
 h_cpu="11:29:00"
-h_vmem="40000M"
+h_vmem="24000M"
 tmpdir_size="15G"
 
 if [ $# -lt 6 ]

analysis/CTA.MSCW_ENERGY.qsub_analyse_MC.sh

@@ -127,11 +127,7 @@ fi
 #########################################
 # options for simple stereo reconstruction
 MOPT="$MOPT -redo_stereo_reconstruction -sub_array_sim_telarray_counting $LISFILE"
-if [[ $DSET == *"LaPalma"* ]]; then
-    MOPT="$MOPT -minangle_stereo_reconstruction=15."
-else
-    MOPT="$MOPT -minangle_stereo_reconstruction=10."
-fi
+MOPT="$MOPT -minangle_stereo_reconstruction=5."
 # IMPORTANT: this must be the same or lower value as in dispBDT training
 if [[ $RECID == "1" ]]; then
     MOPT="$MOPT -maxloss=0.1 -minfui=0."
@@ -141,15 +137,16 @@ elif [[ $RECID == "2" ]]; then
     MOPT="$MOPT -maxdistfraction=0.75"
 else
     MOPT="$MOPT -maxloss=0.2 -minfui=0."
-    MOPT="$MOPT -maxdistfraction=0.80"
+# 2025, June - remove -maxdistfraction
+#    MOPT="$MOPT -maxdistfraction=0.80"
 fi
 
 #########################################
 # disp reconstruction
 #
 MVATYPE="BDT"
 # disp main directory name
-DISPSUBDIR="DISPBDT/${MVATYPE}disp.${ARRAY}.R1"
+DISPSUBDIR="DISPBDT/${MVATYPE}disp.${ARRAY}.G1"
 echo "CHECKING ${CTA_USER_DATA_DIR}/analysis/AnalysisData/${DSET}/${DISPSUBDIR/${ARRAY}/HYPERARRAY}"
 if [[ -d ${CTA_USER_DATA_DIR}/analysis/AnalysisData/${DSET}/${DISPSUBDIR/${ARRAY}/HYPERARRAY} ]]; then
     DISPSUBDIR="${DISPSUBDIR/${ARRAY}/HYPERARRAY}"
@@ -161,7 +158,7 @@ echo "DISPDIR $DISPSUBDIR"
 #########################################
 # unpack disp XML files for all telescope
 # types to tmpdir (not all of them might be used)
-for ML in ${MVATYPE}Disp ${MVATYPE}DispError ${MVATYPE}DispEnergy ${MVATYPE}DispCore
+for ML in ${MVATYPE}Disp ${MVATYPE}DispError ${MVATYPE}DispEnergy ${MVATYPE}DispCore ${MVATYPE}DispSign
 do
    MLDDIR="${CTA_USER_DATA_DIR}/analysis/AnalysisData/${DSET}/${DISPSUBDIR}/${ML}/${MCAZ}/"
    echo "Unpacking ${ML} from ${MLDDIR}"
@@ -195,6 +192,11 @@ MOPT="$MOPT -tmva_nimages_max_stereo_reconstruction=100 -tmva_filename_stereo_re
 DISPERRORDIR="${TMPDIR}/${MVATYPE}DispError/${MCAZ}/${MVATYPE}DispError_${MVATYPE}_"
 MOPT="$MOPT -tmva_filename_disperror_reconstruction $DISPERRORDIR -tmva_disperror_weight 50"
 
+##########################################################################################################
+# options for DISP method (direction sign)
+DISPSIGNDIR="${TMPDIR}/${MVATYPE}DispSign/${MCAZ}/${MVATYPE}DispSign_${MVATYPE}_"
+MOPT="$MOPT -tmva_filename_dispsign_reconstruction $DISPSIGNDIR"
+
 ##########################################################################################################
 # options for DISP method (core)
 # (switch on for single-telescope analysis)

analysis/CTA.TMVA.qsub_train.sh

@@ -20,7 +20,6 @@ do
 
     echo ${PFIL}.runparameter
 
-    echo "JOB ID ${SGE_JOB_ID}" > "${PFIL}".log
     ${EVNDISPSYS}/bin/trainTMVAforGammaHadronSeparation "${PFIL}".runparameter >> "${PFIL}".log
 
     CDIR=$(dirname "$PFIL".log)