Skip to content

Commit 70462e4

Browse files
GernotMaierGernotMaier
authored
Merge pull request #58 from Eventdisplay/v2.4-dev1
v2.4 dev1 - mostly for construction layouts
2 parents 69b4891 + 8961899 commit 70462e4

14 files changed

+230
-265
lines changed

‎CTA.mainRunScriptsReduced.sh‎

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ if [ $# -lt 2 ]; then
2424
prod6-South-20deg
2525
2626
run modes:
27-
MAKETABLES DISPBDT ANATABLES PREPARETMVA TRAIN ANGRES QC CUTS PHYS CLEANUP
27+
MAKETABLES PREPAREFILELISTS DISPBDT ANATABLES PREPARETMVA TRAIN ANGRES QC CUTS PHYS CLEANUP
2828
2929
optional run modes: TRAIN_RECO_QUALITY TRAIN_RECO_METHOD
3030
@@ -44,8 +44,10 @@ fi
4444
RECID="0"
4545

4646
# run scripts are collected here
47-
RUNSCRIPTDIR="${CTA_USER_LOG_DIR}/jobs/$(uuidgen)"
48-
mkdir -p ${RUNSCRIPTDIR}
47+
if [[ ${RUN} != "CLEANUP" ]] && [[ ${RUN} != "PREPAREFILELISTS" ]]; then
48+
RUNSCRIPTDIR="${CTA_USER_LOG_DIR}/jobs/$(uuidgen)"
49+
mkdir -p ${RUNSCRIPTDIR}
50+
fi
4951

5052
if [[ ${RUN} == "MAKETABLES" ]] || [[ ${RUN} == "DISPBDT" ]] || [[ ${RUN} == "ANATABLES" ]] || [[ ${RUN} == "PREPARETMVA" ]]; then
5153
./CTA.runAnalysis.sh ${P2} ${RUN} ${RECID} 2 2 2 2 ${RUNSCRIPTDIR}
@@ -56,6 +58,9 @@ if [[ ${RUN} == "MAKETABLES" ]] || [[ ${RUN} == "DISPBDT" ]] || [[ ${RUN} == "AN
5658
fi
5759
elif [[ ${RUN} == "CLEANUP" ]]; then
5860
./CTA.runAnalysis.sh ${P2} ${RUN} ${RECID} 2 2 2 2 ${RUNSCRIPTDIR}
61+
elif [[ ${RUN} == "PREPAREFILELISTS" ]]; then
62+
./CTA.runAnalysis.sh ${P2} ${RUN} ${RECID} 2 2 2 2 ${RUNSCRIPTDIR}
63+
./CTA.runAnalysis.sh ${P2}-sub ${RUN} ${RECID} 2 2 2 2 ${RUNSCRIPTDIR}
5964
else
6065
while IFS= read -r mult
6166
do
@@ -71,5 +76,7 @@ else
7176
fi
7277
fi
7378

74-
echo "#####"
75-
echo "RUNSCRIPTDIR: ${RUNSCRIPTDIR}/${RUN}"
79+
if [[ ${RUN} != "CLEANUP" ]] && [[ ${RUN} != "PREPAREFILELISTS" ]]; then
80+
echo "#####"
81+
echo "RUNSCRIPTDIR: ${RUNSCRIPTDIR%/}/${RUN}"
82+
fi

‎CTA.runAnalysis.sh‎

Lines changed: 16 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ then
2828
prod6-North-20deg prod6-North-40deg prod6-North-52deg prod6-North-60deg
2929
prod6-South-20deg
3030
31-
possible run modes are EVNDISP MAKETABLES DISPBDT/DISPMLP ANATABLES PREPARETMVA TRAIN ANGRES QC CUTS PHYS CLEANUP
31+
possible run modes are EVNDISP MAKETABLES PREPAREFILELISTS DISPBDT ANATABLES PREPARETMVA TRAIN ANGRES QC CUTS PHYS CLEANUP
3232
3333
optional run modes: TRAIN_RECO_QUALITY TRAIN_RECO_METHOD
3434
@@ -60,9 +60,6 @@ echo "Telescope multiplicities: LST ${LST} MST ${MST} SST ${SST} SCMST ${SCMST}"
6060
# even without using gridengine: do not remove this
6161
QSUBOPT="_M_P_X_cta_high_X__M_js_X_9"
6262

63-
#####################################
64-
# output directory for script parameter files
65-
mkdir -p "${PDIR%/}/tempRunParameterDir/"
6663

6764
#####################################
6865
# analysis dates and table dates
@@ -268,7 +265,6 @@ then
268265
ARRAY=( "subArray.prod5.South-SV3f-v3-sub.list" )
269266
ARRAY=( "subArray.prod5.South-AlphaC8aj-sub.list" )
270267
ARRAY=( "subArray.prod5.South-Beta-sub.list" )
271-
# ARRAY=( "subArray.prod5.South-70SSTs-sub.list" )
272268
fi
273269
if [[ $P2 == *"Hyper"* ]] || [[ $P2 == *"hyper"* ]]; then
274270
# ARRAY=( "subArray.prod5.South-HyperSST.list" )
@@ -296,6 +292,8 @@ then
296292
ARRAYDIR="prod5"
297293
TDATE="g20250826"
298294
ANADATE="${TDATE}"
295+
ANADATE="g20250905"
296+
ANADATE="g20250906"
299297
TMVADATE="${ANADATE}"
300298
EFFDATE="${ANADATE}"
301299
PHYSDATE="${EFFDATE}"
@@ -347,6 +345,7 @@ then
347345
ARRAYDIR="prod6"
348346
TDATE="g20250822"
349347
ANADATE="${TDATE}"
348+
ANADATE="g20250906"
350349
TMVADATE="${ANADATE}"
351350
EFFDATE="${ANADATE}"
352351
PHYSDATE="${EFFDATE}"
@@ -358,7 +357,7 @@ fi
358357
if [[ -z ${PHYSDATE} ]]; then
359358
PHYSDATE=${EFFDATE}
360359
fi
361-
# should be either onSource or cone (default is cone)
360+
# NOT USED ANYMORE! Keep 'cone'
362361
OFFAXIS="cone"
363362

364363
#####################################
@@ -427,7 +426,12 @@ fi
427426
# remove from PHYS directory any unreasonable files (e.g. LST4 requirement for 2 LST array)
428427
if [[ $RUN == "CLEANUP" ]]; then
429428
PHYSDIR="${CTA_USER_DATA_DIR}/analysis/AnalysisData/${SITE}${EDM}/Phys-${PHYSDATE}"
430-
./utilities/removeUnreaseonablePhysFiles.sh ${PHYSDIR}
429+
./utilities/removeUnreasonablePhysFiles.sh ${PHYSDIR}
430+
exit
431+
fi
432+
# Prepare file lists required for DispBDT training
433+
if [[ $RUN == "PREPAREFILELISTS" ]]; then
434+
./analysis/CTA.separateDispTrainingEvndispFiles.sh "${SITE}${EDM}" "${ARRAYDIR}/$ARRAY"
431435
exit
432436
fi
433437
##########################################
@@ -436,6 +440,9 @@ if [[ ! -e ${ARRAYDIR}/$ARRAY ]]; then
436440
echo "Error: array file not found: ${ARRAYDIR}/$ARRAY"
437441
exit
438442
fi
443+
#####################################
444+
# output directory for script parameter files
445+
mkdir -p "${PDIR%/}/tempRunParameterDir/"
439446
NXARRAY=$(cat ${ARRAYDIR}/$ARRAY)
440447
NFILARRAY=${PDIR%/}/tempRunParameterDir/temp.$ARRAY.list
441448
rm -f "$NFILARRAY"
@@ -448,13 +455,8 @@ done
448455
# dispBDT training
449456
if [[ $RUN == "DISP"* ]]
450457
then
451-
if [[ $RUN == "DISPMLP" ]]; then
452-
BDTDIR="MLPdisp."
453-
RUNPAR="${CTA_EVNDISP_AUX_DIR}/ParameterFiles/TMVA.MLPDisp.runparameter"
454-
else
455-
BDTDIR="BDTdisp."
456-
RUNPAR="${CTA_EVNDISP_AUX_DIR}/ParameterFiles/TMVA.BDTDisp.runparameter"
457-
fi
458+
BDTDIR="BDTdisp."
459+
RUNPAR="${CTA_EVNDISP_AUX_DIR}/ParameterFiles/TMVA.BDTDisp.runparameter"
458460
QCPAR="${CTA_EVNDISP_AUX_DIR}/ParameterFiles/TMVA.BDTDispQualityCuts.runparameter"
459461
DDIR="${CTA_USER_DATA_DIR}/analysis/AnalysisData/${SITE}${EDM}/"
460462
for A in $NXARRAY
@@ -610,7 +612,6 @@ do
610612
then
611613
./CTA.prepareTMVA.sub_train.sh \
612614
"$NFILARRAY" \
613-
$OFFAXIS \
614615
${SITE}${EDM} \
615616
"$PARA" \
616617
$QSUBOPT \
@@ -635,7 +636,6 @@ do
635636
./CTA.TMVA.sub_train.sh \
636637
"$TMVA_RUN_MODE" \
637638
"$NFILARRAY" \
638-
$OFFAXIS \
639639
${SITE}${EDM} \
640640
"$PARA" \
641641
$QSUBOPT \

‎NIM-South.dat‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,3 @@
1-
2 1 2 2
2-
2 1 3 2
3-
2 1 3 3
4-
2 1 4 2
5-
2 1 4 3
6-
2 1 5 3
71
2 2 2 2
82
2 2 3 2
93
2 2 3 3

‎analysis/CTA.DISPTRAINING.sub_analyse.sh‎

Lines changed: 6 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -169,7 +169,8 @@ do
169169

170170
let "NSTEP = $NSTEP + 1"
171171
# output directory (match CTA.MSCW_ENERGY.qsub_analyse_MC.sh)
172-
OFFDIR=${ODIR}.G${NSTEP}
172+
# e.g. J1
173+
OFFDIR=${ODIR}.J${NSTEP}
173174
####################
174175
# output directory
175176
TDIR="${OFFDIR}/${MLP}/${MCAZ}/"
@@ -184,31 +185,11 @@ do
184185
echo "=========================================================================="
185186

186187
####################
187-
# input file list
188-
rm -f $SHELLDIR/tempList.list
189-
find $CTA_USER_DATA_DIR/analysis/AnalysisData/$DSET/$ARRAY/${EVNDISP}/gamma_cone/ -name "*[_,.]${MCAZ}*.root" > $SHELLDIR/tempList.list
190-
NFIL=`wc -l $SHELLDIR/tempList.list | awk '{print $1}'`
191-
echo "Total number of files available: $NFIL"
192-
# only use NN% of all evndisp files for training
193-
# (for LSTs: use more, as there are less telescopes)
194-
# South: 10%
195-
# North: 20%
196-
k=`expr 0.2*$NFIL | bc`
197-
k=$(echo $NFIL | awk '{printf "%d\n",$1*0.20}')
198-
# SV1 arrays: mix directories by hand!!
199-
if [[ $ARRAY == *"SV1"* ]]
200-
then
201-
k=$(echo $NFIL)
202-
fi
203-
# experimental only
204-
if [[ $ARRAY == *"DISP"* ]]
205-
then
206-
k=$(echo $NFIL | awk '{printf "%d\n",$1*0.50}')
207-
fi
188+
# input file list (based on lists generated by separateDispTrainingEvndispFiles)
208189
TLIST="$SHELLDIR/EDISP-$DSET-$ARRAY-$SCALING-$MCAZ-$TELTYPE-$MLP-$NSTEP.list"
209-
rm -f $TLIST
210-
shuf -n $k $SHELLDIR/tempList.list > $TLIST
211-
echo "List of $k input files for training: $TLIST"
190+
cp -f $CTA_USER_DATA_DIR/analysis/AnalysisData/$DSET/$ARRAY/EVNDISP.TRAIN/gamma_cone_${MCAZ}.list ${TLIST}
191+
shuf ${TLIST} -o ${TLIST}
192+
echo "List of $(wc -l ${TLIST}) input files for training: $TLIST"
212193

213194
####################
214195
# prepare run scripts

‎analysis/CTA.MSCW_ENERGY.qsub_analyse_MC.sh‎

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -123,11 +123,19 @@ else
123123
exit
124124
fi
125125

126+
# number of telescopes
127+
NTEL=$(grep -vE '^\s*#|^$' $LISFILE | wc -l)
128+
echo "Number of telescopes in array: $NTEL"
129+
126130

127131
#########################################
128132
# options for simple stereo reconstruction
129133
MOPT="$MOPT -redo_stereo_reconstruction -sub_array_sim_telarray_counting $LISFILE"
130-
MOPT="$MOPT -minangle_stereo_reconstruction=5."
134+
if [[ $NTEL == "2" ]]; then
135+
MOPT="$MOPT -minangle_stereo_reconstruction=15."
136+
else
137+
MOPT="$MOPT -minangle_stereo_reconstruction=5."
138+
fi
131139
# IMPORTANT: this must be the same or lower value as in dispBDT training
132140
if [[ $RECID == "1" ]]; then
133141
MOPT="$MOPT -maxloss=0.1 -minfui=0."
@@ -137,16 +145,14 @@ elif [[ $RECID == "2" ]]; then
137145
MOPT="$MOPT -maxdistfraction=0.75"
138146
else
139147
MOPT="$MOPT -maxloss=0.2 -minfui=0."
140-
# 2025, June - remove -maxdistfraction
141-
# MOPT="$MOPT -maxdistfraction=0.80"
142148
fi
143149

144150
#########################################
145151
# disp reconstruction
146152
#
147153
MVATYPE="BDT"
148154
# disp main directory name
149-
DISPSUBDIR="DISPBDT/${MVATYPE}disp.${ARRAY}.G1"
155+
DISPSUBDIR="DISPBDT/${MVATYPE}disp.${ARRAY}.J1"
150156
echo "CHECKING ${CTA_USER_DATA_DIR}/analysis/AnalysisData/${DSET}/${DISPSUBDIR/${ARRAY}/HYPERARRAY}"
151157
if [[ -d ${CTA_USER_DATA_DIR}/analysis/AnalysisData/${DSET}/${DISPSUBDIR/${ARRAY}/HYPERARRAY} ]]; then
152158
DISPSUBDIR="${DISPSUBDIR/${ARRAY}/HYPERARRAY}"
@@ -256,7 +262,6 @@ fi
256262
if [ -e $TMPDIR/iList.list ] && [ -e $TMPDIR/$TFIL.root ]
257263
then
258264
$EVNDISPSYS/bin/logFile mscwTableList $TMPDIR/$TFIL.root $TMPDIR/iList.list
259-
rm -f $TMPDIR/iList.list
260265
rm -f $IFIL
261266
fi
262267

‎analysis/CTA.MSCW_ENERGY.sub_analyse_MC.sh‎

Lines changed: 28 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,7 @@
1-
#!/bin/sh
1+
#!/bin/bash
22
#
33
# script to analyse CTA MC files with lookup tables
44
#
5-
#
65
SUBC="condor"
76
h_cpu="11:29:00"
87
h_vmem="8000M"
@@ -89,48 +88,43 @@ for SUBAR in $VARRAY
8988
do
9089
echo "STARTING ARRAY $SUBAR"
9190

92-
# output directory
9391
ODIR="${CTA_USER_DATA_DIR}/analysis/AnalysisData/${DSET}/${SUBAR}/${ANADIR}"
9492
mkdir -p ${ODIR}
9593

96-
#########################################
97-
# loop over all particle types
94+
#########################################
95+
# loop over all particle types
9896
for ((m = 0; m < $NPART; m++ ))
9997
do
10098
PART=${VPART[$m]}
10199

102-
# delete all old files (data and log files) for the particle type and azimuth angle
103-
rm -f ${ODIR}/${PART}*ID${RECID}_${MCAZ}*
100+
# delete all old files (data and log files) for the particle type and azimuth angle
101+
rm -f ${ODIR}/${PART}*ID${RECID}_${MCAZ}*
104102

105-
# take $FILEN files and combine them into one mscw file
106-
FILEN=125
107-
if [ $PART = "proton" ]
108-
then
109-
FILEN=500
110-
fi
103+
# take $FILEN files and combine them into one mscw file
104+
FILEN=125
105+
if [ $PART = "proton" ]
106+
then
107+
FILEN=500
108+
fi
111109

112-
#########################################
113-
# input files lists
114-
115-
TMPLIST=${ODIR}/$PART$NC"."$SUBAR"_ID"${RECID}${MCAZ}"-"$DSET".list"
116-
rm -f $TMPLIST
117-
echo $TMPLIST ${MCAZ}
118-
find $CTA_USER_DATA_DIR/analysis/AnalysisData/$DSET/$SUBAR/EVNDISP/$PART/ -name "*[0-9]*[\.,_]${MCAZ}*.root" > $TMPLIST
119-
NTMPLIST=`wc -l $TMPLIST | awk '{print $1}'`
120-
echo "total number of files for particle type $PART ($MCAZ) : $NTMPLIST"
121-
NJOBTOT=$(( NTMPLIST / (FILEN - 1)))
122-
if [[ $NJOBTOT == 0 ]]; then
123-
NJOBTOT=1
124-
fi
125-
echo "total number of jobs: $NJOBTOT"
126-
127-
# output file name for mscw_energy
128-
TFIL=$PART$NC"."$SUBAR"_ID${RECID}_${MCAZ}-"$DSET
129-
130-
# skeleton script
110+
#########################################
111+
# input files lists
112+
TMPLIST=$CTA_USER_DATA_DIR/analysis/AnalysisData/$DSET/$SUBAR/EVNDISP.ANALYSIS/${PART}_${MCAZ}.list
113+
NTMPLIST=$(wc -l < "$TMPLIST")
114+
echo "total number of files for particle type $PART ($MCAZ) : $NTMPLIST"
115+
NJOBTOT=$(( NTMPLIST / (FILEN - 1)))
116+
if [[ $NJOBTOT == 0 ]]; then
117+
NJOBTOT=1
118+
fi
119+
echo "total number of jobs: $NJOBTOT"
120+
121+
# output file name for mscw_energy
122+
TFIL=$PART$NC"."$SUBAR"_ID${RECID}_${MCAZ}-"$DSET
123+
124+
# skeleton script
131125
FSCRIPT="CTA.MSCW_ENERGY.qsub_analyse_MC"
132126

133-
# name of script actually submitted to the queue
127+
# name of script actually submitted to the queue
134128
FNAM="$SHELLDIR/MSCW.ana-$DSET-ID$RECID-$PART-${MCAZ}-array$SUBAR-$6"
135129

136130
sed -e "s|TABLEFILE|$TABLE|" \
@@ -148,7 +142,7 @@ do
148142
echo "run script written to $FNAM.sh"
149143
echo "queue log and error files written to $QLOG"
150144

151-
# submit the job
145+
# submit the job
152146
if [[ $SUBC == *qsub* ]]; then
153147
qsub $QSUBOPT -t 1-$NJOBTOT:1 -l h_cpu=${h_cpu} -l h_rss=${h_vmem} -l tmpdir_size=${tmpdir_size} -V -o $QLOG -e $QLOG "$FNAM.sh"
154148
elif [[ $SUBC == *condor* ]]; then
@@ -161,5 +155,3 @@ do
161155
fi
162156
done
163157
done
164-
165-
exit

0 commit comments

Comments
 (0)