Skip to content

Commit d96feb5

Browse files
GernotMaierGernotMaier
authored
Merge pull request #41 from Eventdisplay/v1.10-dev-3
SV array layout analysis; improved condor submission
2 parents 042b789 + ee6e079 commit d96feb5

18 files changed

+241
-131
lines changed

‎.pre-commit-config.yaml‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
repos:
2+
# https://pycqa.github.io/isort/docs/configuration/black_compatibility.html#integration-with-pre-commit
3+
- repo: https://github.com/pycqa/isort
4+
rev: 5.12.0
5+
hooks:
6+
- id: isort
7+
args: ["--profile", "black", "--filter-files"]
8+
- repo: https://github.com/psf/black
9+
rev: 23.7.0
10+
hooks:
11+
- id: black
12+
args: ["--line-length=100"]
13+
# https://black.readthedocs.io/en/stable/guides/using_black_with_other_tools.html?highlight=other%20tools#flake8
14+
- repo: https://github.com/PyCQA/flake8
15+
rev: 6.1.0
16+
hooks:
17+
- id: flake8
18+
args: ["--max-line-length=100", "--extend-ignore=E203,E712"]
19+
# https://github.com/pre-commit/pre-commit-hooks
20+
- repo: https://github.com/pre-commit/pre-commit-hooks
21+
rev: v4.4.0
22+
hooks:
23+
- id: trailing-whitespace
24+
- id: end-of-file-fixer

‎CTA.mainRunScriptsReduced.sh‎

Lines changed: 28 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -33,20 +33,38 @@ P2="$1"
3333
# run mode
3434
RUN="$2"
3535

36+
SITE="South"
37+
if [[ $P2 == *"North"* ]]; then
38+
SITE="North"
39+
fi
40+
41+
RECID="0"
42+
3643
# run scripts are collected here
3744
RUNSCRIPTDIR="${CTA_USER_LOG_DIR}/jobs/$(uuidgen)"
3845
mkdir -p ${RUNSCRIPTDIR}
3946

4047
if [[ ${RUN} == "MAKETABLES" ]] || [[ ${RUN} == "DISPBDT" ]] || [[ ${RUN} == "ANATABLES" ]] || [[ ${RUN} == "PREPARETMVA" ]]; then
41-
./CTA.runAnalysis.sh ${P2} ${RUN} 0 2 2 2 2 ${RUNSCRIPTDIR}
42-
./CTA.runAnalysis.sh ${P2}-sub ${RUN} 0 2 2 2 2 ${RUNSCRIPTDIR}
48+
./CTA.runAnalysis.sh ${P2} ${RUN} ${RECID} 2 2 2 2 ${RUNSCRIPTDIR}
49+
if [[ $SITE == "South" ]]; then
50+
./CTA.runAnalysis.sh ${P2}-sub ${RUN} ${RECID} 2 2 2 2 ${RUNSCRIPTDIR}
51+
elif [[ $SITE == *"North"* ]]; then
52+
./CTA.runAnalysis.sh ${P2}-LST ${RUN} ${RECID} 2 2 2 2 ${RUNSCRIPTDIR}
53+
fi
4354
else
44-
while IFS= read -r mult
45-
do
46-
./CTA.runAnalysis.sh ${P2} ${RUN} 0 $mult ${RUNSCRIPTDIR}
47-
done < NIM-South.txt
48-
while IFS= read -r mult
49-
do
50-
./CTA.runAnalysis.sh ${P2}-sub ${RUN} 0 $mult ${RUNSCRIPTDIR}
51-
done < NIM-South-sub.txt
55+
while IFS= read -r mult
56+
do
57+
./CTA.runAnalysis.sh ${P2} ${RUN} ${RECID} $mult ${RUNSCRIPTDIR}
58+
done < NIM-${SITE}.dat
59+
if [[ $SITE == "South" ]]; then
60+
while IFS= read -r mult
61+
do
62+
./CTA.runAnalysis.sh ${P2}-sub ${RUN} ${RECID} $mult ${RUNSCRIPTDIR}
63+
done < NIM-South-sub.dat
64+
elif [[ $SITE == "North" ]]; then
65+
./CTA.runAnalysis.sh ${P2}-LST ${RUN} ${RECID} 2 2 2 2 ${RUNSCRIPTDIR}
66+
fi
5267
fi
68+
69+
echo "#####"
70+
echo "RUNSCRIPTDIR: ${RUNSCRIPTDIR}/${RUN}"

‎CTA.runAnalysis.sh‎

Lines changed: 34 additions & 61 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010
if [ $# -lt 2 ]
1111
then
1212
echo "
13-
./CTA.runbAnalysis.sh <S/S40deg> <run mode> [recid] \\\\
13+
./CTA.runAnalysis.sh <production> <run mode> [recid] \\\\
1414
[min number of LSTs] [min number of MSTs] [min number of SSTs] [min number of SCMSTs] [job_dir]
1515
1616
Prod3b analysis:
@@ -185,37 +185,6 @@ then
185185
TMVADATE="${ANADATE}"
186186
EFFDATE="${TMVADATE}"
187187
###############################################################
188-
###############################################################
189-
# PROD5 Analysis
190-
# NOTE: for N, use prod5b analysis
191-
# prod5-N
192-
# prod5-N-moon (NSB5x)
193-
elif [[ $P2 == "prod5-N"* ]]
194-
then
195-
if [[ $P2 == *"moon"* ]]; then
196-
SITE="prod5-LaPalma-20deg-NSB5x"
197-
else
198-
SITE="prod5-LaPalma-20deg"
199-
fi
200-
EDM="-sq08-LL"
201-
ARRAY=( "subArray.prod5.North-MSTF-Arrays.list" )
202-
ARRAY=( "subArray.prod5.North-XST.list" )
203-
# prod5-prod5b comparision
204-
ARRAY=( "subArray.prod5-prod5b.North.list" )
205-
ARRAY=( "subArray.prod5.North-PB.list" )
206-
if [[ $P2 == *"Hyper"* ]]; then
207-
ARRAY=( "subArray.prod5.North-Hyper.list" )
208-
fi
209-
if [[ $P2 == *"LST"* ]]; then
210-
ARRAY=( "subArray.prod5.North-LST.list" )
211-
fi
212-
ARRAYDIR="prod5"
213-
TDATE="g20210610"
214-
ANADATE="${TDATE}"
215-
TMVADATE="${ANADATE}"
216-
EFFDATE="${TMVADATE}"
217-
PHYSDATE="${EFFDATE}"
218-
####################################
219188
# PROD5 Analysis
220189
# prod5b-N (including additional telescopes)
221190
elif [[ $P2 == "prod5b-N"* ]]
@@ -230,28 +199,36 @@ then
230199
if [[ $P2 == *"moon"* ]]; then
231200
SITE="${SITE}-NSB5x"
232201
fi
233-
EDM="-sq40-LL"
234202
# lin is default reconstruction for North
235-
EDM="-lin50-LL"
203+
EDM="-lin51-LL"
236204
if [[ $P2 == *"DL2plus"* ]]; then
237205
EDM="-sq10-LL-DL2plus"
238206
fi
239207
ARRAY=( "subArray.prod5.North-Alpha.list" )
240208
if [[ $P2 == *"sub"* ]]; then
241209
ARRAY=( "subArray.prod5.North-Alpha-sub.list" )
242-
fi
243-
if [[ $P2 == *"LST"* ]]; then
244-
ARRAY=( "subArray.prod5.North-LST.list" )
245-
fi
246-
if [[ $P2 == *"XST"* ]]; then
210+
elif [[ $P2 == *"XST"* ]]; then
247211
ARRAY=( "subArray.prod5.North-XST.list" )
212+
elif [[ $P2 == *"SV"* ]]; then
213+
ARRAY=( "subArray.prod5.North-SV.list" )
214+
if [[ $P2 == *"LST"* ]]; then
215+
ARRAY=( "subArray.prod5.North-SV-LST.list" )
216+
ARRAY=( "subArray.prod5.North-SV-LST34.list" )
217+
# combined lst mst multiplicity
218+
# ARRAY=( "subArray.prod5.North-SV-MSTLST.list" )
219+
fi
220+
if [[ $P2 == *"MST"* ]]; then
221+
ARRAY=( "subArray.prod5.North-SV-MST.list" )
222+
fi
223+
elif [[ $P2 == *"LST"* ]]; then
224+
ARRAY=( "subArray.prod5.North-LST.list" )
248225
fi
249226
ARRAYDIR="prod5"
250-
TDATE="g20221204"
227+
TDATE="g20230614"
251228
ANADATE="${TDATE}"
229+
ANADATE="g20231204"
252230
TMVADATE="${ANADATE}"
253231
EFFDATE="${ANADATE}"
254-
EFFDATE="g20230206"
255232
PHYSDATE="${EFFDATE}"
256233
####################################
257234
# prod5 - Paranal
@@ -269,18 +246,17 @@ then
269246
if [[ $P2 == *"moon"* ]]; then
270247
SITE="${SITE}-NSB5x"
271248
fi
272-
EDM="-sq50-LL"
273-
EDM="-sq60-LL"
249+
EDM="-sq51-LL"
274250
if [[ $P2 == *"DL2plus"* ]]; then
275251
EDM="-sq10-LL-DL2plus"
276252
fi
277253
ARRAY=( "subArray.prod5.South-BL.list" )
278-
ARRAY=( "subArray.prod5.South-Alpha.list" )
279254
ARRAY=( "subArray.prod5.South-Alpha-2LSTs42SSTs.list" )
280255
ARRAY=( "subArray.prod5.South-Alpha-2LSTs42SSTsBeta.list" )
256+
ARRAY=( "subArray.prod5.South-Alpha.list" )
281257
if [[ $P2 == *"sub"* ]]; then
282-
ARRAY=( "subArray.prod5.South-Alpha-sub.list" )
283258
ARRAY=( "subArray.prod5.South-Alpha-2LSTs42SSTsBeta-sub.list")
259+
ARRAY=( "subArray.prod5.South-Alpha-sub.list" )
284260
fi
285261
if [[ $P2 == *"Hyper"* ]] || [[ $P2 == *"hyper"* ]]; then
286262
ARRAY=( "subArray.prod5.South-Hyper.list" )
@@ -298,14 +274,11 @@ then
298274
ARRAY=( "subArray.prod5.South-SV0.list" )
299275
fi
300276
ARRAYDIR="prod5"
301-
TDATE="g20221202"
277+
TDATE="g20230823"
302278
ANADATE="${TDATE}"
303-
ANADATE="g20221204"
304279
TMVADATE="${ANADATE}"
305280
EFFDATE="${ANADATE}"
306-
EFFDATE="g20230206"
307281
PHYSDATE="${EFFDATE}"
308-
PHYSDATE="g20230206"
309282
####################################
310283
# prod6 - Paranal and LaPalma
311284
elif [[ $P2 == "prod6"* ]]
@@ -460,7 +433,7 @@ then
460433
99 \
461434
$QCPAR \
462435
$QSUBOPT \
463-
${PDIR}
436+
${PDIR}/${RUN}
464437
cd ../
465438
done
466439
exit
@@ -495,7 +468,7 @@ do
495468
${AZ} \
496469
${NIMAGESMIN} \
497470
$QSUBOPT \
498-
${PDIR}
471+
${PDIR}/${RUN}
499472
cd ../
500473
continue
501474
##########################################
@@ -515,7 +488,7 @@ do
515488
${AZ} \
516489
${NIMAGESMIN} \
517490
$QSUBOPT \
518-
${PDIR}
491+
${PDIR}/${RUN}
519492
cd ../
520493
continue
521494
fi
@@ -582,7 +555,7 @@ do
582555
fi
583556
echo "OFFAXISFINEBINNING $BFINEBINNING" >> "$PARA"
584557
if [[ ${RUN} == "CUTS" ]] && [[ ${OOTIME} == "50h" ]]; then
585-
if [[ $P2 == *"DL2plus"* ]] || [[ $EDM == *"sq70"* ]]; then
558+
if [[ $P2 == *"DL2plus"* ]] || [[ $EDM == *"sq51"* ]] || [[ $EDM == *"lin51"* ]]; then
586559
echo "DL2FILLING FULLTREES" >> "$PARA"
587560
else
588561
echo "DL2FILLING DL2" >> "$PARA"
@@ -604,7 +577,7 @@ do
604577
"$PARA" \
605578
$QSUBOPT \
606579
$AZ \
607-
${PDIR}
580+
${PDIR}/${RUN}
608581
fi
609582
##########################################
610583
# train BDTs
@@ -620,7 +593,7 @@ do
620593
"$PARA" \
621594
$QSUBOPT \
622595
$AZ \
623-
${PDIR}
596+
${PDIR}/${RUN}
624597
fi
625598
##########################################
626599
# IRFs: angular resolution
@@ -635,7 +608,7 @@ do
635608
${SITE}${EDM} \
636609
2 \
637610
$QSUBOPT \
638-
${PDIR} \
611+
${PDIR}/${RUN} \
639612
$AZ
640613
fi
641614
##########################################
@@ -651,7 +624,7 @@ do
651624
${SITE}${EDM} \
652625
3 \
653626
$QSUBOPT \
654-
${PDIR} \
627+
${PDIR}/${RUN} \
655628
$AZ
656629
fi
657630
##########################################
@@ -669,7 +642,7 @@ do
669642
${SITE}${EDM} \
670643
0 \
671644
$QSUBOPT \
672-
${PDIR} \
645+
${PDIR}/${RUN} \
673646
$AZ
674647
# low multiplicity runs use 95% max signal efficiency (lower requirements on resolution)
675648
else
@@ -681,7 +654,7 @@ do
681654
${SITE}${EDM} \
682655
0 \
683656
$QSUBOPT \
684-
${PDIR} \
657+
${PDIR}/${RUN} \
685658
$AZ
686659
fi
687660
##########################################
@@ -700,7 +673,7 @@ do
700673
${SITE}${EDM} \
701674
$BFINEBINNING \
702675
$PHYSDATE \
703-
${PDIR} \
676+
${PDIR}/${RUN} \
704677
$QSUBOPT
705678
else
706679
./CTA.WPPhysWriter.sub.sh \
@@ -713,7 +686,7 @@ do
713686
${SITE}${EDM} \
714687
$BFINEBINNING \
715688
$PHYSDATE \
716-
${PDIR} \
689+
${PDIR}/${RUN} \
717690
$QSUBOPT
718691
fi
719692
# unknown run set

NIM-South-sub.txt renamed to NIM-North.dat

File renamed without changes.

‎NIM-South-sub.dat‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
2 2 2 2
2+
3 3 3 3
3+
4 4 4 4

NIM-South.txt renamed to NIM-South.dat

File renamed without changes.

‎README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -115,7 +115,7 @@ The script `./CTA.mainRunScriptsReduced.sh` does the following:
115115
- read a list of arrays from a subdirectory specificed for your data set in ./CTA.runAnalysis.sh (e.g., prod3b/subArray.prod3b.South-SCT.list)
116116
- execute scripts to submit jobs from the ./analysis directory
117117
- all output products are written to *${CTA_USER_DATA_DIR}/analysis/AnalysisData/${DSET}*
118-
- for all telescope multiplicity dependent analysis, this is done for the multiplicities defined in `NIM-South.txt` and `NIM-South-sub.txt`.
118+
- for all telescope multiplicity dependent analysis, this is done for the multiplicities defined in `NIM-South.dat` and `NIM-South-sub.dat`.
119119

120120
On the list of arrays:
121121
- arrays are defined by the telescope numbering as defined during the simulations.

‎analysis/CTA.EFFAREA.qsub_analyse_list.sh‎

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,7 @@ if [[ $DSET == *"prod6"* ]]; then
8383
TELLIST=${CTA_EVNDISP_AUX_DIR}/DetectorGeometry/CTA.prod6.teltypes.dat
8484
elif [[ $DSET == *"prod5"* ]]; then
8585
TELLIST=${CTA_EVNDISP_AUX_DIR}/DetectorGeometry/CTA.prod5.teltypes.dat
86+
# TELLIST=${CTA_EVNDISP_AUX_DIR}/DetectorGeometry/CTA.prod5SV.teltypes.dat
8687
elif [[ $DSET == *"prod4"* ]]; then
8788
TELLIST=${CTA_EVNDISP_AUX_DIR}/DetectorGeometry/CTA.prod4.teltypes.dat
8889
elif [[ $DSET == *"prod3"* ]]; then
@@ -158,11 +159,6 @@ mkdir -p $ODIR
158159
######################################################################
159160
# maximum core distance to a telescope
160161
######################################################################
161-
MAXCDISTANCE="500."
162-
if [ $RECID = "1" ]
163-
then
164-
MAXCDISTANCE="200."
165-
fi
166162
MAXCDISTANCE="600."
167163
if [ $RECID = "1" ]
168164
then

‎analysis/CTA.MSCW_ENERGY.qsub_analyse_MC.sh‎

Lines changed: 22 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -134,16 +134,31 @@ else
134134
MOPT="$MOPT -minangle_stereo_reconstruction=10."
135135
fi
136136
# IMPORTANT: this must be the same or lower value as in dispBDT training
137-
MOPT="$MOPT -maxloss=0.2 -minfui=0."
138-
MOPT="$MOPT -maxdistfraction=0.80"
137+
if [[ $RECID == "1" ]]; then
138+
MOPT="$MOPT -maxloss=0.1 -minfui=0."
139+
MOPT="$MOPT -maxdistfraction=0.70"
140+
elif [[ $RECID == "2" ]]; then
141+
MOPT="$MOPT -maxloss=0.2 -minfui=0."
142+
MOPT="$MOPT -maxdistfraction=0.75"
143+
else
144+
MOPT="$MOPT -maxloss=0.2 -minfui=0."
145+
MOPT="$MOPT -maxdistfraction=0.80"
146+
fi
139147

140148
#########################################
141149
# disp reconstruction
142150
#
143-
MVATYPE="MLP"
144151
MVATYPE="BDT"
145152
# disp main directory name
146153
DISPSUBDIR="DISPBDT/${MVATYPE}disp.${ARRAY}.R1"
154+
echo "CHECKING ${CTA_USER_DATA_DIR}/analysis/AnalysisData/${DSET}/${DISPSUBDIR/${ARRAY}/HYPERARRAY}"
155+
if [[ -d ${CTA_USER_DATA_DIR}/analysis/AnalysisData/${DSET}/${DISPSUBDIR/${ARRAY}/HYPERARRAY} ]]; then
156+
DISPSUBDIR="${DISPSUBDIR/${ARRAY}/HYPERARRAY}"
157+
echo "Choosing hyperarray DispDir"
158+
else
159+
echo "Did not find hyperarray DispDir ${DISPSUBDIR/${ARRAY}/HYPERARRAY}"
160+
fi
161+
echo "DISPDIR $DISPSUBDIR"
147162
#########################################
148163
# unpack disp XML files for all telescope
149164
# types to tmpdir
@@ -184,10 +199,10 @@ MOPT="$MOPT -tmva_filename_disperror_reconstruction $DISPERRORDIR -tmva_disperro
184199
##########################################################################################################
185200
# options for DISP method (core)
186201
# (switch on for single-telescope analysis)
187-
DISPCOREDIR="${TMPDIR}/${MVATYPE}DispCore/${MCAZ}/${MVATYPE}DispCore_${MVATYPE}_"
188-
if [[ $ARRAY == *"SV1"* ]]; then
189-
MOPT="$MOPT -tmva_filename_core_reconstruction $DISPCOREDIR"
190-
fi
202+
# DISPCOREDIR="${TMPDIR}/${MVATYPE}DispCore/${MCAZ}/${MVATYPE}DispCore_${MVATYPE}_"
203+
# if [[ $ARRAY == *"SV1"* ]]; then
204+
# MOPT="$MOPT -tmva_filename_core_reconstruction $DISPCOREDIR"
205+
# fi
191206

192207
##########################################################################################################
193208
# options for DISP method (energy)

0 commit comments

Comments
 (0)