v492 dev13

scripts/ANALYSIS.anasum_allcuts.sh

@@ -4,7 +4,7 @@
 #
 
 if [[ ${VERITAS_ANALYSIS_TYPE:0:2} == "AP" ]]; then
-    CUTS="moderate2tel soft2tel hard2tel hard3tel"
+    CUTS="moderate2tel soft2tel hard2tel hard3tel moderate2telXGB soft2telXGB hard2telXGB hard3telXGB"
 else
     CUTS="supersoftNN2tel"
 fi

scripts/IRF.dispXGB.sh

@@ -16,7 +16,7 @@ required parameters:
 
     <input file directory>  directory with input files (will use all *.mscw.root files)
 
-    <output directory>      directory where fits.gz files are written
+    <output directory>      directory where XGB files are written
 "
 exit
 fi

scripts/IRF.generalproduction.sh

@@ -19,7 +19,7 @@ required parameters:
                             EVNDISP,
                             MAKETABLES, COMBINETABLES,
                             TRAINMVANGRES,
-                            TRAINXGBANGRES, ANAXGBANGRES,
+                            TRAINXGBANGRESBINNED, ANAXGBANGRES,
                             TRAINXGBGH, ANAXGBGH,
                             ANALYSETABLES,
                             PRESELECTEFFECTIVEAREAS, COMBINEPRESELECTEFFECTIVEAREAS,

scripts/IRF.production.sh

@@ -22,7 +22,7 @@ required parameters:
                             EVNDISP,
                             MAKETABLES, COMBINETABLES,
                             TRAINMVANGRES,
-                            TRAINXGBANGRES, ANAXGBANGRES,
+                            TRAINXGBANGRESBINNED, ANAXGBANGRES,
                             TRAINXGBGH, ANAXGBGH,
                             ANALYSETABLES,
                             PRESELECTEFFECTIVEAREAS, COMBINEPRESELECTEFFECTIVEAREAS,
@@ -170,8 +170,8 @@ elif [[ "${SIMTYPE}" == "CARE_June2020" ]]; then
     ######################################
     # TEST
     # ZENITH_ANGLES=( 20 )
-    # WOBBLE_OFFSETS=( 0.0 )
-    # NSB_LEVELS=( 350 )
+    # WOBBLE_OFFSETS=( 0.5 )
+    # NSB_LEVELS=( 200 )
     ######################################
     # TRAINMVANGRES production
     # (assume 0.5 deg wobble is done)
@@ -385,6 +385,21 @@ for VX in $EPOCH; do
             continue
        fi
        #################################################
+       # zenith angle bin dependent analysis
+       #################################################
+       if [[ $IRFTYPE == "TRAINXGBANGRESBINNED" ]]; then
+           STEREO_PAR="$VERITAS_EVNDISP_AUX_DIR/ParameterFiles/XGB-stereo-parameter.json"
+           IDS=$(jq -r '.zenith[].id' $STEREO_PAR)
+           for ZAB in $IDS; do
+               # Explicitly remove 0.0 bin
+               FIXEDWOBBLE="0.25 0.5 0.75 1.0 1.25 1.5 1.75 2.0"
+               FIXEDNSB="160 200 350 450"
+                   $(dirname "$0")/IRF.trainXGBforAngularReconstructionBinned.sh \
+                       $VX $ATM $ZAB "$FIXEDWOBBLE" "$FIXEDNSB" 0 \
+                       $SIMTYPE $ANATYPE $UUID
+           done
+           continue
+       fi
        # zenith angle dependent analysis
        for ZA in ${ZENITH_ANGLES[@]}; do
             ######################

scripts/IRF.selectRunsForGammaHadronSeparationTraining.sh

@@ -37,7 +37,26 @@ echo "Minimum observing time: ${MINOBSTIME} s"
 echo "Avoiding bright sources: ${BRIGHTSOURCES[*]}"
 
 # zenith angle bins
-ZEBINS=$( cat "$RUNPAR" | grep "^* ZENBINS " | sed -e 's/* ZENBINS//' | sed -e 's/ /\n/g')
+if [[ "${RUNPAR##*.}" == "json" ]]; then
+    echo "Reading zenith bins from json file"
+    ZEBIN_EDGES=$(jq -r '.zenith_bins_deg[] | "\(.Ze_min) \(.Ze_max)"' "$RUNPAR" | awk '{print $1}')
+    ZEBIN_MAX=$(jq -r '.zenith_bins_deg[-1].Ze_max' "$RUNPAR")
+
+    # Combine into a single space-separated string of unique bin edges
+    ZEBINS=$(echo "$ZEBIN_EDGES" "$ZEBIN_MAX" | tr '\n' ' ' | awk '{
+        # Store unique values in order
+        split($0, arr);
+        prev = "";
+        for (i in arr) {
+            if (arr[i] != prev) {
+                printf "%s ", arr[i];
+                prev = arr[i];
+            }
+        }
+    }')
+else
+    ZEBINS=$( cat "$RUNPAR" | grep "^* ZENBINS " | sed -e 's/* ZENBINS//' | sed -e 's/ /\n/g')
+fi
 echo "Zenith angle definition: $ZEBINS"
 declare -a ZEBINARRAY=( $ZEBINS ) #convert to array
 NZEW=$((${#ZEBINARRAY[@]}-1)) #get number of bins
@@ -47,9 +66,14 @@ if [[ $MEPOCH == "V4" ]]; then
 elif [[ $MEPOCH == "V5" ]]; then
     FLIST=$(find ${2} -name "[4,5,6]*[0-9].mscw.root"  | sed 's/\.root$//')
 else
-    FLIST=$(find ${2} -name "[6-9, 10]*[0-9].mscw.root"  | sed 's/\.root$//')
+    FLIST=$(find "$2" -regextype posix-extended \
+      -regex '.*/(6|7|8|9|1[0-5])[0-9]*\.mscw\.root' \
+      | sed 's/\.root$//')
 fi
 
+echo $FLIST
+exit
+
 mkdir -p ${3}
 
 linkFile()

scripts/IRF.trainXGBforAngularReconstructionBinned.sh

@@ -0,0 +1,139 @@
+#!/bin/bash
+# train XGB for angular reconstruction
+
+h_cpu=47:29:00; h_vmem=16000M; tmpdir_size=100G
+
+EDVERSION=$(cat $VERITAS_EVNDISP_AUX_DIR/IRFVERSION)
+
+if [ $# -lt 7 ]; then
+echo "
+XGB (BDT) training for stereo reconstruction from MC mscw files for different zenith angle bins
+
+IRF.trainXGBforAngularReconstructionBinned.sh <epoch> <atmosphere> <zenith range> <offset angle> <NSB level> <Rec ID> <sim type> [analysis type]
+
+required parameters:
+
+    <epoch>                 array epoch (e.g., V4, V5, V6)
+
+    <atmosphere>            atmosphere model (61 = winter, 62 = summer)
+
+    <zenith range>          zenith angle range (SZE, MZE, LZE, XZE)
+
+    <offset angle>          list of offset angle of simulations [deg]
+
+    <NSB level>             list of NSB level of simulations [MHz]
+
+    <Rec ID>                reconstruction ID
+                            (see EVNDISP.reconstruction.runparameter)
+
+    <sim type>              simulation type (e.g. GRISU, CARE_June1425)
+
+optional parameters:
+
+    [analysis type]         type of analysis (default="")
+
+    [uuid]                  UUID used for submit directory
+
+--------------------------------------------------------------------------------
+"
+exit
+fi
+
+# Run init script
+if [ -z "$EVNDISP_APPTAINER" ]; then
+    bash $(dirname "$0")"/helper_scripts/UTILITY.script_init.sh"
+fi
+[[ $? != "0" ]] && exit 1
+
+EPOCH="$1"
+ATM="$2"
+ZA="$3"
+WOBBLE="$4"
+NOISE="$5"
+RECID="$6"
+SIMTYPE="$7"
+ANALYSIS_TYPE="${8:-}"
+UUID="${9:-$(date +"%y%m%d")-$(uuidgen)}"
+
+if [[ -z "$VERITAS_IRFPRODUCTION_DIR" ]]; then
+    echo "Error: IRF production directory not found: $VERITAS_IRFPRODUCTION_DIR"
+    exit 1
+fi
+# output and log directories
+ODIR="$VERITAS_IRFPRODUCTION_DIR/$EDVERSION/${ANALYSIS_TYPE}/$SIMTYPE/${EPOCH}_ATM${ATM}_gamma/TrainXGBStereoAnalysisBinned/${ZA}/"
+LOGDIR="$VERITAS_IRFPRODUCTION_DIR/$EDVERSION/${ANALYSIS_TYPE}/${SIMTYPE}/${EPOCH}_ATM${ATM}_gamma/submit-trainXGBStereoAnalysis-RECID${RECID}-${UUID}"
+mkdir -p "$ODIR"
+chmod g+w "$ODIR"
+mkdir -p "$LOGDIR"
+echo "Output: $ODIR"
+echo "Logs: $LOGDIR"
+
+# prepare list of input files
+MSCWLIST="$ODIR/xgbFiles.list"
+rm -f ${MSCWLIST}
+touch ${MSCWLIST}
+
+INDIR="$VERITAS_IRFPRODUCTION_DIR/$EDVERSION/${ANALYSIS_TYPE}/$SIMTYPE/${EPOCH}_ATM${ATM}_gamma/MSCW_RECID0_DISP"
+
+STEREO_PAR="$VERITAS_EVNDISP_AUX_DIR/ParameterFiles/XGB-stereo-parameter.json"
+TRAIN_ANGLES=$(jq -r ".zenith[] | select(.id==\"$ZA\") | .train | join(\" \")" $STEREO_PAR)
+if [[ -z "$TRAIN_ANGLES" ]]; then
+    echo "Error: Bin ID $ZA_BIN not found in $JSON_FILE"
+    exit 1
+fi
+
+for W in ${WOBBLE}
+do
+    for N in ${NOISE}
+    do
+        for DEG in ${TRAIN_ANGLES}
+        do
+            FILE_PATH="${INDIR}/${DEG}deg_${W}wob_NOISE${N}.mscw.root"
+            if [[ -f "$FILE_PATH" ]]; then
+                echo "$FILE_PATH" >> "${MSCWLIST}"
+            else
+                echo "Warning: File not found: $FILE_PATH"
+            fi
+        done
+    done
+done
+echo "FILE LIST: ${MSCWLIST}"
+
+SUBSCRIPT=$( dirname "$0" )"/helper_scripts/IRF.trainXGBforAngularReconstruction_sub.sh"
+# loop over telescope multiplicities
+for ((tel=2; tel<=4; tel++)); do
+
+    echo "Processing Telescope $tel Zenith = $ZA, Noise = $NOISE, Wobble = $WOBBLE"
+
+    FSCRIPT="$LOGDIR/trainXGBStereoAnalysis.TEL${tel}ID${RECID}.${EPOCH}.ATM${ATM}.${ZA}.sh"
+    sed -e "s|OUTPUTDIR|$ODIR|" \
+        -e "s|MSCWLIST|$MSCWLIST|" \
+        -e "s|TTYPE|$tel|" "$SUBSCRIPT" > "$FSCRIPT"
+
+    chmod u+x "$FSCRIPT"
+    echo "$FSCRIPT"
+
+    # run locally or on cluster
+    SUBC=`$( dirname "$0" )/helper_scripts/UTILITY.readSubmissionCommand.sh`
+    SUBC=`eval "echo \"$SUBC\""`
+    if [[ $SUBC == *"ERROR"* ]]; then
+        echo $SUBC
+        exit
+    fi
+    if [[ $SUBC == *qsub* ]]; then
+        JOBID=`$SUBC $FSCRIPT`
+        echo "RUN $RUNNUM: JOBID $JOBID"
+    elif [[ $SUBC == *condor* ]]; then
+        $(dirname "$0")/helper_scripts/UTILITY.condorSubmission.sh $FSCRIPT $h_vmem $tmpdir_size
+        echo
+        echo "-------------------------------------------------------------------------------"
+        echo "Job submission using HTCondor - run the following script to submit jobs at once:"
+        echo "$EVNDISPSCRIPTS/helper_scripts/submit_scripts_to_htcondor.sh ${LOGDIR} submit"
+        echo "-------------------------------------------------------------------------------"
+        echo
+    elif [[ $SUBC == *sbatch* ]]; then
+            $SUBC $FSCRIPT
+    elif [[ $SUBC == *parallel* ]]; then
+        echo "$FSCRIPT &> $FSCRIPT.log" >> "$LOGDIR/runscripts.dat"
+    fi
+done

scripts/helper_scripts/ANALYSIS.dispXGB_sub.sh

@@ -79,17 +79,17 @@ if [[ ! -d "${DISPDIR}" ]]; then
 fi
 OFIL=$(basename $MSCW_FILE .root)
 if [[ "${XGB_TYPE}" == "stereo_analysis" ]]; then
-    # requires bc as printRunParameter returns a float
-    if (( $(echo "$ZA < 38" | bc -l) )); then
-        DISPDIR="${DISPDIR}/SZE/"
-    elif (( $(echo "$ZA < 48" | bc -l) )); then
-        DISPDIR="${DISPDIR}/MZE/"
-    elif (( $(echo "$ZA < 58" | bc -l) )); then
-        DISPDIR="${DISPDIR}/LZE/"
-    else
-        DISPDIR="${DISPDIR}/XZE/"
+    STEREO_PAR="$VERITAS_EVNDISP_AUX_DIR/ParameterFiles/XGB-stereo-parameter.json"
+    BIN_ID=$(jq -r --arg za "$ZA" '
+      .zenith[]
+      | select(has("eval_min"))
+      | select(($za|tonumber) >= (.eval_min|tonumber) and ($za|tonumber) < (.eval_max|tonumber))
+      | .id' "$STEREO_PAR")
+    if [[ -z "$BIN_ID" ]]; then
+        echo "Error: No zenith bin found in $JSON_FILE for ZA=$ZA"
+        exit 1
     fi
-    DISPDIR="${DISPDIR}/dispdir_bdt"
+    DISPDIR="${DISPDIR}/${BIN_ID}/dispdir_bdt"
     ML_EXEC="eventdisplay-ml-apply-xgb-stereo"
     OFIL="${ODIR}/${OFIL}.${XGB}_stereo"
 elif [[ "${XGB_TYPE}" == "classification" ]]; then

scripts/helper_scripts/IRF.dispXGB_sub.sh

@@ -65,16 +65,17 @@ if [[ ! -d "${DISPDIR}" ]]; then
 fi
 OFIL=$(basename $MSCW_FILE .root)
 if [[ "${XGB_TYPE}" == "stereo_analysis" ]]; then
-    if [[ "${ZA}" -lt "38" ]]; then
-        DISPDIR="${DISPDIR}/SZE/"
-    elif [[ "${ZA}" -lt "48" ]]; then
-        DISPDIR="${DISPDIR}/MZE/"
-    elif [[ "${ZA}" -lt "58" ]]; then
-        DISPDIR="${DISPDIR}/LZE/"
-    else
-        DISPDIR="${DISPDIR}/XZE/"
+    STEREO_PAR="$VERITAS_EVNDISP_AUX_DIR/ParameterFiles/XGB-stereo-parameter.json"
+    BIN_ID=$(jq -r --arg za "$ZA" '
+      .zenith[]
+      | select(has("eval_min"))
+      | select(($za|tonumber) >= (.eval_min|tonumber) and ($za|tonumber) < (.eval_max|tonumber))
+      | .id' "$STEREO_PAR")
+    if [[ -z "$BIN_ID" ]]; then
+        echo "Error: No zenith bin found in $JSON_FILE for ZA=$ZA"
+        exit 1
     fi
-    DISPDIR="${DISPDIR}/dispdir_bdt"
+    DISPDIR="${DISPDIR}/${BIN_ID}/dispdir_bdt"
     ML_EXEC="eventdisplay-ml-apply-xgb-stereo"
     OFIL="${ODIR}/${OFIL}.${XGB}_stereo"
 elif [[ "${XGB_TYPE}" == "classification" ]]; then

scripts/helper_scripts/IRF.mscw_energy_MC_sub.sh

@@ -236,15 +236,17 @@ run_xgb()
     echo "MSCW file: ${MSCW_FILE} at zenith ${ZA} deg"
 
     DISPDIR="$VERITAS_EVNDISP_AUX_DIR/DispXGB/${ANATYPE}/${EPOCH}_ATM${ATM}/"
-    if [[ "${ZA}" -lt "38" ]]; then
-        DISPDIR="${DISPDIR}/SZE/"
-    elif [[ "${ZA}" -lt "48" ]]; then
-        DISPDIR="${DISPDIR}/MZE/"
-    elif [[ "${ZA}" -lt "58" ]]; then
-        DISPDIR="${DISPDIR}/LZE/"
-    else
-        DISPDIR="${DISPDIR}/XZE/"
+    STEREO_PAR="$VERITAS_EVNDISP_AUX_DIR/ParameterFiles/XGB-stereo-parameter.json"
+    BIN_ID=$(jq -r --arg za "$ZA" '
+      .zenith[]
+      | select(has("eval_min"))
+      | select(($za|tonumber) >= (.eval_min|tonumber) and ($za|tonumber) < (.eval_max|tonumber))
+      | .id' "$STEREO_PAR")
+    if [[ -z "$BIN_ID" ]]; then
+        echo "Error: No zenith bin found in $JSON_FILE for ZA=$ZA"
+        exit 1
     fi
+    DISPDIR="${DISPDIR}/${BIN_ID}/dispdir_bdt"
     echo "DispXGB directory $DISPDIR"
     echo "DispXGB options $XGBVERSION"
     XGBOFIL=$(get_xgb_output_file $1)

scripts/helper_scripts/IRF.trainXGBforAngularReconstruction_sub.sh

@@ -48,13 +48,14 @@ check_conda_installation
 source activate base
 conda activate $env_name
 
-LOGFILE="${ODIR}/XGB_ntel${TEL}.log"
+PREFIX="${ODIR}/dispdir_bdt"
+LOGFILE="${PREFIX}_ntel${TEL}.log"
 rm -f "$LOGFILE"
 
 eventdisplay-ml-train-xgb-stereo \
     --input_file_list "$LLIST" \
     --n_tel $TEL \
-    --model_prefix "${ODIR}/dispdir_bdt" \
+    --model_prefix "${PREFIX}" \
     --train_test_fraction $P --max_events $N >| "${LOGFILE}" 2>&1
 
 python --version >> "${LOGFILE}"