OpenACC directives for SoilBiogeochemCompetition

perf_testing/.gitignore

@@ -2,3 +2,9 @@
 *.mod
 driver
 last_run.txt
+last_run_timings.csv
+last_run_timings_fast.csv
+sbgc_gpu.o*
+sbgc_gpu.e*
+sbgc_dbg.o*
+sbgc_dbg.e*

perf_testing/Makefile.common

@@ -37,6 +37,14 @@ ifeq ($(TIMING),1)
     FFLAGS += -DPERF_TIMING
 endif
 
+# INNER_TIMING=1 turns on the per-loop perf_timers_mod instrumentation.
+# Independent of TIMING/PERF_TIMING (which controls the driver-level
+# system_clock around the whole iteration loop).
+INNER_TIMING ?= 0
+ifeq ($(INNER_TIMING),1)
+    FFLAGS += -DINNER_TIMING
+endif
+
 driver: $(OBJ)
 	$(FC) $(FFLAGS) $(LDFLAGS) -o $@ $(OBJ)
 

perf_testing/SoilBiogeochemCompetition/Makefile

@@ -1,6 +1,12 @@
-OBJ := SoilBiogeochemCompetition.o driver.o
+# Pick up the shared perf_timers_mod from the parent perf_testing/ dir.
+VPATH := ..
 
-# driver.F90 uses the SoilBiogeochemCompetition_mod module
-driver.o: SoilBiogeochemCompetition.o
+OBJ := SoilBiogeochemCompetition.o perf_timers_mod.o driver.o
+
+# Module-use ordering. driver.F90 uses SoilBiogeochemCompetition_mod;
+# both driver.F90 and SoilBiogeochemCompetition.F90 use perf_timers_mod
+# (start/stop calls inside the routine; print/dump calls in the driver).
+driver.o: SoilBiogeochemCompetition.o perf_timers_mod.o
+SoilBiogeochemCompetition.o: perf_timers_mod.o
 
 include ../Makefile.common

perf_testing/SoilBiogeochemCompetition/baseline_checksum.txt

@@ -1,9 +1,7 @@
+mode                 all
 ncol                 8000
 nlevdecomp           10
 ndct                 8
 numfc                8000
-niters               1
-use_nitrif_denitrif  T
-carbon_only          F
-decomp_method        2
-checksum               9.5970435393765438E+04
+niters               100
+checksum               7.6772246368780300E+07

perf_testing/SoilBiogeochemCompetition/baseline_checksum_fast.txt

@@ -0,0 +1,7 @@
+mode                 fast
+ncol                 8000
+nlevdecomp           10
+ndct                 8
+numfc                8000
+niters               100
+checksum               9.5857105051752981E+06

perf_testing/SoilBiogeochemCompetition/build.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+# Source the project env file, then `make clean && make "$@"`.
+# Use this instead of inlining `. ../env.sh && make ...` in shell commands —
+# scripted entry points stay whitelistable across runs.
+#
+# Usage:
+#   ./build.sh                                        # serial
+#   ./build.sh EXTRA_FFLAGS="-mp"                     # OpenMP
+#   ./build.sh EXTRA_FFLAGS="-acc=gpu -gpu=cc80"      # GPU
+#   ./build.sh EXTRA_FFLAGS="-acc=gpu -gpu=cc80" INNER_TIMING=1
+#
+# Filter output (grep, tail, head, etc.) at the call site, not in here.
+
+set -euo pipefail
+cd "$(dirname "$0")"
+
+# shellcheck disable=SC1091
+. ../env.sh >/dev/null 2>&1
+
+make clean
+make "$@"

perf_testing/SoilBiogeochemCompetition/debug_gpu.sh

@@ -0,0 +1,49 @@
+#!/bin/bash
+# Debug helper: submit a PBS job that runs ./driver directly (no
+# verify.sh grep filter) so any GPU runtime error is fully visible.
+#
+# Assumes ./driver was already built on the login node (e.g. via:
+#   make clean && make EXTRA_FFLAGS="-acc=gpu -gpu=cc80"
+# ).
+#
+# Usage:
+#   ./debug_gpu.sh                  # runs ./driver --fast
+#   ./debug_gpu.sh --all            # runs ./driver --all
+#   ./debug_gpu.sh --fast           # explicit
+#
+# Output is written to ./sbgc_dbg.o<jobid> (gitignored) and cat'd here.
+
+set -euo pipefail
+cd "$(dirname "$0")"
+
+driver_args="${*:---fast}"
+
+job_id=$(qsub -W block=true \
+              -A ucsg0003 -q develop \
+              -l select=1:ncpus=1:ngpus=1 \
+              -l walltime=00:05:00 \
+              -N sbgc_dbg \
+              -j oe \
+              <<EOF
+#!/bin/bash
+cd "\$PBS_O_WORKDIR"
+. ../env.sh
+echo "=== nvidia-smi ==="
+nvidia-smi || echo "(nvidia-smi failed)"
+echo "=== ./driver $driver_args ==="
+./driver $driver_args
+# nsys profile -t cuda -o sbgcc_profile_report ./driver $driver_args
+echo "=== exit status: \$? ==="
+EOF
+)
+
+job_num=${job_id%%.*}
+out_file="sbgc_dbg.o${job_num}"
+
+echo "=== job: $job_id (output: $out_file) ==="
+if [ -f "$out_file" ]; then
+    cat "$out_file"
+else
+    echo "(output file $out_file not found)"
+    exit 1
+fi