Average over 2o runs

Author: OliverRietmann

.github/scripts/csv_to_md.py

@@ -19,16 +19,15 @@
 
 table = []
 for benchmark, baseline in zip(table_benchmark, table_baseline):
-  assert(benchmark[0] == baseline[0] and benchmark[2:4] == baseline[2:4])
+  assert(benchmark[0] == baseline[0])
   name = benchmark[0]
   time = benchmark[1]
+  stdev = benchmark[1]
   d = float(baseline[1]) - float(benchmark[1])
   emoji = ':red_circle:' if 0 < d else ':green_circle:'
   difference = pretty(d)
   percent = pretty(100 * d / float(baseline[1]))
-  count = benchmark[2]
-  events = benchmark[3]
-  table.append([name, time, emoji, difference, percent, count, events])
+  table.append([name, time, stdev, emoji, difference, percent])
 
-header = ["name", "time", "", "difference", "percent", "count", "events"]
+header = ["name", "time", "stdev", "", "difference", "percent"]
 print(tab.tabulate(table, header, tablefmt="github"))

.github/scripts/merge_runs.py

@@ -0,0 +1,27 @@
+import sys
+import csv
+import statistics
+
+in_csv = sys.argv[1]
+out_csv = sys.argv[2]
+
+time_dict = dict({})
+
+with open(in_csv) as csv_file:
+  csv_reader = csv.reader(csv_file)
+  next(csv_reader)
+  for name, time, _, _ in csv_reader:
+    if name in time_dict.keys():
+      time_dict[name].append(float(time))
+    else:
+      time_dict[name] = [float(time)]
+
+data = [["name", "time", "stdev"]]
+for name, time_list in time_dict.items():
+  mean = statistics.mean(time_list)
+  stdev = statistics.stdev(time_list)
+  data.append([name, mean, stdev])
+
+with open(out_csv, 'w') as csv_file:
+  csv_writer = csv.writer(csv_file)
+  csv_writer.writerows(data)

.github/workflows/standalone-benchmark.yml

@@ -33,7 +33,7 @@ jobs:
       ALIBUILD_ARCH_PREFIX: el9-x86_64/Packages
       MODULEPATH: /cvmfs/alice.cern.ch/etc/toolchain/modulefiles/el9-x86_64:/cvmfs/alice.cern.ch/el9-x86_64/Modules/modulefiles
       STANDALONE_DIR: /root/standalone
-      ARTIFACT_FILE: /root/benchmark.csv
+      BENCHMARK_CSV: /root/benchmark.csv
       LD_LIBRARY_PATH: /usr/local/cuda-13.0/compat
 
     name: ${{ matrix.name }}
@@ -87,14 +87,15 @@ jobs:
           source /etc/profile.d/modules.sh
           module load ninja/fortran-v1.11.1.g9-15 Vc/1.4.5-10 boost/v1.83.0-alice2-57 fmt/11.1.2-14 CMake/v3.31.6-10 ms_gsl/4.2.1-3 Clang/v20.1.7-9 TBB/v2022.3.0-3 ROOT/v6-36-04-alice9-15 ONNXRuntime/v1.22.0-71 GLFW/3.3.2-25
           cd ${STANDALONE_DIR}
-          ${STANDALONE_DIR}/ca -e 50kHz -g --memSize 15000000000 --sync --runs 1 --debug 1 --PROCtimingCSV ${ARTIFACT_FILE}
+          ${STANDALONE_DIR}/ca -e 50kHz -g --memSize 15000000000 --sync --runs 20 --debug 1 --PROCtimingCSV ${BENCHMARK_CSV}
           rm -rf ${STANDALONE_DIR}/events/50kHz ${STANDALONE_DIR}/build
 
       - name: Display table on GitHub web
         run: |
           source /etc/profile.d/modules.sh
           module load ninja/fortran-v1.11.1.g9-15 Vc/1.4.5-10 boost/v1.83.0-alice2-57 fmt/11.1.2-14 CMake/v3.31.6-10 ms_gsl/4.2.1-3 Clang/v20.1.7-9 TBB/v2022.3.0-3 ROOT/v6-36-04-alice9-15 ONNXRuntime/v1.22.0-71 GLFW/3.3.2-25
-          python3 ${GITHUB_WORKSPACE}/.github/scripts/csv_to_md.py ${ARTIFACT_FILE} ${STANDALONE_DIR}/baseline/${{ matrix.name }}.csv >> ${GITHUB_STEP_SUMMARY}
+          python3 ${GITHUB_WORKSPACE}/.github/scripts/merge_runs.py ${BENCHMARK_CSV} ${BENCHMARK_CSV}
+          python3 ${GITHUB_WORKSPACE}/.github/scripts/csv_to_md.py ${BENCHMARK_CSV} ${BENCHMARK_CSV} #${STANDALONE_DIR}/baseline/${{ matrix.name }}.csv >> ${GITHUB_STEP_SUMMARY}
           rm -rf ${STANDALONE_DIR}/baseline
 
       - name: Upload Artifact