Add tune-params subcommand for cpuctExploration tuning via QRS optimization

README.md

@@ -153,6 +153,10 @@ Run a high-performance match engine that will play a pool of bots against each o
 
    * `./katago match -config <MATCH_CONFIG>.cfg -log-file match.log -sgf-output-dir <DIR TO WRITE THE SGFS>`
 
+Tune PUCT search hyperparameters via QRS-Tune sequential optimization:
+
+   * `./katago tune-params -config <TUNE_CONFIG>.cfg`
+
 Force OpenCL tuner to re-tune:
 
    * `./katago tuner -config <GTP_CONFIG>.cfg`

cpp/CMakeLists.txt

@@ -307,6 +307,7 @@ add_executable(katago
   tests/tinymodel.cpp
   tests/tinymodeldata.cpp
   distributed/client.cpp
+  qrstune/QRSOptimizer.cpp
   command/commandline.cpp
   command/analysis.cpp
   command/benchmark.cpp
@@ -318,6 +319,7 @@ add_executable(katago
   command/gputest.cpp
   command/gtp.cpp
   command/match.cpp
+  command/tuneparams.cpp
   command/misc.cpp
   command/runtests.cpp
   command/sandbox.cpp

cpp/README.md

@@ -35,6 +35,7 @@ Summary of source folders, in approximate dependency order, from lowest level to
 * `distributed` - Code for talking to https webserver for volunteers to contribute distributed self-play games for training.
 * `tests` - A variety of tests.
   * `models` - A directory with a small number of small-sized (and not very strong) models for running tests.
+* `qrstune` - QRS-Tune (Quadratic Response Surface) optimizer for hyperparameter tuning.
 * `command` - Top-level subcommands callable by users. GTP, analysis commands, benchmarking, selfplay data generation, etc.
   * `commandline.{cpp,h}` - Common command line logic shared by all subcommands.
   * `gtp.cpp` - Main GTP engine.
@@ -44,6 +45,7 @@ Summary of source folders, in approximate dependency order, from lowest level to
   * `selfplay.cpp` - Selfplay data generation engine.
   * `gatekeeper.cpp` - Gating engine to filter neural nets for selfplay data generation.
   * `match.cpp` - Match engine for testing different parameters that can use huge batch sizes to efficiently play games in parallel.
+  * `tuneparams.cpp` - Tune PUCT search hyperparameters via QRS-Tune sequential optimization.
 
 Other folders:
 

cpp/command/match.cpp

@@ -1,4 +1,6 @@
 #include "../core/global.h"
+#include "../core/elo.h"
+#include "../core/fancymath.h"
 #include "../core/fileutils.h"
 #include "../core/makedir.h"
 #include "../core/config_parser.h"
@@ -11,6 +13,7 @@
 #include "../command/commandline.h"
 #include "../main.h"
 
+#include <array>
 #include <csignal>
 
 using namespace std;
@@ -250,10 +253,13 @@ int MainCmds::match(const vector<string>& args) {
   int64_t gameCount = 0;
   std::map<string,double> timeUsedByBotMap;
   std::map<string,double> movesByBotMap;
+  map<pair<string,string>, array<int64_t,3>> pairStats;
+  //key: {nameA, nameB} with nameA < nameB lexicographically
+  //value: {winsA, winsB, draws}
 
   auto runMatchLoop = [
     &gameRunner,&matchPairer,&sgfOutputDir,&logger,&gameSeedBase,&patternBonusTables,
-    &statsMutex, &gameCount, &timeUsedByBotMap, &movesByBotMap
+    &statsMutex, &gameCount, &timeUsedByBotMap, &movesByBotMap, &pairStats
   ](
     uint64_t threadHash
   ) {
@@ -303,6 +309,20 @@ int MainCmds::match(const vector<string>& args) {
           movesByBotMap[gameData->bName] += (double)gameData->bMoveCount;
           movesByBotMap[gameData->wName] += (double)gameData->wMoveCount;
 
+          //Update pairwise W/L/D stats
+          {
+            const string& bName = gameData->bName;
+            const string& wName = gameData->wName;
+            Player winner = gameData->endHist.winner;
+            bool aIsBlack = (bName < wName);
+            const string& nameA = aIsBlack ? bName : wName;
+            const string& nameB = aIsBlack ? wName : bName;
+            auto& ps = pairStats[{nameA, nameB}];
+            if(winner == P_BLACK)      { if(aIsBlack) ps[0]++; else ps[1]++; }
+            else if(winner == P_WHITE) { if(aIsBlack) ps[1]++; else ps[0]++; }
+            else                       { ps[2]++; }
+          }
+
           int64_t x = gameCount;
           while(x % 2 == 0 && x > 1) x /= 2;
           if(x == 1 || x == 3 || x == 5) {
@@ -344,6 +364,58 @@ int MainCmds::match(const vector<string>& args) {
   for(int i = 0; i<threads.size(); i++)
     threads[i].join();
 
+  //Final match statistics
+  if(!pairStats.empty()) {
+    vector<string> activeBots;
+    {
+      set<string> seen;
+      for(auto& kv : pairStats) {
+        seen.insert(kv.first.first);
+        seen.insert(kv.first.second);
+      }
+      activeBots.assign(seen.begin(), seen.end());
+    }
+
+    vector<double> elo, eloStderr;
+    bool eloConverged = ComputeElos::computeBradleyTerryElo(activeBots, pairStats, elo, eloStderr);
+
+    logger.write("");
+    if(!eloConverged)
+      logger.write("Warning: Bradley-Terry Elo estimation did not fully converge");
+    logger.write("=== match Results ===");
+    logger.write("Global Elo (Bradley-Terry MLE, reference=" + activeBots[0] + "):");
+    for(int i = 0; i < (int)activeBots.size(); i++) {
+      string sign = (elo[i] >= 0) ? "+" : "";
+      string line = "  " + activeBots[i] + " : " +
+        sign + Global::strprintf("%.1f", elo[i]) + " +/- " + Global::strprintf("%.1f", eloStderr[i]);
+      if(i == 0) line += "  (reference)";
+      logger.write(line);
+    }
+    logger.write("");
+    logger.write("Pairwise summary:");
+    for(auto& kv : pairStats) {
+      int64_t wA = kv.second[0], wB = kv.second[1], d = kv.second[2];
+      int64_t total = wA + wB + d;
+      if(total == 0) continue;
+      double wins = wA + 0.5 * d;
+      double lo, hi;
+      FancyMath::wilsonCI95(wins, (double)total, lo, hi);
+      double pval = FancyMath::oneTailedPValue(wins, (double)total);
+      string sig = (pval < 0.05) ? " *" : "";
+      logger.write(
+        "  " + kv.first.first + " vs " + kv.first.second +
+        " : Games=" + Global::int64ToString(total) +
+        " W=" + Global::int64ToString(wA) +
+        " L=" + Global::int64ToString(wB) +
+        " D=" + Global::int64ToString(d) +
+        " | " + kv.first.first + " winrate=" + Global::strprintf("%.3f", wins/total) +
+        " [95% CI: " + Global::strprintf("%.3f", lo) + ", " + Global::strprintf("%.3f", hi) + "]" +
+        " | p=" + Global::strprintf("%.4f", pval) + sig
+      );
+    }
+    logger.write("");
+  }
+
   delete matchPairer;
   delete gameRunner;
 

cpp/command/runtests.cpp

@@ -5,6 +5,7 @@
 #include "../core/rand.h"
 #include "../core/elo.h"
 #include "../core/fancymath.h"
+#include "../qrstune/QRSOptimizer.h"
 #include "../core/config_parser.h"
 #include "../core/datetime.h"
 #include "../core/fileutils.h"
@@ -35,6 +36,7 @@ int MainCmds::runtests(const vector<string>& args) {
   DateTime::runTests();
   FancyMath::runTests();
   ComputeElos::runTests();
+  QRSTune::runTests();
   Base64::runTests();
   ThreadTest::runTests();