training rework, best weights so far*

After quite a bit of experimentation it would appear that a far better way to train on self-play data in this setup is to randomise not the moves but the opening. The new selfplay and match binaries have lost the epsilon parameter and now play a random opening of some length before switching to pure, deterministic engine inference. This also resulted in a cleanup of nn_train.py.

As an added bonus, in this configuration it makes sense to harvest training data from both sides in self-play and consequently the two sides are given distinct game ids (this was almost surely an oversight in the original version of TD(lambda) that i had). I also noticed that rand32 was not using the top 32bits of xorshitf*.

*Under default settings nn_train.py produces a 5x5 weight set that doesn't lose a single game to any other cttei that i cared to look at, when measured using `racetrack` (which has a diverse balanced opening book).

3 files changed, 147 insertions, 131 deletions

diff --git a/src/match.c b/src/match.c
index 49ff9c9..eff1931 100644
--- a/src/match.c
+++ b/src/match.c
@@ -73,31 +73,27 @@ void negamax_display_progress(const uint8_t cur_depth,
 // Game driver
 // ===================================================================
 
-// Returns the winner's colour, or -1 for a draw / abort
 static int play_game(const nn_function current_fn, const nn_function cand_fn,
-                     const enum COLOUR current_fn_colour, const uint32_t eps10000) {
+                     const enum COLOUR current_fn_colour,
+                     const uint8_t random_opening_length) {
   reset_state(state, state->board_size);
   char ptn[16];
 
   for (uint32_t ply = 0; ply < MAX_PLY; ply++) {
-    // Whose evaluator is it?
-    const nn_function fn =
-      (state->current_colour == current_fn_colour) ? current_fn : cand_fn;
-    negamax_set_evaluator(fn);
-
-    if (rand32() % 10000 < eps10000) {
-      // Epsilon: uniform random legal move
+    if (ply < random_opening_length) {
       uint32_t n_actions;
       action_t *actions = actions_generate(state, &n_actions);
       action_to_ptn(actions[rand32() % n_actions], ptn);
     } else {
-      // Greedy: the side's best move
+      const nn_function fn =
+        (state->current_colour == current_fn_colour) ? current_fn : cand_fn;
+      negamax_set_evaluator(fn);
       negamax_generate(state);
       strcpy(ptn, negamax_ptn);
     }
 
-    const enum ACT_RESULT r = do_ptn(state, ptn);
-    if (r == GAME_END)
+      const enum ACT_RESULT r = do_ptn(state, ptn);
+      if (r == GAME_END)
       break;
     if (r != ACT_OK) {
       fprintf(stderr, "match: unexpected result %d for ptn %s\n", r, ptn);
@@ -125,18 +121,12 @@ static int play_game(const nn_function current_fn, const nn_function cand_fn,
 // ===================================================================
 
 const char *usage =
-  "match, one Tak game of the current weights against the candidate.\n\
-\n                                                                    \
-usage: match <size> <eps> <depth>\n\
-\n\
-This binary is linked with both the include/weight_* files and the include/cand_weight_* files. It plays a single game, with a random side choice and both sides using epsilon-greedy move noise. The result format is:\n\
-\n\
-  <current|candidate|draw> <plies>\n\
-\n\
-nn_train.py runs many of these in parallel to score the candidate against the current weights.\n\
-\n\
-Copyright (C) 2026, tslil clingman\n\
-\n\
+  "match, one Tak game of the current weights against the candidate.\n\n\
+usage: match <size> <random_opening_length> <depth>\n\n\
+This binary is linked with both the include/weight_* files and the include/cand_weight_* files. It plays a single <size>x<size> game, with a random side choice and both sides using <random_opening_length> random moves at the start of the game. The result format is:\n\n\
+\t<current|candidate|draw> <plies>\n\n\
+nn_train.py runs many of these in parallel to score the candidate against the current weights.\n\n\
+Copyright (C) 2026, tslil clingman\n\n\
 This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.";
 
 int main(int argc, char **argv) {
@@ -146,22 +136,16 @@ int main(int argc, char **argv) {
   }
 
   const uint8_t size = (uint8_t)atoi(argv[1]);
-  const uint32_t eps10000 = (uint32_t)(atof(argv[2]) * 10000.0 + 0.5);
+  const uint8_t random_opening_length = (uint8_t)atoi(argv[2]);
   negamax_search_depth = (uint8_t)atoi(argv[3]);
 
   if (size != 5 && size != 6) {
     fprintf(stderr, "match: size must be 5 or 6\n");
     return EXIT_FAILURE;
   }
-  if (eps10000 >= 10000) {
-    fprintf(stderr, "match: eps must be < 1\n");
-    return EXIT_FAILURE;
-  }
-
-  // Processes launched in the same second must not share a seed
   xors = ((uint64_t)time(NULL) ^ ((uint64_t)getpid() << 32)) *
-             0x9E3779B97F4A7C15ULL +
-         0xDEADBEEF;
+    0x9E3779B97F4A7C15ULL +
+    0xC0FFEE;
 
   state = new_tak_state(size);
   negamax_init(size);
@@ -173,7 +157,7 @@ int main(int argc, char **argv) {
     : &six_cand;
 
   const enum COLOUR current_fn_colour = rand32() % 2 ? C_WHITE : C_BLACK;
-  const int winner = play_game(current_fn, cand_fn, current_fn_colour, eps10000);
+  const int winner = play_game(current_fn, cand_fn, current_fn_colour, random_opening_length);
 
   const char *name;
   if (winner == -1)
diff --git a/src/nn_train.py b/src/nn_train.py
index 73a5eaf..e9a2d2c 100755
--- a/src/nn_train.py
+++ b/src/nn_train.py
@@ -239,7 +239,7 @@ class Net(Module):
         train_targets = targets[train_idx]
 
         # training widgets
-        optimizer = torch.optim.Adam(self.parameters(), lr=lr)
+        optimizer = torch.optim.AdamW(self.parameters(), lr=lr)
         loss_fn = MSELoss()
 
         best_corr = -1.0
@@ -367,16 +367,16 @@ class TournamentResult:
 
 def run_matches(
     *,
-    size: int,
     depth: int,
-    tournament_eps: float,
     games: int,
     procs: int,
     root: Path,
+    size: int,
+    match_random_opening_length: int,
 ) -> TournamentResult:
     outs = list(
         fan_out(
-            ["./match", str(size), f"{tournament_eps:.4f}", str(depth)],
+            ["./match", str(size), f"{match_random_opening_length}", str(depth)],
             games,
             procs,
             "match",
@@ -413,12 +413,11 @@ def run_step(
     champ: Net,
     depth: int,
     epochs: int,
-    eps_decay: float,
-    eps_min: float,
-    eps_start: float,
     games: int,
     iter: int,
     lr: float,
+    matches: int,
+    match_random_opening_length: int,
     min_ply: int,
     procs: int,
     promote_rate: float,
@@ -426,8 +425,6 @@ def run_step(
     root: Path,
     size: int,
     td_lambda: float,
-    tournament_eps: float,
-    tournament_games: int,
     val_frac: float,
 ) -> Net:
     def make_csv(i: int) -> Path:
@@ -435,16 +432,16 @@ def run_step(
 
     print(f"\n== iteration {iter} ==")
 
-    # 1. rebuild selfplay
+    # 1. rebuild
     _ = subprocess.run(["make", "selfplay"], cwd=root, check=True, capture_output=True)
+    _ = subprocess.run(["make", "native"], cwd=root, check=True, capture_output=True)
 
     # 2. generate self-play data
-    eps = max(eps_min, eps_start * (eps_decay ** (iter - 1)))
     csv = make_csv(iter)
-    print(f"selfplay, eps={eps:.4f}, depth {depth}, {games} games on {procs} procs")
+    print(f"selfplay: {size=}, {depth=}, {min_ply=}")
     out_rows: list[str] = []
     for out in fan_out(
-        ["./selfplay", str(size), f"{eps:.4f}", str(depth), str(min_ply)],
+        ["./selfplay", str(size), str(depth), str(min_ply)],
         games,
         procs,
         "selfplay",
@@ -455,14 +452,13 @@ def run_step(
         _ = f.write("\n".join(out_rows))
         if out_rows:
             _ = f.write("\n")
-    rows = len(out_rows)
-    print(f"  {rows} rows")
 
     # 3. fine-tune the champion on TD(lambda) targets -> candidate
     device = next(champ.parameters()).device
     csvs = [make_csv(i) for i in range(max(iter - replay_iters, 1), iter + 1)]
-    print("Loading data from", [str(c) for c in csvs])
+    print("Loading data from", [str(c) for c in csvs], "...", end=" ")
     records = SelfPlayRecords.load_merge(csvs, size, device)
+    print(f"{records.results.shape[0]:,} rows")
     cand = copy.deepcopy(champ)
 
     corr, mse = cand.fit(
@@ -483,21 +479,25 @@ def run_step(
     # 4. build the match driver (champion + candidate)
     _ = subprocess.run(["make", "match"], cwd=root, check=True, capture_output=True)
 
-    # 5. tournament + decision
-    tournament_result = run_matches(
-        root=root,
-        size=size,
-        depth=depth,
-        tournament_eps=tournament_eps,
-        games=tournament_games,
-        procs=procs,
-    )
-    decided = tournament_result.champ + tournament_result.cand
-    rate = tournament_result.cand / decided if decided else 0.0
-    print(
-        f"champ {tournament_result.champ} - {tournament_result.cand} cand, {tournament_result.draws} draws, winrate={rate:.3f}"
-    )
-    promote = decided >= int(0.25 * tournament_games) and rate >= promote_rate
+    promote = True
+    rate = -1.0
+    if promote_rate > 0:
+        # 5. tournament + decision
+        print(f"match: {size=}, {match_random_opening_length=}, {depth=}")
+        tournament_result = run_matches(
+            depth=depth,
+            games=matches,
+            procs=procs,
+            root=root,
+            size=size,
+            match_random_opening_length=match_random_opening_length,
+        )
+        decided = tournament_result.champ + tournament_result.cand
+        rate = tournament_result.cand / decided if decided else 0.0
+        print(
+            f"champ {tournament_result.champ} - {tournament_result.cand} cand, {tournament_result.draws} draws, winrate={rate:.3f}"
+        )
+        promote = decided >= int(0.25 * matches) and rate > promote_rate
 
     if promote:
         cand.to_c(
@@ -523,16 +523,28 @@ def main() -> None:
         formatter_class=argparse.ArgumentDefaultsHelpFormatter,
     )
     _ = p.add_argument(
-        "--size", type=int, choices=sorted(NET_DIMS), default=5, help="board size"
+        "--size",
+        type=int,
+        choices=sorted(NET_DIMS),
+        default=5,
+        help="board size",
+    )
+    _ = p.add_argument(
+        "--iters",
+        type=int,
+        default=100,
+        help="max loop iterations",
     )
-    _ = p.add_argument("--iters", type=int, default=10, help="max loop iterations")
     _ = p.add_argument(
-        "--games", type=int, default=100, help="selfplay games per iteration"
+        "--games",
+        type=int,
+        default=1000,
+        help="selfplay games per iteration",
     )
     _ = p.add_argument(
         "--replay-iters",
         type=int,
-        default=0,
+        default=4,
         help="number of previous self-play instances to mix in with current dataset during training",
     )
     _ = p.add_argument(
@@ -542,17 +554,41 @@ def main() -> None:
         help="initial self-play epsilon: in asymmetric self-play one side (chosen at random per game) plays a uniform random move with this probability while the greedy side has its plies recorded",
     )
     _ = p.add_argument(
-        "--eps-min", type=float, default=0.05, help="self-play epsilon floor"
+        "--eps-min",
+        type=float,
+        default=0.01,
+        help="self-play epsilon floor",
     )
     _ = p.add_argument(
-        "--eps-decay", type=float, default=0.9, help="per-iteration epsilon decay"
+        "--eps-decay",
+        type=float,
+        default=0.9,
+        help="per-iteration epsilon decay",
     )
-    _ = p.add_argument("--td-lambda", type=float, default=0.6, help="TD(lambda) lambda")
     _ = p.add_argument(
-        "--epochs", type=int, default=20, help="candidate training epochs"
+        "--td-lambda",
+        type=float,
+        default=0.7,
+        help="TD(lambda) lambda",
+    )
+    _ = p.add_argument(
+        "--epochs",
+        type=int,
+        default=20,
+        help="candidate training epochs",
+    )
+    _ = p.add_argument(
+        "--lr",
+        type=float,
+        default=1e-4,
+        help="learning rate",
+    )
+    _ = p.add_argument(
+        "--batch",
+        type=int,
+        default=128,
+        help="training batch size",
     )
-    _ = p.add_argument("--lr", type=float, default=1e-3, help="learning rate")
-    _ = p.add_argument("--batch", type=int, default=128, help="training batch size")
     _ = p.add_argument(
         "--val-frac",
         type=float,
@@ -560,22 +596,22 @@ def main() -> None:
         help="fraction of rows held out for validation",
     )
     _ = p.add_argument(
-        "--tournament-eps",
-        type=float,
-        default=0.01,
-        help="tournament epsilon (random-move probability)",
+        "--matchs",
+        type=int,
+        default=200,
+        help="how many matches in a tournament",
     )
     _ = p.add_argument(
-        "--tournament-games",
+        "--match-random-opening-length",
         type=int,
-        default=None,
-        help="tournament games (1/tournament-eps if unset)",
+        default=2,
+        help="length of random opening in a match",
     )
     _ = p.add_argument(
         "--promote",
         type=float,
-        default=0.0,
-        help="candidate win rate among non-drawn tournament games needed to promote",
+        default=0.53,
+        help="candidate win rate among non-drawn matches needed to promote",
     )
     _ = p.add_argument(
         "--procs",
@@ -587,16 +623,17 @@ def main() -> None:
         "--depth",
         type=int,
         default=4,
-        help="search depth for selfplay and the tournament",
+        help="search depth for selfplay and the matches",
     )
     _ = p.add_argument(
-        "--min-ply", type=int, default=4, help="record positions at this ply or later"
+        "--min-ply",
+        type=int,
+        default=8,
+        help="record positions at this ply or later",
     )
 
     args = p.parse_args()
 
-    if args.tournament_games is None:
-        args.tournament_games = max(100, int(round(1 / args.tournament_eps)))
     if args.procs is None:
         args.procs = max(1, (os.cpu_count() or 1) // 2)
 
@@ -608,12 +645,11 @@ def main() -> None:
             champ=champ,
             depth=args.depth,
             epochs=args.epochs,
-            eps_decay=args.eps_decay,
-            eps_min=args.eps_min,
-            eps_start=args.eps_start,
             games=args.games,
             iter=iter,
             lr=args.lr,
+            matches=args.matches,
+            match_random_opening_length=args.match_random_opening_length,
             min_ply=args.min_ply,
             procs=args.procs,
             promote_rate=args.promote,
@@ -621,8 +657,6 @@ def main() -> None:
             root=root,
             size=args.size,
             td_lambda=args.td_lambda,
-            tournament_eps=args.tournament_eps,
-            tournament_games=args.tournament_games,
             val_frac=args.val_frac,
         )
     print(f"\n== done, champion in include/weights_{args.size}.c ==")
diff --git a/src/selfplay.c b/src/selfplay.c
index 8cd11b9..e8ff506 100644
--- a/src/selfplay.c
+++ b/src/selfplay.c
@@ -25,7 +25,6 @@
 
 #include "actions.h"
 #include "negamax.h"
-#include "nn.h"
 #include "pv.h"
 #include "tak.h"
 #include "xorshift64.h"
@@ -34,18 +33,21 @@
 #error "selfplay must be built with -DRECORD_PV (see the selfplay Makefile target)"
 #endif
 
-#define MAX_PLY 256
+#define MAX_PLY 255
 #define MAX_CELLS (6 * 6)
 
 static tak_state_p state;
-static uint32_t min_ply;
 
 static float row_head[MAX_PLY][3];
 static uint8_t row_cells[MAX_PLY][MAX_CELLS];
 static uint8_t row_term[MAX_PLY];
 static float row_terminal_value[MAX_PLY];
+static uint32_t row_game_id[MAX_PLY];
 static uint32_t n_rows;
-static uint32_t game_id; // PID, tags every row so games can be grouped
+
+static uint32_t game_id_black;
+static uint32_t game_id_white;
+
 
 void negamax_display_progress(const uint8_t cur_depth,
                               const uint8_t init_depth,
@@ -54,47 +56,45 @@ void negamax_display_progress(const uint8_t cur_depth,
   (void)(init_depth);
   (void)(length);
 }
+
 static float win_value(const enum WIN_TYPE w) {
   switch (w) {
-  case WIN_FLAT_BLACK:
-  case WIN_ROAD_BLACK:
+    case WIN_FLAT_BLACK:
+    case WIN_ROAD_BLACK:
     return 1.0f;
-  case WIN_FLAT_WHITE:
-  case WIN_ROAD_WHITE:
+    case WIN_FLAT_WHITE:
+    case WIN_ROAD_WHITE:
     return -1.0f;
-  default:
+    default:
     return 0.0f;
   }
 }
 
-static int play_game(const uint32_t eps10000) {
+static int play_game(const uint8_t min_ply) {
   reset_state(state, state->board_size);
   n_rows = 0;
 
-  const enum COLOUR eps_side = (rand32() & 1) ? C_BLACK : C_WHITE;
-
   char ptn[16];
   for (uint32_t ply = 0; ply < MAX_PLY; ply++) {
-    negamax_generate(state);
-
-    if (state->current_colour == eps_side && rand32() % 10000 < eps10000) {
+    if (ply < min_ply) {
       uint32_t n_actions;
       action_t *actions = actions_generate(state, &n_actions);
       action_to_ptn(actions[rand32() % n_actions], ptn);
     } else {
+      negamax_generate(state);
+
       strcpy(ptn, negamax_ptn);
-    }
 
-    const pv_leaf_t *leaf = pv_get_leaf();
-    if (leaf->current_colour != eps_side && ply >= min_ply &&
-        n_rows < MAX_PLY) {
+      const pv_leaf_t *leaf = pv_get_leaf();
       memcpy(row_head[n_rows], leaf->head, sizeof(row_head[n_rows]));
       memcpy(row_cells[n_rows], leaf->cells, sizeof(row_cells[n_rows]));
       row_term[n_rows] = leaf->terminal;
       row_terminal_value[n_rows] = leaf->terminal_value;
+      row_game_id[n_rows] = state->current_colour == C_BLACK ? game_id_black : game_id_white;
       n_rows++;
     }
 
+
     const enum ACT_RESULT r = do_ptn(state, ptn);
     if (r == GAME_END)
       break;
@@ -118,7 +118,7 @@ static int play_game(const uint32_t eps10000) {
             row_head[k][2]);
     for (uint8_t c = 0; c < n_cells; c++)
       fprintf(stdout, "%u,", (unsigned)row_cells[k][c]);
-    fprintf(stdout, "%u,%.0f,%u,%.0f\n", game_id, R,
+    fprintf(stdout, "%u,%.0f,%u,%.0f\n", row_game_id[k], R,
             (unsigned)row_term[k], row_terminal_value[k]);
   }
 
@@ -126,27 +126,23 @@ static int play_game(const uint32_t eps10000) {
 }
 
 const char *usage =
-  "selfplay, one game of asymmetric self-play training data.\n\
-Usage: selfplay <size> <eps> <depth> <min_ply>\n\
-Plays a single asymmetric game of <size>x<size> self-play: one side plays the engine's best move at every ply (pure minimax) and the other side plays the engine's best move except that with probability eps it plays a uniform random legal move; the colour assignment of these two players is random. Only the deterministic side's plies are written to stdout, one CSV row per recorded search, from ply <min_ply> onwards:\n\
-\n\
-parity,white_flats,black_flats,<cell ids...>,game_id,R,terminal,leaf_R\n\
-nn_train.py runs many of these in parallel to generate training data.\n\
-\n\
-Copyright (C) 2026, tslil clingman\n\
-\n\
+  "selfplay, one game of asymmetric self-play training data.\n\n\
+Usage: selfplay <size> <depth> <min_ply>\n\n                            \
+Plays a single game of <size>x<size> self-play, with <min_ply> moves made randomly. Only the deterministic plies are written to stdout, one CSV row per recorded search, from ply <min_ply> onwards:\n\n \
+\tparity,white_flats,black_flats,<cell ids...>,game_id,R,terminal,leaf_R\n\n \
+nn_train.py runs many of these in parallel to generate training data.\n\n \
+Copyright (C) 2026, tslil clingman\n\n                                  \
 This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.";
 
 int main(int argc, char **argv) {
-  if (argc != 5) {
+  if (argc != 4) {
     puts(usage);
     return EXIT_FAILURE;
   }
 
   const uint8_t size = (uint8_t)atoi(argv[1]);
-  const uint32_t eps10000 = (uint32_t)(atof(argv[2]) * 10000.0 + 0.5);
-  negamax_search_depth = (uint8_t)atoi(argv[3]);
-  min_ply = (uint32_t)atoi(argv[4]);
+  negamax_search_depth = (uint8_t)atoi(argv[2]);
+  const uint8_t min_ply = (uint8_t)atoi(argv[3]);
 
   if (size != 5 && size != 6) {
     fprintf(stderr, "selfplay: size must be 5 or 6\n");
@@ -156,21 +152,23 @@ int main(int argc, char **argv) {
     fprintf(stderr, "selfplay: depth must be <= %d\n", NEGAMAX_MAX_DEPTH);
     return EXIT_FAILURE;
   }
-  if (eps10000 >= 10000) {
-    fprintf(stderr, "selfplay: eps must be < 1\n");
+  if (min_ply >= MAX_PLY) {
+    fprintf(stderr, "selfplay: min_ply must be < %d\n", MAX_PLY);
     return EXIT_FAILURE;
   }
 
   // Processes launched in the same second must not share a seed
-  game_id = (uint32_t)getpid();
   xors = ((uint64_t)time(NULL) ^ ((uint64_t)getpid() << 32)) *
     0x9E3779B97F4A7C15ULL +
-    0xC0FFEEEE;
+    0xC0FFEE;
+
+  game_id_black = rand32();
+  game_id_white = rand32();
 
   state = new_tak_state(size);
   negamax_init(size);
 
-  play_game(eps10000);
+  play_game(min_ply);
 
   negamax_free();
   free_tak_state(state);