From 6e90f1145b2c5c0f2dcc82d21ec45432274b8cad Mon Sep 17 00:00:00 2001 From: tslil Date: Fri, 11 Sep 2026 12:56:12 +0100 Subject: move from single candidate td-leaf with gating to tournament of a population The goal in this change was to overcome the rock-paper-scissors phenomenon that was happening in capability space by having on a single candidate learn to beat the most recent version of itself. In some sense this lead to rotating around in capability space instead of chasing monotonic improvements. I hope that flipping to a system in which many (N=5) candidates play each other and learn from their own set of diverse games in this pool, combined with --replay-iters, will lead to a given network having to have a much more comprehensive understanding of the space of tactics. I haven't yet had time to train this thorougly, will run those experiments on my desktop computer. --- src/match.c | 173 ---------------------- src/nn_train.py | 433 +++++++++++++++++++++++++++++--------------------------- src/selfplay.c | 125 +++++++++++----- 3 files changed, 310 insertions(+), 421 deletions(-) delete mode 100644 src/match.c (limited to 'src') diff --git a/src/match.c b/src/match.c deleted file mode 100644 index 459fab5..0000000 --- a/src/match.c +++ /dev/null @@ -1,173 +0,0 @@ -/* - match, one game of candidate weights against the candidate weights - - Copyright (C) 2026, tslil clingman - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - General Public License for more details. - - You should have received a copy of the GNU General Public License - along with ctak. If not, see . -*/ -#include -#include -#include -#include -#include - -#include "actions.h" -#include "negamax.h" -#include "nn.h" -#include "tak.h" -#include "xorshift64.h" - -// a little gymnastics to insert candidate neural networks -#include "weights_5.h" -#include "weights_6.h" -#include "cand_weights_5.c" -#include "cand_weights_6.c" - -static inline float five_cand(tak_state_p state) { - static float cur_board[FIVE_INP_NUM] NN_ALIGNED; - - nn_prepare_input(cur_board, state, cand_five_embedding); - return eval_net(cur_board, FIVE_INP_NUM, &cand_five_dense_weights[0][0], - cand_five_dense_biases, cand_five_logit_weights, cand_five_logit_bias, - FIVE_DENSE_NUM, state->ply & 1 ? 1.0f : -1.0f); -} - -static inline float six_cand(tak_state_p state) { - static float cur_board[SIX_INP_NUM] NN_ALIGNED; - - nn_prepare_input(cur_board, state, cand_six_embedding); - return eval_net(cur_board, SIX_INP_NUM, &cand_six_dense_weights[0][0], - cand_six_dense_biases, cand_six_logit_weights, cand_six_logit_bias, - SIX_DENSE_NUM, state->ply & 1 ? 1.0f : -1.0f); -} - - -#define MAX_PLY 255 - -static tak_state_p state; - -void negamax_display_progress(const uint8_t cur_depth, - const uint8_t init_depth, - const uint32_t length) { - (void)(cur_depth); - (void)(init_depth); - (void)(length); -} - -// =================================================================== -// Game driver -// =================================================================== - -static int play_game(const nn_function current_fn, const nn_function cand_fn, - const enum COLOUR current_fn_colour, - const uint8_t random_opening_length) { - reset_state(state, state->board_size); - char ptn[16]; - - for (uint32_t ply = 0; ply + negamax_search_depth < MAX_PLY; ply++) { - if (ply < random_opening_length) { - uint32_t n_actions; - action_t *actions = actions_generate(state, &n_actions); - action_to_ptn(actions[rand32() % n_actions], ptn); - } else { - const nn_function fn = - (state->current_colour == current_fn_colour) ? current_fn : cand_fn; - negamax_set_evaluator(fn); - negamax_generate(state); - strcpy(ptn, negamax_ptn); - } - - const enum ACT_RESULT r = do_ptn(state, ptn); - if (r == GAME_END) - break; - if (r != ACT_OK) { - fprintf(stderr, "match: unexpected result %d for ptn %s\n", r, ptn); - return -2; - } - } - - switch (state->won) { - case WIN_DRAW: - return -1; - case WIN_FLAT_BLACK: - case WIN_ROAD_BLACK: - return C_BLACK; - case WIN_FLAT_WHITE: - case WIN_ROAD_WHITE: - return C_WHITE; - default: - // ply cap reached with no winner: a draw - return -1; - } -} - -void do_game(const uint8_t size, const uint8_t random_opening_length, const enum COLOUR current_fn_colour) { - state = new_tak_state(size); - negamax_init(size); - - const nn_function current_fn = - (size == 5) ? &five_nn_fn : &six_nn_fn; - const nn_function cand_fn = (size == 5) ? &five_cand : &six_cand; - - const int winner = play_game(current_fn, cand_fn, current_fn_colour, random_opening_length); - - const char *name; - if (winner == -1) - name = "draw"; - else if (winner == (int)current_fn_colour) - name = "current"; - else - name = "candidate"; - printf("%s %u\n", name, state->ply); - - negamax_free(); - free_tak_state(state); -} - - // =================================================================== - // Main - // =================================================================== - - const char *usage = - "match, one Tak game of the current weights against the candidate.\n\n\ -usage: match \n\n\ -This binary is linked with both the include/weight_* files and the include/cand_weight_* files. It plays a single x game, with a random side choice and both sides using random moves at the start of the game. The result format is:\n\n\ -\t \n\n\ -nn_train.py runs many of these in parallel to score the candidate against the current weights.\n\n\ -Copyright (C) 2026, tslil clingman\n\n\ -This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version."; - -int main(int argc, char **argv) { - if (argc != 4) { - puts(usage); - return EXIT_FAILURE; - } - - const uint8_t size = (uint8_t)atoi(argv[1]); - const uint8_t random_opening_length = (uint8_t)atoi(argv[2]); - negamax_search_depth = (uint8_t)atoi(argv[3]); - - if (size != 5 && size != 6) { - fprintf(stderr, "match: size must be 5 or 6\n"); - return EXIT_FAILURE; - } - xors = ((uint64_t)time(NULL) ^ ((uint64_t)getpid() << 32)) * - 0x9E3779B97F4A7C15ULL + - 0xC0FFEE; - - do_game(size, random_opening_length, C_BLACK); - do_game(size, random_opening_length, C_WHITE); - - return EXIT_SUCCESS; -} diff --git a/src/nn_train.py b/src/nn_train.py index 0e08da7..2eb9b0c 100755 --- a/src/nn_train.py +++ b/src/nn_train.py @@ -35,6 +35,7 @@ from collections.abc import Iterator from dataclasses import dataclass +from math import log, ceil import argparse import concurrent.futures @@ -114,67 +115,43 @@ def td_lambda_targets( return targets -@dataclass(frozen=True) -class SelfPlayRecords: - positions: Tensor # [n, inp_dim] - game_ids: Tensor # [n] id of the game the position belongs to - results: Tensor # [n] final played result from black's perspective - terminals: Tensor # [n] true when the recorded position is a terminal leaf - terminal_values: Tensor # [n] exact value of a terminal leaf, otherwise 0 - parity_signs: Tensor # [n] ±1 converting the net's output to "absolute" perspective +@dataclass +class Performance: + win: int + loss: int + draw: int - @classmethod - def load_selfplay( - cls, fn: Path, size: int, device: torch.device - ) -> SelfPlayRecords: - # [net input] [game_id] [played result] [terminal flag] [leaf result] [parity sign] - df = pd.read_csv(fn, header=None) - inp_dim = NET_DIMS[size][0] - return cls( - positions=torch.tensor( - df.iloc[:, 0:inp_dim].to_numpy(dtype=np.float32), - dtype=torch.float32, - device=device, - ), - game_ids=torch.tensor( - df.iloc[:, inp_dim].to_numpy(dtype=np.int64), - device=device, - ), - results=torch.tensor( - df.iloc[:, inp_dim + 1].to_numpy(dtype=np.float32), - dtype=torch.float32, - device=device, - ), - terminals=torch.tensor( - df.iloc[:, inp_dim + 2].to_numpy(dtype=np.bool_), - dtype=torch.bool, - device=device, - ), - terminal_values=torch.tensor( - df.iloc[:, inp_dim + 3].to_numpy(dtype=np.float32), - dtype=torch.float32, - device=device, - ), - parity_signs=torch.tensor( - df.iloc[:, inp_dim + 4].to_numpy(dtype=np.float32), - dtype=torch.float32, - device=device, - ), - ) + def rate(self) -> float: + return float(self.win) / float(max(1, self.win + self.loss + self.draw)) + + def __str__(self) -> str: + width = ceil(log(max(1, self.win, self.loss, self.draw), 10)) + return f"+{str(self.win).zfill(width)}-{str(self.loss).zfill(width)}={str(self.draw).zfill(width)}" - @classmethod - def load_merge( - cls, files: list[Path], size: int, device: torch.device - ) -> SelfPlayRecords: - sprs = [SelfPlayRecords.load_selfplay(file, size, device) for file in files] - return cls( - positions=torch.concat([spr.positions for spr in sprs]), - game_ids=torch.concat([spr.game_ids for spr in sprs]), - results=torch.concat([spr.results for spr in sprs]), - terminals=torch.concat([spr.terminals for spr in sprs]), - terminal_values=torch.concat([spr.terminal_values for spr in sprs]), - parity_signs=torch.concat([spr.parity_signs for spr in sprs]), - ) + +@dataclass(frozen=True) +class CandidateRecords: + positions: list[Tensor] # [n, inp_dim] + game_ids: list[Tensor] # [n] + results: list[Tensor] # [n] + cand_results: list[Tensor] # [n] + terminals: list[Tensor] # [n] + terminal_values: list[Tensor] # [n] + parity_signs: list[Tensor] # [n] + + def performance(self) -> Performance: + result = Performance(0, 0, 0) + for game_id in torch.unique(self.game_ids): + in_game = self.game_ids == game_id + game_result = self.cand_results[in_game][0].item() + match game_result: + case 1: + result.win += 1 + case -1: + result.loss += 1 + case _: + result.draw += 1 + return result def make_training_data( self, model: Module, td_lambda: float @@ -201,6 +178,85 @@ class SelfPlayRecords: return self.positions, targets, self.parity_signs +@dataclass(frozen=True) +class PlayRecords: + positions: list[Tensor] # [candidate, n_i, inp_dim] + game_ids: list[Tensor] # [candidate, n_i] + results: list[Tensor] # [candidate, n_i] + cand_results: list[Tensor] # [candidate, n_i] + terminals: list[Tensor] # [candidate, n_i] + terminal_values: list[Tensor] # [candidate, n_i] + parity_signs: list[Tensor] # [candidate, n_i] + + def index(self, index: int) -> CandidateRecords: + return CandidateRecords( + positions=self.positions[index], + game_ids=self.game_ids[index], + results=self.results[index], + cand_results=self.cand_results[index], + terminals=self.terminals[index], + terminal_values=self.terminal_values[index], + parity_signs=self.parity_signs[index], + ) + + @classmethod + def load_selfplay(cls, fn, size): + df = pd.read_csv(fn, header=None) + inp_dim = NET_DIMS[size][0] + + fields = { + "parity_signs": df.iloc[:, 0].to_numpy(dtype=np.float32), + "positions": df.iloc[:, 1 : inp_dim + 1].to_numpy(dtype=np.float32), + "results": df.iloc[:, inp_dim + 1].to_numpy(dtype=np.float32), + "terminals": df.iloc[:, inp_dim + 2].to_numpy(dtype=np.bool_), + "terminal_values": df.iloc[:, inp_dim + 3].to_numpy(dtype=np.float32), + "game_ids": df.iloc[:, inp_dim + 4].to_numpy(dtype=np.int64), + "cand_results": df.iloc[:, inp_dim + 6].to_numpy(dtype=np.float32), + } + cand_ids = df.iloc[:, inp_dim + 5].to_numpy(dtype=np.int64) + + n_cands = int(cand_ids.max()) + 1 + gathered = {k: [] for k in fields} + for cid in range(n_cands): + mask = cand_ids == cid + for k, arr in fields.items(): + gathered[k].append(torch.from_numpy(arr[mask])) + + return cls(**gathered) + + @classmethod + def load_merge(cls, files: list[Path], size: int) -> PlayRecords: + sprs = [PlayRecords.load_selfplay(file, size) for file in files] + # for each field, gather [file0[i], file1[i], ...] for every index i + # and concat along dim 0 -> merged list indexed by cand id + return cls( + positions=[ + torch.concat(vals, dim=0) for vals in zip(*(s.positions for s in sprs)) + ], + game_ids=[ + torch.concat(vals, dim=0) for vals in zip(*(s.game_ids for s in sprs)) + ], + results=[ + torch.concat(vals, dim=0) for vals in zip(*(s.results for s in sprs)) + ], + cand_results=[ + torch.concat(vals, dim=0) + for vals in zip(*(s.cand_results for s in sprs)) + ], + terminals=[ + torch.concat(vals, dim=0) for vals in zip(*(s.terminals for s in sprs)) + ], + terminal_values=[ + torch.concat(vals, dim=0) + for vals in zip(*(s.terminal_values for s in sprs)) + ], + parity_signs=[ + torch.concat(vals, dim=0) + for vals in zip(*(s.parity_signs for s in sprs)) + ], + ) + + # =================================================================== # Network # =================================================================== @@ -225,7 +281,7 @@ class Net(Module): def fit( self, *, - self_play_records: SelfPlayRecords, + records: CandidateRecords, td_lambda: float, val_frac: float, epochs: int, @@ -233,9 +289,7 @@ class Net(Module): lr: float, ) -> tuple[float, float]: # data - positions, targets, parity_signs = self_play_records.make_training_data( - self, td_lambda - ) + positions, targets, parity_signs = records.make_training_data(self, td_lambda) n = len(positions) perm = torch.randperm(n, device=positions.device) n_val = max(1, int(n * val_frac)) @@ -258,7 +312,7 @@ class Net(Module): best_mse = float("nan") pbar = tqdm(range(epochs), desc="train", unit="ep") - for epoch in pbar: + for _ in pbar: self.train() order = torch.randperm(len(train_positions), device=positions.device) for start in range(0, len(train_positions), batch): @@ -295,13 +349,14 @@ class Net(Module): def to_c( self, + *, size: int, + id: int | None, path: Path, - performance: tuple[str, ...] = (), - cand: bool = False, + note: str, ) -> None: - prefix = ("cand_" if cand else "") + ("five" if size == 5 else "six") upper = "FIVE" if size == 5 else "SIX" + prefix = (f"cand_{id}_" if id is not None else "") + upper E = self.embedding.weight.detach().cpu().numpy() W = self.hidden.weight.detach().cpu().numpy() @@ -311,8 +366,7 @@ class Net(Module): with open(path, "w") as f: _ = f.write("/* generated by nn_train.py, do not edit by hand */\n") - for p in performance: - _ = f.write(f"/* {p} */\n") + _ = f.write(f"/* {note} */\n") _ = f.write(f'#include "weights_{size}.h"\n\n') _ = f.write( f"const float {prefix}_embedding[{len(E)}] =\n" @@ -338,7 +392,9 @@ class Net(Module): # =================================================================== -def fan_out(cmd: list[str], n: int, procs: int, tag: str, root: Path) -> Iterator[str]: +def fan_out( + cmds: list[list[str]], n: int, procs: int, tag: str, root: Path +) -> Iterator[str]: with concurrent.futures.ThreadPoolExecutor(max_workers=procs) as pool: futs = [ pool.submit( @@ -350,57 +406,32 @@ def fan_out(cmd: list[str], n: int, procs: int, tag: str, root: Path) -> Iterato text=True, ) for _ in range(n) + for cmd in cmds ] for fut in tqdm( - concurrent.futures.as_completed(futs), total=n, desc=tag, unit="game" + concurrent.futures.as_completed(futs), + total=n * len(cmds), + desc=tag, + unit="game", ): yield fut.result().stdout -def cand_copy_for(size: int, root: Path) -> None: +def cand_copy_for(size: int, id: int, root: Path) -> None: src = root / "include" / f"weights_{size}.c" if not src.is_file(): raise SystemExit(f"{src} not found; build a size-{size} champion first") - prefix = "five" if size == 5 else "six" - txt = src.read_text().replace(f"{prefix}_", f"cand_{prefix}_") - _ = (root / "include" / f"cand_weights_{size}.c").write_text(txt) - - -# =================================================================== -# Tournament -# =================================================================== - - -@dataclass(frozen=True) -class TournamentResult: - champ: int - cand: int - draws: int - - -def run_matches( - *, - depth: int, - games: int, - procs: int, - root: Path, - size: int, - match_random_opening_length: int, -) -> TournamentResult: - outs = list( - fan_out( - ["./match", str(size), f"{match_random_opening_length}", str(depth)], - games, - procs, - "match", - root, - ) - ) - winners = [line.split()[0] for line in outs if line.strip()] - champ = winners.count("current") - cand = winners.count("candidate") - draws = winners.count("draw") - return TournamentResult(champ=champ, cand=cand, draws=draws) + upper = "FIVE" if size == 5 else "SIX" + txt = src.read_text() + for sym in ( + "embedding", + "dense_weights", + "dense_biases", + "logit_weights", + "logit_bias", + ): + txt = txt.replace(f"{upper}_{sym}", f"cand_{id}_{upper}_{sym}") + _ = (root / "include" / f"cand_{id}_weights_{size}.c").write_text(txt) # =================================================================== @@ -408,120 +439,115 @@ def run_matches( # =================================================================== -def setup(size: int, root: Path) -> Net: +def setup(num_candidates: int, size: int, root: Path) -> list[Net]: inp_dim, hidden = NET_DIMS[size] - champ = Net(inp_dim, hidden) - champ.to_c( - size, - root / "include" / f"weights_{size}.c", - ("v000, random init",), - ) - cand_copy_for(6 if size == 5 else 5, root) - return champ + candidates = [Net(inp_dim, hidden) for _ in range(num_candidates)] + for id, cand in enumerate(candidates): + cand.to_c( + size=size, + id=id, + path=root / "include" / f"cand_{id}_weights_{size}.c", + note=f"{id=}, v000, random init", + ) + cand_copy_for(6 if size == 5 else 5, id, root) + return candidates def run_step( *, batch: int, - champ: Net, + candidates: list[Net], depth: int, epochs: int, games: int, iter: int, lr: float, - matches: int, - match_random_opening_length: int, min_ply: int, procs: int, - promote_rate: float, replay_iters: int, root: Path, size: int, td_lambda: float, val_frac: float, -) -> Net: - def make_csv(i: int) -> Path: +): + def csv_path(i: int) -> Path: return root / "data" / f"selfplay-{size}-v{i:03d}.csv" print(f"\n== iteration {iter} ==") + num_candidates = len(candidates) + cand_pairs = [ + (i, j) for i in range(num_candidates) for j in range(i + 1, num_candidates) + ] + # 1. rebuild - _ = subprocess.run(["make", "selfplay"], cwd=root, check=True, capture_output=True) - _ = subprocess.run(["make", "native"], cwd=root, check=True, capture_output=True) + for i, j in cand_pairs: + _ = subprocess.run( + ["make", f"selfplay-{i}-{j}"], cwd=root, check=True, capture_output=True + ) # 2. generate self-play data - csv = make_csv(iter) - print(f"selfplay: {size=}, {depth=}, {min_ply=}") + csv = csv_path(iter) + print(f"selfplay: {num_candidates=}, {size=}, {depth=}, {min_ply=}") out_rows: list[str] = [] for out in fan_out( - ["./selfplay", str(size), str(depth), str(min_ply)], + [ + [f"./selfplay-{i}-{j}", str(size), str(depth), str(min_ply)] + for (i, j) in cand_pairs + ], games, procs, "selfplay", root, ): out_rows.extend(out.splitlines()) + print("writing to", csv, "with", len(out_rows)) with open(csv, "w") as f: _ = f.write("\n".join(out_rows)) if out_rows: _ = f.write("\n") - # 3. fine-tune the champion on TD(lambda) targets -> candidate - device = next(champ.parameters()).device - csvs = [make_csv(i) for i in range(max(iter - replay_iters, 1), iter + 1)] + # 3. compute performance + csvs = [csv_path(i) for i in range(max(iter - replay_iters, 1), iter + 1)] print("Loading data from", [str(c) for c in csvs], "...", end=" ") - records = SelfPlayRecords.load_merge(csvs, size, device) - print(f"{records.results.shape[0]:,} rows") - cand = copy.deepcopy(champ) - - corr, mse = cand.fit( - self_play_records=records, - td_lambda=td_lambda, - val_frac=val_frac, - epochs=epochs, - batch=batch, - lr=lr, - ) - cand.to_c( - size, - root / "include" / f"cand_weights_{size}.c", - (f"v{iter:03d}, val_corr={corr:.4f}, val_mse={mse:.4f}",), - cand=True, - ) - - # 4. build the match driver (champion + candidate) - _ = subprocess.run(["make", "match"], cwd=root, check=True, capture_output=True) - - promote = True - rate = -1.0 - if promote_rate > 0: - # 5. tournament + decision - print(f"match: {size=}, {match_random_opening_length=}, {depth=}") - tournament_result = run_matches( - depth=depth, - games=matches, - procs=procs, - root=root, - size=size, - match_random_opening_length=match_random_opening_length, + records = PlayRecords.load_merge(csvs, size) + print("candidates have", [f"{r.shape[0]:,}" for r in records.results], "rows") + results = [ + (i, cand, records.index(i).performance()) for i, cand in enumerate(candidates) + ] + + for idx, _, perf in sorted(results, key=lambda t: t[2].rate(), reverse=True): + print(f"\t cand {idx:03d} {perf} {perf.rate() * 100.0:.2f}%") + + # 4. fine-tune the champion on TD(lambda) targets -> candidate + for idx, cand in enumerate(candidates): + corr, mse = cand.fit( + records=records.index(idx), + td_lambda=td_lambda, + val_frac=val_frac, + epochs=epochs, + batch=batch, + lr=lr, ) - decided = tournament_result.champ + tournament_result.cand - rate = tournament_result.cand / decided if decided else 0.0 - print( - f"champ {tournament_result.champ} - {tournament_result.cand} cand, {tournament_result.draws} draws, winrate={rate:.3f}" - ) - promote = decided >= int(0.25 * matches) and rate > promote_rate - - if promote: cand.to_c( - size, - root / "include" / f"weights_{size}.c", - (f"v{iter:03d}, val_corr={corr:.4f}, rate={rate:.4f}",), + size=size, + id=idx, + path=root / "include" / f"cand_{idx}_weights_{size}.c", + note=f"id={idx}, v{iter:03d}, val_corr={corr:.4f}, val_mse={mse:.4f}", ) - print(f"== candidate v{iter:03d} PROMOTED (include/weights_{size}.c) ==") - return cand - print(f"== candidate v{iter:03d} rejected ==") - return champ + + # 5. write champ + _, champ, perf = max(results, key=lambda t: t[2].rate()) + champ.to_c( + size=size, + id=None, + path=root / "include" / f"weights_{size}.c", + note=f"{iter=}, perf={str(perf)}", + ) + _ = subprocess.run(["make", "clean"], cwd=root, check=True, capture_output=True) + for i, j in cand_pairs: + (root / f"selfplay-{i}-{j}").unlink(missing_ok=True) + _ = subprocess.run(["make", "native"], cwd=root, check=True, capture_output=True) # =================================================================== @@ -551,14 +577,20 @@ def main() -> None: _ = p.add_argument( "--games", type=int, - default=1000, + default=100, help="selfplay games per iteration", ) + _ = p.add_argument( + "--num-candidates", + type=int, + default=5, + help="number of distinct candidates to train", + ) _ = p.add_argument( "--replay-iters", type=int, default=4, - help="number of previous self-play instances to mix in with current dataset during training", + help="number of previous data sources to retain", ) _ = p.add_argument( "--td-lambda", @@ -590,24 +622,6 @@ def main() -> None: default=0.05, help="fraction of rows held out for validation", ) - _ = p.add_argument( - "--matches", - type=int, - default=200, - help="how many matches in a tournament", - ) - _ = p.add_argument( - "--match-random-opening-length", - type=int, - default=2, - help="length of random opening in a match", - ) - _ = p.add_argument( - "--promote", - type=float, - default=0.53, - help="candidate win rate among non-drawn matches needed to promote", - ) _ = p.add_argument( "--procs", type=int, @@ -633,25 +647,22 @@ def main() -> None: args.procs = max(1, (os.cpu_count() or 1) // 2) (root / "data").mkdir(exist_ok=True) - champ = setup(size=args.size, root=root) + candidates = setup(num_candidates=args.num_candidates, size=args.size, root=root) - num_params = sum(p.numel() for p in champ.parameters() if p.requires_grad) + num_params = sum(p.numel() for p in candidates[0].parameters() if p.requires_grad) print(f"Number of trainable parameters: {num_params:,}") for iter in range(1, args.iters + 1): - champ = run_step( + run_step( batch=args.batch, - champ=champ, + candidates=candidates, depth=args.depth, epochs=args.epochs, games=args.games, iter=iter, lr=args.lr, - matches=args.matches, - match_random_opening_length=args.match_random_opening_length, min_ply=args.min_ply, procs=args.procs, - promote_rate=args.promote, replay_iters=args.replay_iters, root=root, size=args.size, diff --git a/src/selfplay.c b/src/selfplay.c index 42d82d2..f6736c0 100644 --- a/src/selfplay.c +++ b/src/selfplay.c @@ -30,25 +30,45 @@ #include "xorshift64.h" #ifndef RECORD_PV -#error "selfplay must be built with -DRECORD_PV (see the selfplay Makefile target)" +#error "selfplay must be built with -DRECORD_PV" #endif -#define MAX_PLY 255 -#define MAX_CELLS (6 * 6) +#ifndef CAND_A +#error "selfplay must be built with -DCAND_A=" +#endif -static tak_state_p state; +#ifndef CAND_B +#error "selfplay must be built with -DCAND_B=" +#endif + +#include CAND_A_WEIGHTS_5 +#include CAND_B_WEIGHTS_5 +#include CAND_A_WEIGHTS_6 +#include CAND_B_WEIGHTS_6 + +// sorry not sorry +#define STR2U(a, b) a##_##b +#define STR4U(a, b, c, d) a##_##b##_##c##_##d +#define STR4U_EXPAND(a, b, c, d) STR4U(a, b, c, d) +#define C(variant, size, suffix) STR4U_EXPAND(cand, STR2U(CAND, variant), size, suffix) +#define NETWORK(size, called) \ + static inline float STR2U(size, called)(tak_state_p state) { \ + static float cur_board[STR2U(size, INP_NUM)] NN_ALIGNED; \ + nn_prepare_input(cur_board, state, C(called, size, embedding)); \ + return eval_net(cur_board, STR2U(size, INP_NUM), &C(called, size, dense_weights)[0][0], \ + C(called, size, dense_biases), C(called, size, logit_weights), C(called, size, logit_bias), \ + STR2U(size, DENSE_NUM), state->ply & 1 ? 1.0f : -1.0f); \ + } -static float row_head[MAX_PLY][2]; -static uint8_t row_cells[MAX_PLY][MAX_CELLS]; -static uint8_t row_term[MAX_PLY]; -static float row_terminal_value[MAX_PLY]; -static float row_parity_sign[MAX_PLY]; -static uint32_t row_game_id[MAX_PLY]; -static uint32_t n_rows; +NETWORK(FIVE, A) +NETWORK(FIVE, B) +NETWORK(SIX, A) +NETWORK(SIX, B) -static uint32_t game_id_black; -static uint32_t game_id_white; +#define MAX_PLY 255 +#define MAX_CELLS (6 * 6) +static tak_state_p state; void negamax_display_progress(const uint8_t cur_depth, const uint8_t init_depth, @@ -58,30 +78,49 @@ void negamax_display_progress(const uint8_t cur_depth, (void)(length); } -static float win_value(const enum WIN_TYPE w) { +static int8_t win_value(const enum WIN_TYPE w) { switch (w) { case WIN_FLAT_BLACK: case WIN_ROAD_BLACK: - return 1.0f; + return 1; case WIN_FLAT_WHITE: case WIN_ROAD_WHITE: - return -1.0f; + return -1; default: - return 0.0f; + return 0; } } -static int play_game(const uint8_t min_ply) { +static int play_game(const uint8_t min_ply, const nn_function fn_1, + const nn_function fn_2, const uint32_t cand_id_1, + const uint32_t cand_id_2, + const uint32_t game_id, char record[][16], uint8_t replay) { + float row_head[MAX_PLY][2]; + uint8_t row_cells[MAX_PLY][MAX_CELLS]; + uint8_t row_term[MAX_PLY]; + float row_terminal_value[MAX_PLY]; + float row_parity_sign[MAX_PLY]; + uint32_t row_cand_id[MAX_PLY]; + int8_t row_mover_sign[MAX_PLY]; + uint32_t n_rows; + reset_state(state, state->board_size); n_rows = 0; char ptn[16]; for (uint32_t ply = 0; ply + negamax_search_depth < MAX_PLY; ply++) { if (ply < min_ply) { - uint32_t n_actions; - action_t *actions = actions_generate(state, &n_actions); - action_to_ptn(actions[rand32() % n_actions], ptn); + if (replay == 0) { + uint32_t n_actions; + action_t *actions = actions_generate(state, &n_actions); + action_to_ptn(actions[rand32() % n_actions], ptn); + strcpy(record[ply],ptn); + } else { + strcpy(ptn, record[ply]); + } } else { + const nn_function fn = (state->current_colour & 1) ? fn_1 : fn_2; + negamax_set_evaluator(fn); negamax_generate(state); strcpy(ptn, negamax_ptn); @@ -92,7 +131,8 @@ static int play_game(const uint8_t min_ply) { row_term[n_rows] = leaf->terminal; row_terminal_value[n_rows] = leaf->terminal_value; row_parity_sign[n_rows] = leaf->parity_sign; - row_game_id[n_rows] = state->current_colour == C_BLACK ? game_id_black : game_id_white; + row_cand_id[n_rows] = (state->current_colour&1) ? cand_id_1 : cand_id_2; + row_mover_sign[n_rows] = (state->current_colour&1) ? 1 : -1; n_rows++; } @@ -113,25 +153,25 @@ static int play_game(const uint8_t min_ply) { const uint8_t n_cells = state->board_size * state->board_size; // final outcome of the played game, from black's perspective - const float R = win_value(state->won); + const int8_t R = win_value(state->won); for (uint32_t k = 0; k < n_rows; k++) { - fprintf(stdout, "%.8f,%.8f,", row_head[k][0], row_head[k][1]); + fprintf(stdout, "%.1f,%.8f,%.8f,", row_parity_sign[k], row_head[k][0], row_head[k][1]); for (uint8_t c = 0; c < n_cells; c++) - fprintf(stdout, "%u,", (unsigned)row_cells[k][c]); - fprintf(stdout, "%u,%.0f,%u,%.0f,%.0f\n", row_game_id[k], R, - (unsigned)row_term[k], row_terminal_value[k], row_parity_sign[k]); + fprintf(stdout, "%u,", row_cells[k][c]); + fprintf(stdout, "%i,%u,%.0f,%u,%u,%i\n", R, row_term[k], row_terminal_value[k], game_id, row_cand_id[k], + row_mover_sign[k] * R); } return (int)n_rows; } const char *usage = - "selfplay, one game of asymmetric self-play training data.\n\n\ + "selfplay, one game of training data between candidates.\n\n\ Usage: selfplay \n\n\ -Plays a single game of x self-play, with moves made randomly. Only the deterministic plies are written to stdout, one CSV row per recorded search, from ply onwards:\n\n\ -\tmover_flats,opponent_flats,,game_id,R,terminal,leaf_R,parity_sign\n\n\ -nn_train.py runs many of these in parallel to generate training data.\n\n\ +Plays a single game of x, with moves made randomly. Only the deterministic plies are written to stdout, one CSV row per recorded search, from ply onwards:\n\n\ +\tparity_sign,mover_flats,opponent_flats,,R,terminal,leaf_R,game_id,cand_id,cand_R\n\n\ +R is the outcome from black's perspective, cand_R the outcome from the recorded candidate's perspective. nn_train.py runs many of these in parallel to generate training data.\n\n\ Copyright (C) 2026, tslil clingman\n\n\ This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version."; @@ -160,16 +200,27 @@ int main(int argc, char **argv) { // Processes launched in the same second must not share a seed xors = ((uint64_t)time(NULL) ^ ((uint64_t)getpid() << 32)) * - 0x9E3779B97F4A7C15ULL + - 0xC0FFEE; - - game_id_black = rand32(); - game_id_white = rand32(); + 0x9E3779B97F4A7C15ULL + + 0xC0FFEE; state = new_tak_state(size); negamax_init(size); - play_game(min_ply); + char record[min_ply][16]; + + play_game(min_ply, + size == 5 ? FIVE_A : SIX_A, size == 5 ? FIVE_B : SIX_B, + CAND_A, + CAND_B, + rand32(), + record, 0); + play_game(min_ply, + size == 5 ? FIVE_B : SIX_B, size == 5 ? FIVE_A : SIX_A, + CAND_B, + CAND_A, + rand32(), + record, 1); + negamax_free(); free_tak_state(state); -- cgit v1.3.1