diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/ctlm.c | 6 | ||||
| -rw-r--r-- | src/match.c | 196 | ||||
| -rwxr-xr-x[-rw-r--r--] | src/nn_train.py | 643 | ||||
| -rw-r--r-- | src/pptdb.c | 271 | ||||
| -rw-r--r-- | src/selfplay.c | 170 |
5 files changed, 933 insertions, 353 deletions
@@ -401,11 +401,11 @@ help, info, load <file.ptn>, log, new, play (b|w), self-play, square\ } else if (!strcmp(line, "info")) { print_info(); } else if (!strcmp(line, "eval")) { - float eval = nn1986_evaluate_black_win(state) * 100; + float eval = (state->board_size == 5 ? five_nn_fn(state) : six_nn_fn(state))*100; if (state->ply & 1) { - printf("Black heuristic chance: %s%.2f%s\n", blk, eval, rst); + printf("Black heuristic: %s%.2f%s\n", blk, eval, rst); } else { - printf("White heruistic chance: %s%.2f%s\n", wht, -eval, rst); + printf("White heruistic: %s%.2f%s\n", wht, -eval, rst); } } else if (!strcmp(line, "log")) { puts(gamelog); diff --git a/src/match.c b/src/match.c new file mode 100644 index 0000000..1c706cd --- /dev/null +++ b/src/match.c @@ -0,0 +1,196 @@ +/* + match, one game of candidate weights against the candidate weights + + Copyright (C) 2026, tslil clingman + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with ct. If not, see <https://www.gnu.org/licenses/>. +*/ +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <time.h> +#include <unistd.h> + +#include "actions.h" +#include "negamax.h" +#include "nn.h" +#include "tak.h" +#include "xorshift64.h" + +// a little gymnastics to insert candidate neural networks +#include "weights_5.h" +#include "weights_6.h" +#include "cand_weights_5.c" +#include "cand_weights_6.c" + +static inline float five_cand(tak_state_p state) { + static float cur_board[FIVE_INP_NUM] NN_ALIGNED; + + nn_prepare_input(cur_board, state); + return eval_net(cur_board, FIVE_INP_NUM, &cand_five_dense_weights[0][0], + cand_five_dense_biases, cand_five_logit_weights, cand_five_logit_bias, + FIVE_DENSE_NUM); +} + +static inline float six_cand(tak_state_p state) { + static float cur_board[SIX_INP_NUM] NN_ALIGNED; + + nn_prepare_input(cur_board, state); + return eval_net(cur_board, SIX_INP_NUM, &cand_six_dense_weights[0][0], + cand_six_dense_biases, cand_six_logit_weights, cand_six_logit_bias, + SIX_DENSE_NUM); +} + + +#define MAX_PLY 250 + +static tak_state_p state; + +// =================================================================== +// negamax progress hook (silent) +// =================================================================== + +void negamax_display_progress(const uint8_t cur_depth, + const uint8_t init_depth, + const uint32_t length) { + (void)(cur_depth); + (void)(init_depth); + (void)(length); +} + +// =================================================================== +// Game driver +// =================================================================== + +// Returns the winner's colour, or -1 for a draw / abort +static int play_game(const nn_function current_fn, const nn_function cand_fn, + const enum COLOUR current_fn_colour, const uint32_t eps10000) { + reset_state(state, state->board_size); + char ptn[16]; + + for (uint32_t ply = 0; ply < MAX_PLY; ply++) { + // Whose evaluator is it? + const nn_function fn = + (state->current_colour == current_fn_colour) ? current_fn : cand_fn; + negamax_set_evaluator(fn); + + if (rand32() % 10000 < eps10000) { + // Epsilon: uniform random legal move + uint32_t n_actions; + action_t *actions = actions_generate(state, &n_actions); + action_to_ptn(actions[rand32() % n_actions], ptn); + } else { + // Greedy: the side's best move + negamax_generate(state); + strcpy(ptn, negamax_ptn); + } + + const enum ACT_RESULT r = do_ptn(state, ptn); + if (r == GAME_END) + break; + if (r != ACT_OK) { + fprintf(stderr, "match: unexpected result %d for ptn %s\n", r, ptn); + return -2; + } + } + + switch (state->won) { + case WIN_DRAW: + return -1; + case WIN_FLAT_BLACK: + case WIN_ROAD_BLACK: + return C_BLACK; + case WIN_FLAT_WHITE: + case WIN_ROAD_WHITE: + return C_WHITE; + default: + fprintf(stderr, "match: game did not end cleanly (won=%d)\n", state->won); + return -2; + } +} + +// =================================================================== +// Main +// =================================================================== + +const char *license = + "match, one Tak game of the current weights against the candidate.\n\ +\n \ +usage: match <size> <eps> <depth>\n\ +\n\ +This binary is linked with both the include/weight_* files and the include/cand_weight_* files. It plays a single game, with a random side choice and both sides using epsilon-greedy move noise. The result format is:\n\ +\n\ + <current|candidate|draw> <plies>\n\ +\n\ +nn_train.py runs many of these in parallel to score the candidate against the current weights.\n\ +\n\ +Copyright (C) 2026, tslil clingman\n\ +\n\ +This program is free software: you can redistribute it and/or modify\n\ +it under the terms of the GNU General Public License as published by\n\ +the Free Software Foundation, either version 3 of the License, or\n\ +(at your option) any later version."; + +int main(int argc, char **argv) { + if (argc != 4) { + puts(license); + return EXIT_FAILURE; + } + + const uint8_t size = (uint8_t)atoi(argv[1]); + const uint32_t eps10000 = (uint32_t)(atof(argv[2]) * 10000.0 + 0.5); + negamax_search_depth = (uint8_t)atoi(argv[3]); + + if (size != 5 && size != 6) { + fprintf(stderr, "match: size must be 5 or 6\n"); + return EXIT_FAILURE; + } + if (eps10000 >= 10000) { + fprintf(stderr, "match: eps must be < 1\n"); + return EXIT_FAILURE; + } + + // Processes launched in the same second must not share a seed + xors = ((uint64_t)time(NULL) ^ ((uint64_t)getpid() << 32)) * + 0x9E3779B97F4A7C15ULL + + 0xDEADBEEF; + + state = new_tak_state(size); + negamax_init(size); + + const nn_function current_fn = + (size == 5) ? &five_nn_fn : &six_nn_fn; + const nn_function cand_fn = + (size == 5) ? &five_cand + : &six_cand; + + const enum COLOUR current_fn_colour = rand32() % 2 ? C_WHITE : C_BLACK; + const int winner = play_game(current_fn, cand_fn, current_fn_colour, eps10000); + + const char *name; + if (winner == -2) + name = "abort"; + else if (winner == -1) + name = "draw"; + else if (winner == (int)current_fn_colour) + name = "current"; + else + name = "candidate"; + printf("%s %u\n", name, state->ply); + + negamax_free(); + free_tak_state(state); + + return EXIT_SUCCESS; +} diff --git a/src/nn_train.py b/src/nn_train.py index 03852f5..52549d8 100644..100755 --- a/src/nn_train.py +++ b/src/nn_train.py @@ -1,6 +1,24 @@ -# cnn_train.py, train a small CNN to recognise winning Tak positions +#!/usr/bin/env -S uv run --script # -# Copyright (C) 2021, tslil clingman +# /// script +# requires-python = ">=3.14" +# dependencies = [ +# "pandas>=3.0.5", +# "torch>=2.14.0", +# "tqdm>=4.70.0", +# ] +# +# [tool.uv.sources] +# torch = { index = "pytorch-cpu" } +# +# [[tool.uv.index]] +# name = "pytorch-cpu" +# url = "https://download.pytorch.org/whl/cpu" +# explicit = true +# /// + +# +# This file is part of ctak. # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -13,95 +31,562 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License -# along with this program. If not, see <https://www.gnu.org/licenses/>. +# along with ctak. If not, see <https://www.gnu.org/licenses/>. +from collections.abc import Iterator +from dataclasses import dataclass + +import argparse +import concurrent.futures +import copy +import os +from pathlib import Path +import subprocess -# Loading data -import pandas as pd import numpy as np +import pandas as pd +import torch +from tqdm import tqdm + +from torch.nn import MSELoss, Linear, Module +from torch import Tensor + + +# =================================================================== +# Net dimensions +# =================================================================== + +# (input dim, hidden dim) per board size; must match include/weights_<size>.h +NET_DIMS: dict[int, tuple[int, int]] = { + 5: (28, 64), # FIVE_INP_NUM, FIVE_DENSE_NUM + 6: (39, 64), # SIX_INP_NUM, SIX_DENSE_NUM +} + + +# =================================================================== +# C weight formatting +# =================================================================== + + +def _fmt_flat(a: np.ndarray, per_line: int = 8) -> str: + a = np.ravel(np.asarray(a, dtype=np.float32)) + lines: list[str] = [] + for i in range(0, len(a), per_line): + chunk = ", ".join(f"{v:.9g}" for v in a[i : i + per_line]) + if i + per_line < len(a): + chunk += "," + lines.append(" " + chunk) + return "\n".join(lines) + + +def _fmt_matrix(mat: np.ndarray, per_line: int = 8) -> str: + mat = np.asarray(mat, dtype=np.float32) + rows: list[str] = [] + for r in mat: + chunks = [ + ", ".join(f"{v:.9g}" for v in r[i : i + per_line]) + for i in range(0, len(r), per_line) + ] + rows.append(" {" + ",\n ".join(chunks) + "},") + return "\n".join(rows) + + +# =================================================================== +# Self-play records +# =================================================================== + + +def td_lambda_targets( + values: Tensor, + result: float, + td_lambda: float, +) -> Tensor: + # values: V(s_t) for t = 0..T-1, shape [T] + # result: final result R of the game + + # This is ema, computed backwards + # target[T-1] = R + # target[t] = (1 - lambda) * V(s_{t+1}) + lambda * target[t+1] + length = len(values) + targets = torch.empty_like(values) + targets[-1] = result + for ply in range(length - 2, -1, -1): + targets[ply] = (1.0 - td_lambda) * values[ply + 1] + td_lambda * targets[ + ply + 1 + ] + return targets + + +@dataclass(frozen=True) +class SelfPlayRecords: + positions: Tensor # [n, inp_dim] feature vector of each position + game_ids: Tensor # [n] id of the game the position belongs to + plies: Tensor # [n] ply number (position is before this move) + results: Tensor # [n] final result from black's perspective (+1 black won, -1 white won, 0 draw) + + @classmethod + def load_selfplay( + cls, fn: Path, size: int, device: torch.device + ) -> SelfPlayRecords: + # [ net input: 0-inp_dim ] [game_id] [ply] [game result] + df = pd.read_csv(fn, header=None) + inp_dim = NET_DIMS[size][0] + return cls( + positions=torch.tensor( + df.iloc[:, 0:inp_dim].to_numpy(dtype=np.float32), + dtype=torch.float32, + device=device, + ), + game_ids=torch.tensor( + df.iloc[:, inp_dim].to_numpy(dtype=np.int64), + device=device, + ), + plies=torch.tensor( + df.iloc[:, inp_dim + 1].to_numpy(dtype=np.int64), + device=device, + ), + results=torch.tensor( + df.iloc[:, inp_dim + 2].to_numpy(dtype=np.float32), + dtype=torch.float32, + device=device, + ), + ) + + def make_training_data( + self, model: Module, td_lambda: float + ) -> tuple[Tensor, Tensor]: + model.eval() + with torch.no_grad(): + values = model(self.positions) + + targets = torch.empty_like(values) + + for game_id in torch.unique(self.game_ids): + in_game = self.game_ids == game_id + game_values = values[in_game] + game_result = self.results[in_game][0].item() + targets[in_game] = td_lambda_targets(game_values, game_result, td_lambda) + + return self.positions, targets + + +# =================================================================== +# Network +# =================================================================== + + +class Net(Module): + def __init__(self, inp_dim: int, hidden: int) -> None: + super().__init__() + self.hidden = Linear(inp_dim, hidden, bias=True) + self.output = Linear(hidden, 1, bias=True) + + def forward(self, x: Tensor) -> Tensor: + hidden = torch.relu(self.hidden(x)) + return torch.tanh(self.output(hidden)).squeeze(-1) + + def fit( + self, + *, + self_play_records: SelfPlayRecords, + td_lambda: float, + val_frac: float, + epochs: int, + batch: int, + lr: float, + ) -> tuple[float, float]: + # data + positions, targets = self_play_records.make_training_data(self, td_lambda) + n = len(positions) + perm = torch.randperm(n, device=positions.device) + n_val = max(1, int(n * val_frac)) + val_idx = perm[:n_val] + train_idx = perm[n_val:] + + val_positions = positions[val_idx] + val_targets = targets[val_idx] + train_positions = positions[train_idx] + train_targets = targets[train_idx] + + # training widgets + optimizer = torch.optim.Adam(self.parameters(), lr=lr) + loss_fn = MSELoss() + + best_corr = -1.0 + best_state: dict[str, Tensor] | None = None + best_mse = float("nan") + + pbar = tqdm(range(epochs), desc="train", unit="ep") + for epoch in pbar: + self.train() + order = torch.randperm(len(train_positions), device=positions.device) + for start in range(0, len(train_positions), batch): + batch_idx = order[start : start + batch] + optimizer.zero_grad() + predictions = self(train_positions[batch_idx]) + loss = loss_fn(predictions, train_targets[batch_idx]) + loss.backward() + optimizer.step() + + with torch.no_grad(): + self.eval() + predictions = self(val_positions) + mse = loss_fn(predictions, val_targets).item() + corr = torch.corrcoef(torch.stack([predictions, val_targets]))[ + 0, 1 + ].item() + + pbar.set_postfix(val_corr=f"{corr:.4f}", val_mse=f"{mse:.4f}") + + if corr > best_corr: + best_corr = corr + best_state = { + name: param.clone() for name, param in self.state_dict().items() + } + best_mse = mse + + if best_state is not None: + self.load_state_dict(best_state) + + return best_corr, best_mse + + def to_c( + self, + size: int, + path: Path, + performance: tuple[str, ...] = (), + cand: bool = False, + ) -> None: + prefix = ("cand_" if cand else "") + ("five" if size == 5 else "six") + upper = "FIVE" if size == 5 else "SIX" + + W = self.hidden.weight.detach().cpu().numpy() + B = self.hidden.bias.detach().cpu().numpy() + LW = self.output.weight.detach().cpu().numpy().ravel() + LB = float(self.output.bias.detach().cpu().item()) + + with open(path, "w") as f: + _ = f.write("/* generated by nn_train.py, do not edit by hand */\n") + for p in performance: + _ = f.write(f"/* {p} */\n") + _ = f.write(f'#include "weights_{size}.h"\n\n') + _ = f.write( + f"const float {prefix}_dense_weights[{upper}_DENSE_NUM]" + f"[{upper}_INP_NUM] NN_ALIGNED =\n{{\n{_fmt_matrix(W)}\n}};\n\n" + ) + _ = f.write( + f"const float {prefix}_dense_biases[{upper}_DENSE_NUM] =\n" + f"{{\n{_fmt_flat(B)}\n}};\n\n" + ) + _ = f.write( + f"const float {prefix}_logit_weights[{upper}_DENSE_NUM]" + f" NN_ALIGNED =\n{{\n{_fmt_flat(LW)}\n}};\n\n" + ) + _ = f.write(f"const float {prefix}_logit_bias = {LB:.9g};\n") + + +# =================================================================== +# Process helpers +# =================================================================== + + +def fan_out(cmd: list[str], n: int, procs: int, tag: str, root: Path) -> Iterator[str]: + with concurrent.futures.ThreadPoolExecutor(max_workers=procs) as pool: + futs = [ + pool.submit( + subprocess.run, + cmd, + cwd=root, + check=True, + capture_output=True, + text=True, + ) + for _ in range(n) + ] + for fut in tqdm( + concurrent.futures.as_completed(futs), total=n, desc=tag, unit="game" + ): + yield fut.result().stdout + + +def cand_copy_for(size: int, root: Path) -> None: + src = root / "include" / f"weights_{size}.c" + if not src.is_file(): + raise SystemExit(f"{src} not found; build a size-{size} champion first") + prefix = "five" if size == 5 else "six" + txt = src.read_text().replace(f"{prefix}_", f"cand_{prefix}_") + _ = (root / "include" / f"cand_weights_{size}.c").write_text(txt) + + +# =================================================================== +# Tournament +# =================================================================== + + +@dataclass(frozen=True) +class TournamentResult: + champ: int + cand: int + draws: int + cand_rate: float + + +def run_matches( + *, + size: int, + depth: int, + tournament_eps: float, + games: int, + procs: int, + root: Path, +) -> TournamentResult: + outs = list( + fan_out( + ["./match", str(size), f"{tournament_eps:.4f}", str(depth)], + games, + procs, + "match", + root, + ) + ) + winners = [line.split()[0] for line in outs if line.strip()] + champ = winners.count("current") + cand = winners.count("candidate") + draws = winners.count("draw") + aborts = len(winners) - champ - cand - draws + played = len(winners) - aborts + return TournamentResult( + champ=champ, + cand=cand, + draws=draws, + cand_rate=(cand + 0.5 * draws) / played if played else 0.0, + ) + + +# =================================================================== +# The bootstrap loop +# =================================================================== + + +def setup(size: int, root: Path) -> Net: + inp_dim, hidden = NET_DIMS[size] + champ = Net(inp_dim, hidden) + champ.to_c( + size, + root / "include" / f"weights_{size}.c", + ("v000, random init",), + ) + cand_copy_for(6 if size == 5 else 5, root) + return champ + + +def run_step( + *, + iter: int, + size: int, + eps_min: float, + champ: Net, + eps_start: float, + eps_decay: float, + depth: int, + min_ply: int, + games: int, + procs: int, + td_lambda: float, + val_frac: float, + epochs: int, + batch: int, + lr: float, + tournament_eps: float, + tournament_games: int, + promote_rate: float, + root: Path, +) -> Net: + print(f"\n== iteration {iter} ==") + + # 1. self-play with the installed champion (include/weights_<size>.c) + _ = subprocess.run(["make", "selfplay"], cwd=root, check=True, capture_output=True) + + # 2. generate self-play data (one game per process, in parallel) + eps = max(eps_min, eps_start * (eps_decay ** (iter - 1))) + csv = root / "data" / f"selfplay-{size}-v{iter:03d}.csv" + print(f"selfplay, eps={eps:.4f}, depth {depth}, {games} games on {procs} procs") + out_rows: list[str] = [] + for out in fan_out( + ["./selfplay", str(size), f"{eps:.4f}", str(depth), str(min_ply)], + games, + procs, + "selfplay", + root, + ): + out_rows.extend(out.splitlines()) + with open(csv, "w") as f: + _ = f.write("\n".join(out_rows)) + if out_rows: + _ = f.write("\n") + rows = len(out_rows) + print(f" {rows} rows") + + # 3. fine-tune the champion on TD(lambda) targets -> candidate + device = next(champ.parameters()).device + records = SelfPlayRecords.load_selfplay(csv, size, device) + cand = copy.deepcopy(champ) -# Output of weights -from tensorflow import transpose + corr, mse = cand.fit( + self_play_records=records, + td_lambda=td_lambda, + val_frac=val_frac, + epochs=epochs, + batch=batch, + lr=lr, + ) + cand.to_c( + size, + root / "include" / f"cand_weights_{size}.c", + (f"v{iter:03d}, val_corr={corr:.4f}, val_mse={mse:.4f}",), + cand=True, + ) -# Building models -from tensorflow.keras.models import Model -from tensorflow.keras.layers import Input, Dense + # 4. build the match driver (champion + candidate linked) + _ = subprocess.run(["make", "match"], cwd=root, check=True, capture_output=True) + # 5. tournament + decision + tournament_result = run_matches( + root=root, + size=size, + depth=depth, + tournament_eps=tournament_eps, + games=tournament_games, + procs=procs, + ) + decided = tournament_result.champ + tournament_result.cand + rate = tournament_result.cand / decided if decided else 0.0 + print( + f"champ {tournament_result.champ} - {tournament_result.cand} cand, {tournament_result.draws} draws, " + + f"winrate={rate:.3f}, cand_rate={tournament_result.cand_rate:.3f}" + ) + promote = decided >= int(0.25 * tournament_games) and rate >= promote_rate -def load_data(size): - output_len = 2 - # Load training data - tr_fn = "data/training-"+str(size)+".csv" - training_csv = pd.read_csv(tr_fn) - training_data = training_csv - training_input = np.array(training_data.iloc[:, 0:-output_len]) - training_outcome = np.array(training_data.iloc[:, -output_len:]) - # Load validation data - val_fn = "data/validation-"+str(size)+".csv" - val_data = pd.read_csv(val_fn) - val_input = np.array(val_data.iloc[:, 0:-output_len]) - val_outcome = np.array(val_data.iloc[:, -output_len:]) - return ((training_input, training_outcome), (val_input, val_outcome)) + if promote: + cand.to_c( + size, + root / "include" / f"weights_{size}.c", + (f"v{iter:03d}, val_corr={corr:.4f}",), + ) + print(f"== candidate v{iter:03d} PROMOTED (include/weights_{size}.c) ==") + return cand + print(f"== candidate v{iter:03d} rejected ==") + return champ -def train(size, model, data, iterations=1, epochs=10, batch=None): - (tra_input, tra_outcome), (val_input, val_outcome) = data - results = [] - for i in range(0, iterations): - print("Iteration {0}/{1}".format(i+1, iterations)) - model.fit(tra_input, tra_outcome, epochs=epochs, - validation_data=(val_input, val_outcome), - verbose=True, batch_size=batch) - val_res = model.evaluate(val_input, val_outcome, verbose=False) - tra_res = model.evaluate(tra_input, tra_outcome, verbose=False) - results.append((tra_res, val_res)) - print(val_res) - write_weights(size, model, str(i+1).zfill(len(str(iterations))), (tra_res, val_res)) - print("\nScores") - for i, data in enumerate(results): - print(f"Iteration {i+1}: {data}") - return results +# =================================================================== +# Main +# =================================================================== -def make_model(size, magic): - inputs = Input(shape=(size * size + 3,)) - model = inputs - model = Dense(magic, activation="relu", use_bias=True)(model) - model = Dense(2, activation="relu", use_bias=True)(model) - model = Model(inputs=inputs, outputs=model) - model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy']) - model.summary() - return model +def main() -> None: + root = Path(__file__).resolve().parent.parent + p = argparse.ArgumentParser( + description="the TD(lambda) self-play bootstrap loop", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + _ = p.add_argument( + "--size", type=int, choices=sorted(NET_DIMS), default=5, help="board size" + ) + _ = p.add_argument("--iters", type=int, default=10, help="max loop iterations") + _ = p.add_argument( + "--games", type=int, default=2000, help="selfplay games per iteration" + ) + _ = p.add_argument( + "--eps-start", + type=float, + default=0.15, + help="initial self-play epsilon (random-move probability); kept close " + "to --tour-eps so the value net trains on the policy it is deployed " + "under", + ) + _ = p.add_argument("--eps-min", type=float, default=0.05, help="epsilon floor") + _ = p.add_argument( + "--eps-decay", type=float, default=0.8, help="per-iteration epsilon decay" + ) + _ = p.add_argument( + "--depth", + type=int, + default=4, + help="search depth for selfplay AND the tournament", + ) + _ = p.add_argument( + "--min-ply", type=int, default=8, help="record positions at this ply or later" + ) + _ = p.add_argument("--td-lambda", type=float, default=0.6, help="TD(lambda) lambda") + _ = p.add_argument( + "--epochs", type=int, default=20, help="candidate training epochs" + ) + _ = p.add_argument("--lr", type=float, default=1e-3, help="learning rate") + _ = p.add_argument("--batch", type=int, default=128, help="training batch size") + _ = p.add_argument( + "--val-frac", + type=float, + default=0.05, + help="fraction of rows held out for validation", + ) + _ = p.add_argument( + "--tournament-eps", + type=float, + default=0.05, + help="tournament epsilon (random-move probability)", + ) + _ = p.add_argument( + "--tournament-games", + type=int, + default=None, + help="tournament games (10/tour_eps if unset)", + ) + _ = p.add_argument( + "--promote", + type=float, + default=0.57, + help="candidate win rate among non-drawn tournament games needed to promote (~p<0.03 one-sided binomial at the default 200 games)", + ) + _ = p.add_argument( + "--procs", + type=int, + default=None, + help="parallel game processes (half the CPUs if unset)", + ) + args = p.parse_args() + if args.tournament_games is None: + args.tournament_games = max(20, int(round(10 / args.tournament_eps))) + if args.procs is None: + args.procs = max(1, (os.cpu_count() or 1) // 2) -def write_weights(size, model, iteration, performance): - def fix(val): - string = str(np.array(val).tolist()) - string = string.replace("[", "{").replace("]", "}") - return string - size_str = "five" if size==5 else "six" - dense_weights = transpose(model.trainable_variables[0], perm=[1, 0]) - dense_biases = model.trainable_variables[1] - output_weights = transpose(model.trainable_variables[2], perm=[1, 0]) - output_bias = model.trainable_variables[3] - # Prepare output - to_output = [(f"{size_str}_dense_weights[{size_str.upper()}_DENSE_NUM][{size_str.upper()}_INP_NUM]", dense_weights), - (f"{size_str}_dense_biases[{size_str.upper()}_DENSE_NUM]", dense_biases), - (f"{size_str}_output_weights[2][{size_str.upper()}_DENSE_NUM]", output_weights), - (f"{size_str}_output_bias[2]", output_bias)] - # Write to file - f = open(f"{size_str}_weights-"+iteration+".txt", "w") - f.write("/*\n") - model.summary(print_fn=lambda l: f.write(" * "+l+"\n")) - f.write(" * "+str(performance)+"\n*/\n\n") - f.write(f"#include \"weights_{size}.h\"\n\n") - for (name, val) in to_output: - f.write("const float "+name+" =\n"+fix(val)+";\n\n") - f.close() + (root / "data").mkdir(exist_ok=True) + champ = setup(size=args.size, root=root) + for iter in range(1, args.iters + 1): + champ = run_step( + iter=iter, + size=args.size, + eps_min=args.eps_min, + champ=champ, + eps_start=args.eps_start, + eps_decay=args.eps_decay, + depth=args.depth, + min_ply=args.min_ply, + games=args.games, + procs=args.procs, + td_lambda=args.td_lambda, + val_frac=args.val_frac, + epochs=args.epochs, + batch=args.batch, + lr=args.lr, + tournament_eps=args.tournament_eps, + tournament_games=args.tournament_games, + promote_rate=args.promote, + root=root, + ) + print(f"\n== done, champion in include/weights_{args.size}.c ==") -for size in (5,6): - data = load_data(size) - model = make_model(size, 64) - print("Before training", model.evaluate(data[1][0], data[1][1], verbose=False, batch_size=16)) - results = train(size, model, data, iterations=5, epochs=10, batch=128) +if __name__ == "__main__": + main() diff --git a/src/pptdb.c b/src/pptdb.c deleted file mode 100644 index cdc0004..0000000 --- a/src/pptdb.c +++ /dev/null @@ -1,271 +0,0 @@ -/* - pptdb, generate neural network training data from a playtak.com - database dump - - Copyright (C) 2021, tslil clingman - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see <https://www.gnu.org/licenses/>. -*/ - -#include <stdio.h> -#include <stdlib.h> -#include <string.h> - -#include <tak.h> - -int generate; -uint64_t heights[16]; -FILE *training_fh = NULL; -float max_flats; -uint8_t outcome_black; - -tak_state_p state; - -static void write_input(const int dx, const int dy, const uint8_t swap) { - // Two numbers for flats remaining - fprintf(training_fh, "%d,%.8f,%.8f,", state->ply & 1 ? 1 : -1, - (float)(state->white_count & 127) / max_flats, - (float)(state->black_count & 127) / max_flats); - - // Write the board layers - float val; - int col, row; - row = (dy > 0) ? -1 : state->board_size; - for (int i = 0; i < state->board_size; i++) { - row += dy; - col = (dx > 0) ? -1 : state->board_size; - for (int j = 0; j < state->board_size; j++) { - col += dx; - const uint8_t k = (swap) ? THE_COORDS(state->board_size, row, col) - : THE_COORDS(state->board_size, col, row); - val = 0; - if (COUNT_AT(state, k) > 0) { - // Top layer of stacks is handled differently to indicate - // stone type - if (STONE_AT(state, k) == STONE_STANDING) { - val = (state->colours[k] & 1) ? +0.25 : -0.25; - } else if (STONE_AT(state, k) == STONE_CAPSTONE) { - val = (state->colours[k] & 1) ? +1.00 : -1.00; - } else { - val = (state->colours[k] & 1) ? +0.50 : -0.50; - } - } - fprintf(training_fh, "%.2f,", val); - } - } - fprintf(training_fh, "%d,%d\n", outcome_black ? 1 : 0, outcome_black ? 0 : 1); -} - -// Warning: performs _no_ checks on input whatsoever -static enum ACT_RESULT parse_line(const char *pt, const ssize_t read) { - ssize_t idx; - enum ACT_RESULT r; - int total_plies = 0; - - for (idx = 0; idx < read; idx++) { - if (pt[idx] == ',') - total_plies++; - } - for (idx = 0;;) { - if (pt[idx] == 'P') { - // P [A-F][1-6] [CF]?, - idx += 2; - enum STONE_VARIANT stone; - const uint8_t col = pt[idx] - 'A', row = pt[idx + 1] - '1'; - - if (idx + 3 < read) { - switch (pt[idx + 3]) { - case 'W': { - stone = STONE_STANDING; - break; - } - case 'C': { - stone = STONE_CAPSTONE; - break; - } - default: { - stone = STONE_FLAT; - break; - } - } - } else { - stone = STONE_FLAT; - } - - r = try_place(state, THE_COORDS(state->board_size, col, row), - state->current_colour, stone); - if (r != ACT_OK) - return r; - } else if (pt[idx] == 'M') { - // M [A-F][1-6] [A-F][1-6]( [1-6])+, - idx += 2; - uint8_t drops[state->board_size]; - const uint8_t s_col = pt[idx] - 'A', s_row = pt[idx + 1] - '1', - d_col = pt[idx + 3] - 'A', d_row = pt[idx + 4] - '1'; - idx += 4; - - enum MOVE_DIRECTION dir = M_RIGHT; - if (s_col < d_col) - dir = M_RIGHT; - else if (s_col > d_col) - dir = M_LEFT; - else if (s_row < d_row) - dir = M_UP; - else if (s_row > d_row) - dir = M_DOWN; - - uint8_t steps = 0; - do { - idx += 2; - drops[steps++] = pt[idx] - '0'; - } while (idx + 2 < read && pt[idx + 1] != ','); - - r = try_move(state, THE_COORDS(state->board_size, s_col, s_row), dir, - steps, drops); - - if (r != ACT_OK) - return r; - - if (generate == 0) { - // Measure height of stacks exceeding 1 - for (int k = 0; k < state->board_size * state->board_size; k++) { - if (COUNT_AT(state, k) > 1) - heights[COUNT_AT(state, k)] += 1; - } - } - } - // Generate training data, not too early in the game and not at - // the end, under all eight symmetries of the board - if (generate && state->ply > 7 && state->ply < total_plies && - state->ply + 10 >= total_plies) { - write_input(+1, +1, 1); - write_input(+1, +1, 0); - write_input(+1, -1, 1); - write_input(+1, -1, 0); - write_input(-1, +1, 1); - write_input(-1, +1, 0); - write_input(-1, -1, 1); - write_input(-1, -1, 0); - } - // Parse next action - while (idx < read && pt[idx++] != ',') - ; - if (idx >= read) - return ACT_OK; - next_ply(state); - } - return ACT_OK; -} - -const char *license = - "pptdb, generate neural network training data from a playtak.com database dump\n\ -\n\ -Copyright (C) 2021, tslil clingman\n\ -\n\ -This program comes with ABSOLUTELY NO WARRANTY; and is made available under the terms of the GNU GPL v3 license. This is free software, and you are welcome to redistribute it under certain conditions; see COPYING for details.\n"; - -int main(int argc, char **argv) { - (void)(argc); - - enum ACT_RESULT r; - enum WIN_TYPE win; - uint32_t games = 0, overflow = 0, illegal = 0; - uint32_t road_wins = 0, flat_wins = 0, road_turns = 0, flat_turns = 0, - white_wins = 0, black_wins = 0; - - for (int k = 0; k < 16; k++) - heights[k] = 0; - - size_t len = 0; - ssize_t read = 0; - FILE *playtak_fh = NULL; - char *line = NULL, td_fn[65]; - - const uint8_t size = argv[1][0] - '0'; - - playtak_fh = fopen(argv[2], "r"); - if (playtak_fh == NULL) - exit(EXIT_FAILURE); - - if (argc > 3 && (!strncmp("generate", argv[3], 8))) { - generate = 1; - max_flats = (size == 5) ? 21.0 : 30.0; - snprintf(td_fn, 64, "data/parsed-%d.csv", size); - training_fh = fopen(td_fn, "w"); - if (training_fh == NULL) - exit(EXIT_FAILURE); - } else - generate = 0; - - state = new_tak_state(size); - while ((read = getline(&line, &len, playtak_fh)) != -1) { - // Reset everything - reset_state(state, size); - // Store the outcome of this game. Black win = 1 - outcome_black = (line[read - 4] == '0'); - // Parse the line - r = parse_line(line, read - 4); - // Adjust counts if we're not generating training data - if (generate == 0) { - if (r == ACT_ILLEGAL) { - illegal++; - printf("Illegal:\n%s", line); - } else if (r == ACT_OVERFLOW) { - printf("Overflow:\n%s", line); - overflow++; - } else { - win = check_win(state); - if (win == WIN_FLAT_BLACK || win == WIN_FLAT_WHITE || win == WIN_DRAW) { - flat_wins++; - flat_turns += state->ply / 2 + 1; - } else { - road_wins++; - road_turns += state->ply / 2 + 1; - } - if (win == WIN_FLAT_BLACK || win == WIN_ROAD_BLACK) - black_wins++; - else if (win == WIN_FLAT_WHITE || win == WIN_ROAD_WHITE) - white_wins++; - } - } - games++; - } - - fclose(playtak_fh); - if (generate) - fclose(training_fh); - if (line) - free(line); - - if (illegal || overflow) - putchar('\n'); - printf("Read %d games\n", games); - - if (generate == 0) { - printf("Illegals: %d\nOverflows: %d\n\ -Black wins: %.3f%%\n\ -Road wins: %d\nFlat wins: %d\n\ -Average turns to road win: %.3f\n\ -Average turns to flat win: %.3f\n", - illegal, overflow, - (double)black_wins / (double)(black_wins + white_wins) * 100, - road_wins, flat_wins, (double)(road_turns) / (double)(road_wins), - (double)(flat_turns) / (double)(flat_wins)); - for (int k = 2; k < 16; k++) { - printf("Height %2d: %7ld\n", k, heights[k]); - } - } - - exit(EXIT_SUCCESS); -} diff --git a/src/selfplay.c b/src/selfplay.c new file mode 100644 index 0000000..6da4f6e --- /dev/null +++ b/src/selfplay.c @@ -0,0 +1,170 @@ +/* + selfplay: generate training data + + Copyright (C) 2026, tslil clingman + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with ct. If not, see <https://www.gnu.org/licenses/>. +*/ + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <time.h> +#include <unistd.h> + +#include "actions.h" +#include "negamax.h" +#include "nn.h" +#include "tak.h" +#include "xorshift64.h" + +#define MAX_PLY 256 +#define MAX_INP (3 + 6 * 6) + +static tak_state_p state; +static uint32_t min_ply; + +// Positions buffered for the game (the net input only) +static float rows[MAX_PLY][MAX_INP]; +static uint32_t row_ply[MAX_PLY]; +static uint32_t n_rows; +static uint32_t game_id; // PID, tags every row so games can be grouped + +void negamax_display_progress(const uint8_t cur_depth, + const uint8_t init_depth, + const uint32_t length) { + (void)(cur_depth); + (void)(init_depth); + (void)(length); +} +static int play_game(const uint32_t eps10000) { + reset_state(state, state->board_size); + n_rows = 0; + + char ptn[16]; + for (uint32_t ply = 0; ply < MAX_PLY; ply++) { + // The engine's best move for this position (sets negamax_ptn) + negamax_generate(state); + + if (rand32() % 10000 < eps10000) { + // Epsilon: uniform random legal move + uint32_t n_actions; + action_t *actions = actions_generate(state, &n_actions); + action_to_ptn(actions[rand32() % n_actions], ptn); + } else { + // Greedy: the engine's best move + strcpy(ptn, negamax_ptn); + } + + // Buffer this position, before the move changes the board + if (ply >= min_ply && n_rows < MAX_PLY) { + nn_prepare_input(rows[n_rows], state); + row_ply[n_rows] = ply; + n_rows++; + } + + const enum ACT_RESULT r = do_ptn(state, ptn); + if (r == GAME_END) + break; + if (r != ACT_OK) { + fprintf(stderr, "selfplay: unexpected result %d for ptn %s\n", r, + ptn); + return 0; + } + } + + if (n_rows == 0) + return 0; + + const uint8_t n_cells = state->board_size * state->board_size; + + // final outcome from black's perspective + float R; + switch (state->won) { + case WIN_FLAT_BLACK: + case WIN_ROAD_BLACK: + R = 1.0f; + break; + case WIN_FLAT_WHITE: + case WIN_ROAD_WHITE: + R = -1.0f; + break; + default: + R = 0.0f; + break; + } + + for (uint32_t k = 0; k < n_rows; k++) { + fprintf(stdout, "%d,%.8f,%.8f,", (int)rows[k][0], rows[k][1], + rows[k][2]); + for (uint8_t c = 0; c < n_cells; c++) + fprintf(stdout, "%.2f,", rows[k][3 + c]); + fprintf(stdout, "%u,%u,%.0f\n", game_id, row_ply[k], R); + } + + return (int)n_rows; +} + +const char *license = + "selfplay, one game of self-play training data for the ct net\n\ +Usage: selfplay <size> <eps> <depth> <min_ply>\n\ +Plays a single game of <size>x<size>. On each ply the engine's best move is played with probability 1-eps, and a uniform random legal move with probability eps. Every position from ply <min_ply> onwards is written to stdout as one CSV row:\n\ +\n\ + parity,white_flats,black_flats,<board values...>,game_id,ply,R\n\ +\n\ +The first 3 + size*size columns are the training input, game_id identifies the game, ply is the move number the position precedes, and R is the game's final outcome from black's perspective (+1 black won, -1 white won, 0 draw). nn_train.py runs many of these in parallel, pools the rows, and derives TD(lambda) targets from them.\n\ +\n\ +Copyright (C) 2026, tslil clingman\n\ +\n\ +This program is free software: you can redistribute it and/or modify\n \ +it under the terms of the GNU General Public License as published by\n \ +the Free Software Foundation, either version 3 of the License, or\n\ +(at your option) any later version."; + +int main(int argc, char **argv) { + if (argc != 5) { + puts(license); + return EXIT_FAILURE; + } + + const uint8_t size = (uint8_t)atoi(argv[1]); + const uint32_t eps10000 = (uint32_t)(atof(argv[2]) * 10000.0 + 0.5); + negamax_search_depth = (uint8_t)atoi(argv[3]); + min_ply = (uint32_t)atoi(argv[4]); + + if (size != 5 && size != 6) { + fprintf(stderr, "selfplay: size must be 5 or 6\n"); + return EXIT_FAILURE; + } + if (eps10000 >= 10000) { + fprintf(stderr, "selfplay: eps must be < 1\n"); + return EXIT_FAILURE; + } + + // Processes launched in the same second must not share a seed + game_id = (uint32_t)getpid(); + xors = ((uint64_t)time(NULL) ^ ((uint64_t)getpid() << 32)) * + 0x9E3779B97F4A7C15ULL + + 0xC0FFEEEE; + + state = new_tak_state(size); + negamax_init(size); + + play_game(eps10000); + + negamax_free(); + free_tak_state(state); + + return EXIT_SUCCESS; +} |