TD-leaf and another negamax bug

TD-lambda and TD-leaf are separated by the observation that minimax with a parameterised leaf heuristic is differentiable* in the parameters. "All" that had to be done was to extract the "principal variation": which one position survived all of the pruning to become the reported value.

This was a bit fiddly and uncovered a subtle bug (again) in negamax.c: we were not being complete in our overriding of the TT entry and by omitting key it was possible that later searches would find incorrect values.

The more interesting question to me was: how do we introduce enough variety while not corrupting the PV chain? I settled on having selfplay have one of the players be epsilon-random but the other greedy (minimax) and simply report only the greedy portion PV chain. This appears to work well enough in practice and doesn't introduce nonsense training data for the net (we don't force its actions to be nonsensical). I also deciced, perhaps in a departure from tradition, that we had good supervised labels for PVs that happened to be winning states, so the training overrides the targets in that case. Finally rolled back some of the ambition in nn_train.py default arguments.

*almost always

1 files changed, 18 insertions, 63 deletions

diff --git a/include/nn.h b/include/nn.h
index 18e222a..9e576c1 100644
--- a/include/nn.h
+++ b/include/nn.h
@@ -18,79 +18,34 @@
 #ifndef NN_H
 #define NN_H
 
-#include <tak.h>
-#include "weights_5.h"
-#include "weights_6.h"
-
-// ===================================================================
-// Helpers
-// ===================================================================
+#include <stdint.h>
 
-void nn_prepare_input(float *cur_board, tak_state_p state);
+#include <tak.h>
 
-#define RELU(x) ((x) = ((x) < 0) ? 0 : (x))
-
-/*
- * tanh via a degree-11 odd polynomial, least-squares fit of tanh on [-3, 3]
- * (max absolute error ~4e-3), saturating to +-1 outside that interval.
- */
-static inline float fast_tanh(const float x) {
-  if (x > 3.0f)
-    return 1.0f;
-  if (x < -3.0f)
-    return -1.0f;
-  const float x2 = x * x;
-  return x * (0.997136032f + x2 * (-0.305215124f + x2 *
-                                   (0.0832466371f + x2 *
-                                    (-0.0139947924f + x2 *
-                                     (0.00123106314f + x2 * -4.29748644e-05f)))));
-}
+// Number of cell states: empty, then {black, white} x {flat, standing, road}
+#define NN_N_EMB 7
 
 // ===================================================================
-// Implementation of small neural networks
+// Input encoding
 // ===================================================================
 
-static inline float eval_net(const float *board, const unsigned int n_inp,
-                             const float *w, const float *biases,
-                             const float *logit_w, const float logit_b,
-                             const unsigned int n_dense) {
-  float dense1[FIVE_DENSE_NUM]; // FIVE_DENSE_NUM == SIX_DENSE_NUM
-  float z = logit_b;
-
-  for (unsigned int d1 = 0; d1 < n_dense; d1++) {
-    float h = biases[d1];
-    const float *wd = w + d1 * n_inp;
-    for (unsigned int fl = 0; fl < n_inp; fl++)
-      h += board[fl] * wd[fl];
-    RELU(h);
-    dense1[d1] = h;
-  }
-  for (unsigned int d1 = 0; d1 < n_dense; d1++)
-    z += dense1[d1] * logit_w[d1];
-
-  return fast_tanh(z);
-}
+// somewhat internal, important for generating training data
+void nn_cell_idxs(uint8_t *out, tak_state_p state);
+void nn_prepare_head(float out_head[3], tak_state_p state);
 
+void nn_prepare_input(float *cur_board, tak_state_p state,
+                      const float embedding[NN_N_EMB]);
 // ===================================================================
-// Interfaces to the two built-in evaluators
+// Network
 // ===================================================================
 
-static inline float five_nn_fn(tak_state_p state) {
-  static float cur_board[FIVE_INP_NUM] NN_ALIGNED;
-
-  nn_prepare_input(cur_board, state);
-  return eval_net(cur_board, FIVE_INP_NUM, &five_dense_weights[0][0],
-                  five_dense_biases, five_logit_weights, five_logit_bias,
-                  FIVE_DENSE_NUM);
-}
-
-static inline float six_nn_fn(tak_state_p state) {
-  static float cur_board[SIX_INP_NUM] NN_ALIGNED;
+float eval_net(const float *board, const unsigned int n_inp,
+               const float *w, const float *biases,
+               const float *logit_w, const float logit_b,
+               const unsigned int n_dense);
 
-  nn_prepare_input(cur_board, state);
-  return eval_net(cur_board, SIX_INP_NUM, &six_dense_weights[0][0],
-                  six_dense_biases, six_logit_weights, six_logit_bias,
-                  SIX_DENSE_NUM);
-}
+// Evaluators for the two built-in networks.
+float five_nn_fn(tak_state_p state);
+float six_nn_fn(tak_state_p state);
 
 #endif