use silu instead of relu with the intuition being that we avoid dead neurons

In a head-to-head this is +187 elo against the previous weights, although it trained more.

1 files changed, 10 insertions, 4 deletions

diff --git a/include/nn.c b/include/nn.c
index f228297..8c85299 100644
--- a/include/nn.c
+++ b/include/nn.c
@@ -26,8 +26,6 @@
 // Helpers
 // ===================================================================
 
-#define RELU(x) ((x) = ((x) < 0) ? 0 : (x))
-
 static float fast_tanh(const float x) {
   // tanh via a degree-11 odd polynomial, least-squares fit of tanh on [-3, 3]
   //  (max absolute error ~4e-3), saturating to +-1 outside that interval.
@@ -41,6 +39,15 @@ static float fast_tanh(const float x) {
                                     (-0.0139947924f + x2 *
                                      (0.00123106314f + x2 * -4.29748644e-05f)))));
 }
+
+static float fast_silu(const float x) {
+  // [4,2] Pade approximant with cutoff
+  if (x < -4.6f)
+    return 0.0;
+  if (x > 4.6f)
+    return x;
+  return (x * (0.5f + x * (0.25f + x * (0.05f + 0.004167f * x)))) / (1+0.1f*x);
+}
 // ===================================================================
 // Input encoding
 // ===================================================================
@@ -116,8 +123,7 @@ float eval_net(const float *board, const unsigned int n_inp,
     const float *wd = w + d1 * n_inp;
     for (unsigned int fl = 0; fl < n_inp; fl++)
       h += board[fl] * wd[fl];
-    RELU(h);
-    dense1[d1] = h;
+    dense1[d1] = fast_silu(h);
   }
   for (unsigned int d1 = 0; d1 < n_dense; d1++)
     z += dense1[d1] * logit_w[d1];