diff options
Diffstat (limited to 'include/nn.c')
| -rw-r--r-- | include/nn.c | 14 |
1 files changed, 10 insertions, 4 deletions
diff --git a/include/nn.c b/include/nn.c index f228297..8c85299 100644 --- a/include/nn.c +++ b/include/nn.c @@ -26,8 +26,6 @@ // Helpers // =================================================================== -#define RELU(x) ((x) = ((x) < 0) ? 0 : (x)) - static float fast_tanh(const float x) { // tanh via a degree-11 odd polynomial, least-squares fit of tanh on [-3, 3] // (max absolute error ~4e-3), saturating to +-1 outside that interval. @@ -41,6 +39,15 @@ static float fast_tanh(const float x) { (-0.0139947924f + x2 * (0.00123106314f + x2 * -4.29748644e-05f))))); } + +static float fast_silu(const float x) { + // [4,2] Pade approximant with cutoff + if (x < -4.6f) + return 0.0; + if (x > 4.6f) + return x; + return (x * (0.5f + x * (0.25f + x * (0.05f + 0.004167f * x)))) / (1+0.1f*x); +} // =================================================================== // Input encoding // =================================================================== @@ -116,8 +123,7 @@ float eval_net(const float *board, const unsigned int n_inp, const float *wd = w + d1 * n_inp; for (unsigned int fl = 0; fl < n_inp; fl++) h += board[fl] * wd[fl]; - RELU(h); - dense1[d1] = h; + dense1[d1] = fast_silu(h); } for (unsigned int d1 = 0; d1 < n_dense; d1++) z += dense1[d1] * logit_w[d1]; |