From 710d5a5225edb44d171ca9ac6345e9a61aa4f125 Mon Sep 17 00:00:00 2001 From: tslil Date: Thu, 10 Sep 2026 12:31:51 +0100 Subject: use silu instead of relu with the intuition being that we avoid dead neurons In a head-to-head this is +187 elo against the previous weights, although it trained more. --- src/nn_train.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/nn_train.py b/src/nn_train.py index 8f1a73e..9e2b482 100755 --- a/src/nn_train.py +++ b/src/nn_train.py @@ -49,6 +49,7 @@ import torch from tqdm import tqdm from torch.nn import Embedding, MSELoss, Linear, Module +from torch.nn.functional import silu, tanh from torch import Tensor @@ -211,8 +212,8 @@ class Net(Module): # inputs, the rest are raw cell states (0..6) n = x.shape[0] inp = torch.cat([x[:, :3], self.embedding(x[:, 3:].long()).view(n, -1)], dim=1) - hidden = torch.relu(self.hidden(inp)) - return torch.tanh(self.output(hidden)).squeeze(-1) + hidden = silu(self.hidden(inp)) + return tanh(self.output(hidden)).squeeze(-1) def fit( self, -- cgit v1.3.1