new neural network arch (faster + better) & minor changes + fixes

Gone is the convolutional neural network, for it turns out not only is
it more difficult to train, but all of the extra information about
board layers didn't make much of a difference at this size.

So cnn1986 has been replaced by nn1986, a standard, two-layer, dense
nn configured as a binary classifier and (mis)used in that capacity.
Note: total number of parameters is unchanged.

HARK: this new nn exposes a bug somewhere in ctak. Run ctlm with
self-play to see the completely borked board state at the end.

1 files changed, 44 insertions, 45 deletions

diff --git a/resources/extract.sh b/resources/extract.sh
index 2c4b153..d475a1d 100755
--- a/resources/extract.sh
+++ b/resources/extract.sh
@@ -2,67 +2,66 @@
 
 db_file=games_anon.db
 
-#chosen_players="AaaarghBot Tiltak_Bot TakticianBot"
-chosen_players=""
+chosen_players="AaaarghBot Tiltak_Bot TakticianBot"
 
 query() {
-		query="(size == $1) and (result != '1-0') and (result != '0-1') and (result != '0-0') and (result != '1/2-1/2')"
-		selct=""
-		for player in $chosen_players; do
-				selct="$selct(player_black == '$player') or (player_white == '$player') or "
-		done;
-		if [ -n "$selct" ]; then
-				query="$query and (${selct:0:-4})"
-		fi
-		echo "SELECT $2 FROM games WHERE $query;"
+    query="(size == $1) and (result != '1-0') and (result != '0-1') and (result != '0-0') and (result != '1/2-1/2')"
+    selct=""
+    for player in $chosen_players; do
+        selct="$selct(player_black = '$player') or (player_white = '$player') or "
+    done;
+    if [ -n "$selct" ]; then
+        query="$query and (${selct:0:-4})"
+    fi
+    echo "SELECT $2 FROM games WHERE $query;"
 }
 
 extract() {
-		size="$1"
-		things="$2"
-		if [ ! -f "data/$db_file" ]; then
-				curl "https://www.playtak.com/games_anon.db" -o "data/$db_file"
-		fi
-		echo Extracing games of size "$size"...
-		sqlite3 "data/$db_file" "$(query $size $things)" | shuf > "data/playtak-$size"
+    size="$1"
+    things="$2"
+    if [ ! -f "data/$db_file" ]; then
+        wget "https://www.playtak.com/games_anon.db" -O "data/$db_file"
+    fi
+    echo Extracing games of size "$size" with query: "$(query $size $things)"
+    sqlite3 "data/$db_file" "$(query $size $things)" | shuf > "data/playtak-$size"
 }
 
-process() {
-		size=$1
-		num_games=$2
-		fn=$3
-
-		./pptdb "$size" "data/playtak-$size" > "data/check-$size"
-		tail -n22 "data/check-$size"
-		echo -ne "\tStripping overflows and illegal games... "
-		grep -Fvxf "data/check-$size" "data/playtak-$size" > "data/good-playtak-$size-all"
+prepare() {
+    size=$1
+    ./pptdb "$size" "data/playtak-$size" > "data/check-$size"
+    tail -n22 "data/check-$size"
+    echo -ne "\tStripping overflows and illegal games... "
+    grep -Fvxf "data/check-$size" "data/playtak-$size" > "data/good-playtak-$size-all"
+    shuf "data/good-playtak-$size-all" > "data/good-playtak-$size"
+    rm "data/good-playtak-$size-all"
 
-		shuf "data/good-playtak-$size-all" > "data/good-playtak-$size"
-		rm "data/good-playtak-$size-all"
+    echo -en "Done.\n\tGenerating training data... "
+    ./pptdb "$size" "data/good-playtak-$size" generate
+}
 
-		echo -en "Done.\n\tGenerating training data... "
-		./pptdb "$size" "data/good-playtak-$size" generate
+process() {
+    size=$1
+    num_t=$2
+    num_v=$3
+    total=$(( num_t + num_v ))
 
-		echo -en "\tChoosing $num_games of $(wc -l data/training-$size.csv | cut -d\  -f1) samples... "
-		shuf -n $num_games "data/training-$size.csv" > "data/shuf-$size.csv"
+    echo -en "\tChoosing $num_t + $num_v = $total of $(wc -l data/parsed-$size.csv | cut -d\  -f1) samples... "
+    shuf -n $total "data/parsed-$size.csv" > "data/shuf-$size.csv"
+    head -n $num_t "data/shuf-$size.csv" > "data/training-$size.csv"
+    tail -n $num_v "data/shuf-$size.csv" > "data/validation-$size.csv"
 
-		echo -en "Done.\n\tCompressing data... "
-		mv "data/shuf-$size.csv" "data/$fn-$size.csv"
-		if [ -f "data/$fn-$size.csv.zst" ]; then
-							rm "data/$fn-$size.csv.zst"
-		fi
-		zstd --rm -13 "data/$fn-$size.csv"
-		echo "Done, available in data/$fn-$size.csv.zst"
+    echo "Done. "
 }
 
 if [ ! -d data ]; then
-	mkdir data
+    mkdir data
 fi
 
 make pptdb
 
-for size in 6; do
-		echo
-		extract $size notation,result
-		process $size 1000000 training
+for size in 5; do
+    echo
+    extract $size notation,result
+    prepare $size
+    process $size 1000000 20000
 done