From ee216c008a188a9436fedb85c70ee5d1719733b1 Mon Sep 17 00:00:00 2001
From: tslil clingman <tslil@posteo.de>
Date: Sun, 15 Jan 2023 16:03:37 +0100
Subject: new neural network arch (faster + better) & minor changes + fixes

Gone is the convolutional neural network, for it turns out not only is
it more difficult to train, but all of the extra information about
board layers didn't make much of a difference at this size.

So cnn1986 has been replaced by nn1986, a standard, two-layer, dense
nn configured as a binary classifier and (mis)used in that capacity.
Note: total number of parameters is unchanged.

HARK: this new nn exposes a bug somewhere in ctak. Run ctlm with
self-play to see the completely borked board state at the end.
---
 resources/extract.sh | 89 ++++++++++++++++++++++++++--------------------------
 1 file changed, 44 insertions(+), 45 deletions(-)

(limited to 'resources')

diff --git a/resources/extract.sh b/resources/extract.sh
index 2c4b153..d475a1d 100755
--- a/resources/extract.sh
+++ b/resources/extract.sh
@@ -2,67 +2,66 @@
 
 db_file=games_anon.db
 
-#chosen_players="AaaarghBot Tiltak_Bot TakticianBot"
-chosen_players=""
+chosen_players="AaaarghBot Tiltak_Bot TakticianBot"
 
 query() {
-		query="(size == $1) and (result != '1-0') and (result != '0-1') and (result != '0-0') and (result != '1/2-1/2')"
-		selct=""
-		for player in $chosen_players; do
-				selct="$selct(player_black == '$player') or (player_white == '$player') or "
-		done;
-		if [ -n "$selct" ]; then
-				query="$query and (${selct:0:-4})"
-		fi
-		echo "SELECT $2 FROM games WHERE $query;"
+    query="(size == $1) and (result != '1-0') and (result != '0-1') and (result != '0-0') and (result != '1/2-1/2')"
+    selct=""
+    for player in $chosen_players; do
+        selct="$selct(player_black = '$player') or (player_white = '$player') or "
+    done;
+    if [ -n "$selct" ]; then
+        query="$query and (${selct:0:-4})"
+    fi
+    echo "SELECT $2 FROM games WHERE $query;"
 }
 
 extract() {
-		size="$1"
-		things="$2"
-		if [ ! -f "data/$db_file" ]; then
-				curl "https://www.playtak.com/games_anon.db" -o "data/$db_file"
-		fi
-		echo Extracing games of size "$size"...
-		sqlite3 "data/$db_file" "$(query $size $things)" | shuf > "data/playtak-$size"
+    size="$1"
+    things="$2"
+    if [ ! -f "data/$db_file" ]; then
+        wget "https://www.playtak.com/games_anon.db" -O "data/$db_file"
+    fi
+    echo Extracing games of size "$size" with query: "$(query $size $things)"
+    sqlite3 "data/$db_file" "$(query $size $things)" | shuf > "data/playtak-$size"
 }
 
-process() {
-		size=$1
-		num_games=$2
-		fn=$3
-
-		./pptdb "$size" "data/playtak-$size" > "data/check-$size"
-		tail -n22 "data/check-$size"
-		echo -ne "\tStripping overflows and illegal games... "
-		grep -Fvxf "data/check-$size" "data/playtak-$size" > "data/good-playtak-$size-all"
+prepare() {
+    size=$1
+    ./pptdb "$size" "data/playtak-$size" > "data/check-$size"
+    tail -n22 "data/check-$size"
+    echo -ne "\tStripping overflows and illegal games... "
+    grep -Fvxf "data/check-$size" "data/playtak-$size" > "data/good-playtak-$size-all"
+    shuf "data/good-playtak-$size-all" > "data/good-playtak-$size"
+    rm "data/good-playtak-$size-all"
 
-		shuf "data/good-playtak-$size-all" > "data/good-playtak-$size"
-		rm "data/good-playtak-$size-all"
+    echo -en "Done.\n\tGenerating training data... "
+    ./pptdb "$size" "data/good-playtak-$size" generate
+}
 
-		echo -en "Done.\n\tGenerating training data... "
-		./pptdb "$size" "data/good-playtak-$size" generate
+process() {
+    size=$1
+    num_t=$2
+    num_v=$3
+    total=$(( num_t + num_v ))
 
-		echo -en "\tChoosing $num_games of $(wc -l data/training-$size.csv | cut -d\  -f1) samples... "
-		shuf -n $num_games "data/training-$size.csv" > "data/shuf-$size.csv"
+    echo -en "\tChoosing $num_t + $num_v = $total of $(wc -l data/parsed-$size.csv | cut -d\  -f1) samples... "
+    shuf -n $total "data/parsed-$size.csv" > "data/shuf-$size.csv"
+    head -n $num_t "data/shuf-$size.csv" > "data/training-$size.csv"
+    tail -n $num_v "data/shuf-$size.csv" > "data/validation-$size.csv"
 
-		echo -en "Done.\n\tCompressing data... "
-		mv "data/shuf-$size.csv" "data/$fn-$size.csv"
-		if [ -f "data/$fn-$size.csv.zst" ]; then
-							rm "data/$fn-$size.csv.zst"
-		fi
-		zstd --rm -13 "data/$fn-$size.csv"
-		echo "Done, available in data/$fn-$size.csv.zst"
+    echo "Done. "
 }
 
 if [ ! -d data ]; then
-	mkdir data
+    mkdir data
 fi
 
 make pptdb
 
-for size in 6; do
-		echo
-		extract $size notation,result
-		process $size 1000000 training
+for size in 5; do
+    echo
+    extract $size notation,result
+    prepare $size
+    process $size 1000000 20000
 done
-- 
cgit v1.3.1