From 6c1154d80e7f7a04444d01d97942ac9e8138297a Mon Sep 17 00:00:00 2001
From: tslil clingman <tslil@posteo.de>
Date: Fri, 29 Jan 2021 13:06:49 -0500
Subject: Cleaned up build system

---
 resources/extract.sh | 69 ++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 69 insertions(+)
 create mode 100755 resources/extract.sh

(limited to 'resources/extract.sh')

diff --git a/resources/extract.sh b/resources/extract.sh
new file mode 100755
index 0000000..fb6f267
--- /dev/null
+++ b/resources/extract.sh
@@ -0,0 +1,69 @@
+#!/bin/bash
+
+db_file=games_anon.db
+
+chosen_players="fwwwwibib archvenison NohatCoder nqeron ManaT
+SultanPepper orfane rabbitboy84 applemonkeyman Reid Tayacan Breeze
+AaaarghBot"
+
+query() {
+		query="(size == $1) and (result != '1-0') and (result != '0-1') and (result != '0-0') and (result != '1/2-1/2')"
+		selct=""
+		for player in $chosen_players; do
+				selct="$selct(player_black == '$player') or (player_white == '$player') or "
+		done;
+		if [ -n "$selct" ]; then
+				query="$query and (${selct:0:-4})"
+		fi
+		echo "SELECT $2 FROM games WHERE $query;"
+}
+
+extract() {
+		size="$1"
+		things="$2"
+		if [ ! -f "data/$db_file" ]; then
+				curl "https://www.playtak.com/games_anon.db" -o "data/$db_file"
+		fi
+		echo Extracing games of size "$size"...
+		sqlite3 "data/$db_file" "$(query $size $things)" | shuf > "data/playtak-$size"
+}
+
+process() {
+		size=$1
+		num_games=$2
+		fn=$3
+
+		./pptdb "$size" "data/playtak-$size" > "data/check-$size"
+		tail -n22 "data/check-$size"
+		echo -ne "\tStripping overflows and illegal games... "
+		grep -Fvxf "data/check-$size" "data/playtak-$size" > "data/good-playtak-$size"
+
+		echo -en "Done.\n\tGenerating training data... "
+		./pptdb "$size" "data/good-playtak-$size" generate
+
+		echo -en "\tChoosing $num_games of $(wc -l data/training-$size.csv | cut -d\  -f1) samples... "
+		head -n 1 "data/training-$size.csv" > "data/shuf-$size.csv"
+		tail -n+2 "data/training-$size.csv" > "data/tmp"
+		shuf -n $num_games "data/tmp" >> "data/shuf-$size.csv"
+		rm data/tmp
+
+		echo -en "Done.\n\tCompressing data... "
+		mv "data/shuf-$size.csv" "data/$fn-$size.csv"
+		if [ -f "data/$fn-$size.csv.gz" ]; then
+							rm "data/$fn-$size.csv.gz"
+		fi
+		gzip "data/$fn-$size.csv"
+		echo "Done, available in data/$fn-$size.csv.gz"
+}
+
+if [ ! -d data ]; then
+	mkdir data
+fi
+
+make pptdb
+
+for size in 5; do
+		echo
+		extract $size notation,result
+		process $size 400000 training
+done
-- 
cgit v1.3.1