#!/bin/bash db_file=games_anon.db chosen_players="AaaarghBot Tiltak_Bot TakticianBot" query() { query="(size == $1) and (result != '1-0') and (result != '0-1') and (result != '0-0') and (result != '1/2-1/2')" selct="" for player in $chosen_players; do selct="$selct(player_black = '$player') or (player_white = '$player') or " done; if [ -n "$selct" ]; then query="$query and (${selct:0:-4})" fi echo "SELECT $2 FROM games WHERE $query;" } extract() { size="$1" things="$2" if [ ! -f "data/$db_file" ]; then wget "https://www.playtak.com/games_anon.db" -O "data/$db_file" fi echo Extracing games of size "$size" with query: "$(query $size $things)" sqlite3 "data/$db_file" "$(query $size $things)" | shuf > "data/playtak-$size" } prepare() { size=$1 ./pptdb "$size" "data/playtak-$size" > "data/check-$size" tail -n22 "data/check-$size" echo -ne "\tStripping overflows and illegal games... " grep -Fvxf "data/check-$size" "data/playtak-$size" > "data/good-playtak-$size-all" shuf "data/good-playtak-$size-all" > "data/good-playtak-$size" rm "data/good-playtak-$size-all" echo -en "Done.\n\tGenerating training data... " ./pptdb "$size" "data/good-playtak-$size" generate } process() { size=$1 num_t=$2 num_v=$3 total=$(( num_t + num_v )) echo -en "\tChoosing $num_t + $num_v = $total of $(wc -l data/parsed-$size.csv | cut -d\ -f1) samples... " shuf -n $total "data/parsed-$size.csv" > "data/shuf-$size.csv" head -n $num_t "data/shuf-$size.csv" > "data/training-$size.csv" tail -n $num_v "data/shuf-$size.csv" > "data/validation-$size.csv" echo "Done. " } if [ ! -d data ]; then mkdir data fi make pptdb for size in 5; do echo extract $size notation,result prepare $size process $size 1000000 20000 done