diff options
Diffstat (limited to 'extract.sh')
| -rwxr-xr-x | extract.sh | 61 |
1 files changed, 30 insertions, 31 deletions
@@ -3,48 +3,47 @@ db_file=games_anon.db query() { - echo "SELECT $1 FROM games WHERE (size == $2) \ -and ((result == 'R-0') or (result == '0-R'));" -# and (result != '1-0') and (result != '0-1') and (result != '0-0')\ -# and (result != '1/2-1/2');" + query="(size == $1) and (result != '1-0') and (result != '0-1') and (result != '0-0')" + selct="" + # for player in rabbitboy84 fwwwwibib archvenison Simmon AaaarghBot; do + for player in; do + selct="$selct(player_black == '$player') or (player_white == '$player') or " + done; + if [ -n "$selct" ]; then + query="$query and (${selct:0:-4})" + fi + echo "SELECT $2 FROM games WHERE $query;" } extract() { + size="$1" + things="$2" if [ ! -f "data/$db_file" ]; then wget "https://www.playtak.com/games_anon.db" -O "data/$db_file" fi - for size in 5; do - echo Extracing games of size "$size"... - sqlite3 "data/$db_file" "$(query $1 $size)" | shuf > "data/playtak-$size" - done + echo Extracing games of size "$size"... + sqlite3 "data/$db_file" "$(query $size $things)" | shuf > "data/playtak-$size" } process() { - num_games=$1 - echo -e "Beginning to process data\n" - for i in 5; do - echo "Size $i..." - ./pptdb "$i" "data/playtak-$i" > "data/check-$i" - tail -n21 "data/check-$i" - echo -e "\tStripping overflows and illegal games..." - grep -Fvxf "data/check-$i" "data/playtak-$i" > "data/good-playtak-$i" - echo -e "\tChoosing $num_games from what remains ..." - shuf -n $num_games "data/good-playtak-$i" > "data/smalltak-$i" - echo -en "\tGenerating training data... " - ./pptdb "$i" "data/smalltak-$i" generate - echo -e "\tWrote $(wc -l data/training-$i.csv | cut -d\ -f1) samples. Shuffling these..." - tail -n+2 "data/training-$i.csv" | shuf > "data/shuf-$i.csv" - head -n 1 "data/training-$i.csv" | cat "data/shuf-$i.csv" > "data/smalltrain-$i.csv" - rm "data/shuf-$i.csv" - mv "data/smalltrain-5.csv" "data/training-5.csv " - echo -e "\n\tDone! Sample training data in data/training-$i.csv" - done + size=$1 + num_games=$2 + ./pptdb "$size" "data/playtak-$size" > "data/check-$size" + tail -n22 "data/check-$size" + echo -e "\tStripping overflows and illegal games..." + grep -Fvxf "data/check-$size" "data/playtak-$size" > "data/good-playtak-$size" + echo -e "\tChoosing $num_games from what remains ..." + shuf -n $num_games "data/good-playtak-$size" > "data/smalltak-$size" + echo -en "\tGenerating training data... " + ./pptdb "$size" "data/smalltak-$size" generate + echo -e "\n\tDone! Wrote $(wc -l data/training-$size.csv | cut -d\ -f1) samples to data/training-$size.csv." } -if [ ! -f "data/playtak-5" ]; then - extract notation,result -fi make pptdb -process 20000 +for size in 5 6; do + extract $size notation,result + process $size 20000 + echo +done |