diff options
Diffstat (limited to 'resources')
| -rwxr-xr-x | resources/extract.sh | 89 |
1 files changed, 44 insertions, 45 deletions
diff --git a/resources/extract.sh b/resources/extract.sh index 2c4b153..d475a1d 100755 --- a/resources/extract.sh +++ b/resources/extract.sh @@ -2,67 +2,66 @@ db_file=games_anon.db -#chosen_players="AaaarghBot Tiltak_Bot TakticianBot" -chosen_players="" +chosen_players="AaaarghBot Tiltak_Bot TakticianBot" query() { - query="(size == $1) and (result != '1-0') and (result != '0-1') and (result != '0-0') and (result != '1/2-1/2')" - selct="" - for player in $chosen_players; do - selct="$selct(player_black == '$player') or (player_white == '$player') or " - done; - if [ -n "$selct" ]; then - query="$query and (${selct:0:-4})" - fi - echo "SELECT $2 FROM games WHERE $query;" + query="(size == $1) and (result != '1-0') and (result != '0-1') and (result != '0-0') and (result != '1/2-1/2')" + selct="" + for player in $chosen_players; do + selct="$selct(player_black = '$player') or (player_white = '$player') or " + done; + if [ -n "$selct" ]; then + query="$query and (${selct:0:-4})" + fi + echo "SELECT $2 FROM games WHERE $query;" } extract() { - size="$1" - things="$2" - if [ ! -f "data/$db_file" ]; then - curl "https://www.playtak.com/games_anon.db" -o "data/$db_file" - fi - echo Extracing games of size "$size"... - sqlite3 "data/$db_file" "$(query $size $things)" | shuf > "data/playtak-$size" + size="$1" + things="$2" + if [ ! -f "data/$db_file" ]; then + wget "https://www.playtak.com/games_anon.db" -O "data/$db_file" + fi + echo Extracing games of size "$size" with query: "$(query $size $things)" + sqlite3 "data/$db_file" "$(query $size $things)" | shuf > "data/playtak-$size" } -process() { - size=$1 - num_games=$2 - fn=$3 - - ./pptdb "$size" "data/playtak-$size" > "data/check-$size" - tail -n22 "data/check-$size" - echo -ne "\tStripping overflows and illegal games... " - grep -Fvxf "data/check-$size" "data/playtak-$size" > "data/good-playtak-$size-all" +prepare() { + size=$1 + ./pptdb "$size" "data/playtak-$size" > "data/check-$size" + tail -n22 "data/check-$size" + echo -ne "\tStripping overflows and illegal games... " + grep -Fvxf "data/check-$size" "data/playtak-$size" > "data/good-playtak-$size-all" + shuf "data/good-playtak-$size-all" > "data/good-playtak-$size" + rm "data/good-playtak-$size-all" - shuf "data/good-playtak-$size-all" > "data/good-playtak-$size" - rm "data/good-playtak-$size-all" + echo -en "Done.\n\tGenerating training data... " + ./pptdb "$size" "data/good-playtak-$size" generate +} - echo -en "Done.\n\tGenerating training data... " - ./pptdb "$size" "data/good-playtak-$size" generate +process() { + size=$1 + num_t=$2 + num_v=$3 + total=$(( num_t + num_v )) - echo -en "\tChoosing $num_games of $(wc -l data/training-$size.csv | cut -d\ -f1) samples... " - shuf -n $num_games "data/training-$size.csv" > "data/shuf-$size.csv" + echo -en "\tChoosing $num_t + $num_v = $total of $(wc -l data/parsed-$size.csv | cut -d\ -f1) samples... " + shuf -n $total "data/parsed-$size.csv" > "data/shuf-$size.csv" + head -n $num_t "data/shuf-$size.csv" > "data/training-$size.csv" + tail -n $num_v "data/shuf-$size.csv" > "data/validation-$size.csv" - echo -en "Done.\n\tCompressing data... " - mv "data/shuf-$size.csv" "data/$fn-$size.csv" - if [ -f "data/$fn-$size.csv.zst" ]; then - rm "data/$fn-$size.csv.zst" - fi - zstd --rm -13 "data/$fn-$size.csv" - echo "Done, available in data/$fn-$size.csv.zst" + echo "Done. " } if [ ! -d data ]; then - mkdir data + mkdir data fi make pptdb -for size in 6; do - echo - extract $size notation,result - process $size 1000000 training +for size in 5; do + echo + extract $size notation,result + prepare $size + process $size 1000000 20000 done |