diff options
Diffstat (limited to 'extract.sh')
| -rwxr-xr-x | extract.sh | 43 |
1 files changed, 22 insertions, 21 deletions
@@ -5,8 +5,10 @@ db_file=games_anon.db query() { query="(size == $1) and (result != '1-0') and (result != '0-1') and (result != '0-0')" selct="" - # for player in rabbitboy84 fwwwwibib archvenison Simmon AaaarghBot; do - for player in; do + # for player in fwwwwibib archvenison NohatCoder nqeron ManaT rabbitboy84 AaaarghBot; do + for player in NohatCoder AaaarghBot; do + # for player in; do + # selct="$selct(player_black == '$player') or " selct="$selct(player_black == '$player') or (player_white == '$player') or " done; if [ -n "$selct" ]; then @@ -28,36 +30,35 @@ extract() { process() { size=$1 num_games=$2 + fn=$3 + ./pptdb "$size" "data/playtak-$size" > "data/check-$size" tail -n22 "data/check-$size" echo -ne "\tStripping overflows and illegal games... " grep -Fvxf "data/check-$size" "data/playtak-$size" > "data/good-playtak-$size" - echo -en "Done.\n\tChoosing $num_games from what remains... " - shuf -n $num_games "data/good-playtak-$size" > "data/smalltak-$size" + echo -en "Done.\n\tGenerating training data... " - ./pptdb "$size" "data/smalltak-$size" generate - echo -en "\tShuffling $(wc -l data/training-$size.csv | cut -d\ -f1) samples... " - # head -n 1 "data/training-$size.csv" > "data/shuf-$size.csv" - # tail -n+2 "data/training-$size.csv" | shuf >> "data/shuf-$size.csv" - # mv "data/shuf-$size.csv" "data/training-$size.csv" + ./pptdb "$size" "data/good-playtak-$size" generate + + echo -en "\tChoosing $num_games of $(wc -l data/training-$size.csv | cut -d\ -f1) samples... " + head -n 1 "data/training-$size.csv" > "data/shuf-$size.csv" + tail -n+2 "data/training-$size.csv" > "data/tmp" + shuf -n $num_games "data/tmp" >> "data/shuf-$size.csv" + rm data/tmp + echo -en "Done.\n\tCompressing data... " - # if [ -f "data/training-$size.csv.gz" ]; then - # rm "data/training-$size.csv.gz" - # fi - # gzip "data/training-$size.csv" - # echo "Done, available in data/training-$size.csv.gz" - if [ -f "data/validation-$size.csv.gz" ]; then - rm "data/validation-$size.csv.gz" + mv "data/shuf-$size.csv" "data/$fn-$size.csv" + if [ -f "data/$fn-$size.csv.gz" ]; then + rm "data/$fn-$size.csv.gz" fi - mv "data/training-$size.csv" "data/validation-$size.csv" - gzip "data/validation-$size.csv" - echo "Done, available in data/validation-$size.csv.gz" + gzip "data/$fn-$size.csv" + echo "Done, available in data/$fn-$size.csv.gz" } make pptdb -for size in 5 6; do +for size in 5; do echo extract $size notation,result - process $size 50000 + process $size 100000 training done |