diff options
Diffstat (limited to 'extract.sh')
| -rwxr-xr-x | extract.sh | 50 |
1 files changed, 26 insertions, 24 deletions
@@ -10,38 +10,40 @@ and (result != '1/2-1/2');" extract() { cd data + if [ ! -f "$db_file" ]; then + wget "https://www.playtak.com/games_anon.db" + fi for size in 5 6; do echo Extracing games of size "$size"... sqlite3 "$db_file" "$(query $1 $size)" | shuf > "playtak-$size" done } -if [ ! -f "$db_file" ]; then - wget "https://www.playtak.com/games_anon.db" -fi - -extract notation,result +process() { + echo -e "Beginning to process data\n" + for i in 5 6; do + echo "Size $i..." + ./pptdb "$i" "data/playtak-$i" > "data/check-$i" + tail -n21 "data/check-$i" + echo Stripping overflows and illegal games... + grep -Fvxf "data/check-$i" "data/playtak-$i" > "data/good-playtak-$i" + echo -n "Generating training data... " + ./pptdb "$i" "data/good-playtak-$i" generate + echo "Shuffling data..." + shuf "data/training-$i.csv" > "data/shuf-$i.csv" + mv "data/shuf-$i.csv" "data/training-$i.csv" + echo + done +} -echo Preparing pptdb +# extract notation,result if [ ! -f "pptdb" ]; then - make pptdb + echo Preparing pptdb + make pptdb fi -echo -e "Beginning to process data\n" - -for i in 5 6; do - echo "Size $i..." - ./pptdb "$i" "data/playtak-$i" > "data/check-$i" - tail -n21 "data/check-$i" - echo Stripping overflows and illegal games... - grep -Fvxf "data/check-$i" "data/playtak-$i" > "data/good-playtak-$i" - echo -n "Generating training data... " - ./pptdb "$i" "data/good-playtak-$i" generate - echo "Shuffling data..." - shuf "data/training-$i.csv" > "data/shuf-$i.csv" - mv "data/shuf-$i.csv" "data/training-$i.csv" - echo -done +# process -shuf -n50000 data/good-playtak-5 > data/smalltak-5 && ./pptdb 5 data/smalltak-5 generate && \ -head -n1 data/training-5.csv > t && tail -n+2 data/training-5.csv > test.csv && shuf test.csv >> t && mv t test.csv +make pptdb +shuf -n10000 data/good-playtak-5 > data/smalltak-5 && ./pptdb 5 data/smalltak-5 generate && \ +head -n1 data/training-5.csv > t && tail -n+2 data/training-5.csv > data/train5.csv && shuf data/train5.csv >> t && mv t data/train5.csv |