blob: a359c61b38ddf58481dacd0575ec2d1006413508 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
|
#!/bin/bash
db_file=games_anon.db
query() {
query="(size == $1) and (result != '1-0') and (result != '0-1') and (result != '0-0')"
selct=""
# for player in rabbitboy84 fwwwwibib archvenison Simmon AaaarghBot; do
for player in; do
selct="$selct(player_black == '$player') or (player_white == '$player') or "
done;
if [ -n "$selct" ]; then
query="$query and (${selct:0:-4})"
fi
echo "SELECT $2 FROM games WHERE $query;"
}
extract() {
size="$1"
things="$2"
if [ ! -f "data/$db_file" ]; then
wget "https://www.playtak.com/games_anon.db" -O "data/$db_file"
fi
echo Extracing games of size "$size"...
sqlite3 "data/$db_file" "$(query $size $things)" | shuf > "data/playtak-$size"
}
process() {
size=$1
num_games=$2
./pptdb "$size" "data/playtak-$size" > "data/check-$size"
tail -n22 "data/check-$size"
echo -ne "\tStripping overflows and illegal games... "
grep -Fvxf "data/check-$size" "data/playtak-$size" > "data/good-playtak-$size"
echo -en "Done.\n\tChoosing $num_games from what remains... "
shuf -n $num_games "data/good-playtak-$size" > "data/smalltak-$size"
echo -en "Done.\n\tGenerating training data... "
./pptdb "$size" "data/smalltak-$size" generate
echo -en "\tShuffling $(wc -l data/training-$size.csv | cut -d\ -f1) samples... "
# head -n 1 "data/training-$size.csv" > "data/shuf-$size.csv"
# tail -n+2 "data/training-$size.csv" | shuf >> "data/shuf-$size.csv"
# mv "data/shuf-$size.csv" "data/training-$size.csv"
echo -en "Done.\n\tCompressing data... "
# if [ -f "data/training-$size.csv.gz" ]; then
# rm "data/training-$size.csv.gz"
# fi
# gzip "data/training-$size.csv"
# echo "Done, available in data/training-$size.csv.gz"
if [ -f "data/validation-$size.csv.gz" ]; then
rm "data/validation-$size.csv.gz"
fi
mv "data/training-$size.csv" "data/validation-$size.csv"
gzip "data/validation-$size.csv"
echo "Done, available in data/validation-$size.csv.gz"
}
make pptdb
for size in 5 6; do
echo
extract $size notation,result
process $size 50000
done
|
