diff options
| -rwxr-xr-x | data/extract.sh | 98 | ||||
| -rw-r--r-- | src/pptdb.c | 69 |
2 files changed, 62 insertions, 105 deletions
diff --git a/data/extract.sh b/data/extract.sh index b586429..285be26 100755 --- a/data/extract.sh +++ b/data/extract.sh @@ -10,88 +10,34 @@ and (result != '1/2-1/2');" extract() { for size in 5 6; do + echo Extracing games of size "$size"... sqlite3 "$db_file" "$(query $1 $size)" > "playtak-$size" done } -# Correct move notation, assuming only valid stuff -fix_move() { - args=($1) - if [ "${args[0]}" = "M" ]; then - result="" - # Annoying to compute direction - base_col=${args[1]:0:1} - base_row=${args[1]:1:1} - step_col=${args[2]:0:1} - step_row=${args[2]:1:1} - if [[ "$base_col" < "$step_col" ]]; then - result=">" - elif [[ "$base_col" > "$step_col" ]]; then - result="<" - elif [[ "$base_row" > "$step_row" ]]; then - result="-" - elif [[ "$base_row" < "$step_row" ]]; then - result="+" - fi - total=0 - steps=${#args[@]} - for (( i = 3 ; i < steps ; i++ )); do - total=$(( total + ${args[i]} )) - result="$result${args[i]}" - done - echo "$total${args[1],}$result" - else - echo "$1" - fi; -} - -# Use GNU sed to fix placement notation -fix_place="\ -s/P \(..\) C,/C\L\1,/g;\ -s/P \(..\) W,/S\L\1,/g;\ -s/P \(..\),/\L\1,/g" - -# Thiss is way too slow -extract_ptn() { - extract "notation, result" - - size="$1" - dir="ptn-$size" - if [ ! -d "$dir" ]; then - mkdir "$dir"; - fi; - - # Separate each line into a file - count=0 - while read game; do - file="$dir/$count.ptn" - echo "$game" | tr ',' '\n' | sed "$fix_place" > "$file" - # Prepare to reformat log as PTN - ply=1 - prepend="" - new_contents="" - while read line; do - # PTN calls for numbers before every new turn in the game - if [ $(( $ply % 2 )) -eq 1 ]; then - prepend="$(( $ply / 2 + 1 )). " - if [ $ply -gt 1 ]; then - prepend="\n$prepend" - fi - else - prepend=" " - fi - # We have to fix the move notation - line=$(fix_move "$line") - new_contents="$new_contents$prepend$line" - ply=$(( $ply + 1 )) - done < "$file" - echo -e "$new_contents" | tr '|' '\n' > "$file" - count=$(( $count + 1 )) - done < "playtak-$size" -} - if [ ! -f "$db_file" ]; then wget "https://www.playtak.com/games_anon.db" fi extract notation,result + +echo Preparing pptdb +cd .. +if [ ! -f "pptdb" ]; then + make pptdb +fi +echo -e "Beginning to process data\n" + +for i in 5 6; do + echo "Size $i..." + ./pptdb "$i" "data/playtak-$i" > "data/check-$i" + tail -n21 "data/check-$i" + echo Stripping overflows and illegal games... + grep -Fvxf "data/check-$i" "data/playtak-$i" > "data/good-playtak-$i" + echo -n "Generating training data... " + ./pptdb "$i" "data/good-playtak-$i" generate + echo "Shuffling data..." + shuf "data/training-$i.csv" > "data/shuf-$i.csv" + mv "data/shuf-$i.csv" "data/training-$i.csv" + echo +done diff --git a/src/pptdb.c b/src/pptdb.c index eb39e25..34056c4 100644 --- a/src/pptdb.c +++ b/src/pptdb.c @@ -6,7 +6,7 @@ uint64_t heights[16]; FILE *training_fh = NULL; -int wc_col, bc_col, wc_row, bc_row, result; +int wc_col, bc_col, wc_row, bc_row, result, generate; static void write_input(void) { @@ -100,8 +100,10 @@ parse_line(const char *pt, const ssize_t read) { } } // Generate training data - write_input(); - fprintf(training_fh,"%d,%d\n", result, 1-result); + if (generate) { + write_input(); + fprintf(training_fh,"%d,%d\n", result, 1-result); + } // Parse next action while (idx<read && pt[idx++]!=','); if (idx>=read) return ACT_OK; @@ -131,9 +133,12 @@ main(int argc, char **argv) { playtak_fh = fopen(argv[2], "r"); if (playtak_fh == NULL) exit(EXIT_FAILURE); - snprintf(td_fn, 64, "data/training-%d.csv",size); - training_fh = fopen(td_fn, "w"); - if (training_fh == NULL) exit(EXIT_FAILURE); + if (argc > 3 && (!strncmp("generate", argv[3], 8))) { + generate=1; + snprintf(td_fn, 64, "data/training-%d.csv",size); + training_fh = fopen(td_fn, "w"); + if (training_fh == NULL) exit(EXIT_FAILURE); + } else generate=0; while ((read = getline(&line, &len, playtak_fh)) != -1) { // Reset everything @@ -143,42 +148,48 @@ main(int argc, char **argv) { // Store the result of this game if (line[read-4] == '0') result = 1; else result = 0; - // Parse the line and adjust counts + // Parse the line r = parse_line(line,read-4); - if (r == ACT_ILLEGAL) { - illegal++; - printf("Illegal:\n%s",line); - } else if (r == ACT_OVERFLOW) { - printf("Overflow:\n%s",line); - overflow++; - } else { - win = check_win(); - if (win == WIN_FLAT_BLACK - || win == WIN_FLAT_WHITE - || win == WIN_DRAW) { - flat_wins++; - flat_turns += ply/2+1; + // Adjust counts if we're not generating training data + if (generate == 0) { + if (r == ACT_ILLEGAL) { + illegal++; + printf("Illegal:\n%s",line); + } else if (r == ACT_OVERFLOW) { + printf("Overflow:\n%s",line); + overflow++; } else { - road_wins++; - road_turns += ply/2+1; + win = check_win(); + if (win == WIN_FLAT_BLACK + || win == WIN_FLAT_WHITE + || win == WIN_DRAW) { + flat_wins++; + flat_turns += ply/2+1; + } else { + road_wins++; + road_turns += ply/2+1; + } } } games++; } fclose(playtak_fh); - fclose(training_fh); + if (generate) fclose(training_fh); if (line) free(line); if (illegal || overflow) putchar('\n'); printf("Read %d games\n",games); - printf("Illegals: %d\nOverflows: %d\nRoad wins: %d\nFlat wins: %d\n\ + + if (generate==0) { + printf("Illegals: %d\nOverflows: %d\nRoad wins: %d\nFlat wins: %d\n\ Average turns to road win: %.3f\nAverage turns to flat win: %f\n", - illegal,overflow, road_wins, flat_wins, - (double)(road_turns)/(double)(road_wins), - (double)(flat_turns)/(double)(flat_wins)); - for (int k = 2; k < 16; k++) { - printf("Height %2d: %7ld\n",k,heights[k]); + illegal,overflow, road_wins, flat_wins, + (double)(road_turns)/(double)(road_wins), + (double)(flat_turns)/(double)(flat_wins)); + for (int k = 2; k < 16; k++) { + printf("Height %2d: %7ld\n",k,heights[k]); + } } exit(EXIT_SUCCESS); |