First attempt at training data + exclude draws + list overflows too

Take the output and do grep -Fvxf output_file input_file to drop the
bad games from consideration

2 files changed, 139 insertions, 80 deletions

diff --git a/data/extract.sh b/data/extract.sh
index 37ce12c..b586429 100755
--- a/data/extract.sh
+++ b/data/extract.sh
@@ -3,45 +3,46 @@
 db_file=games_anon.db
 
 query() {
-    echo "SELECT $1 FROM games WHERE (size == $2) \
-and (result != '1-0') and (result != '0-1') and (result != '0-0');"
+		echo "SELECT $1 FROM games WHERE (size == $2) \
+and (result != '1-0') and (result != '0-1') and (result != '0-0')\
+and (result != '1/2-1/2');"
 }
 
 extract() {
-    for size in 5 6; do
-        sqlite3 "$db_file" "$(query $1 $size)" > "playtak-$size"
-    done
+		for size in 5 6; do
+				sqlite3 "$db_file" "$(query $1 $size)" > "playtak-$size"
+		done
 }
 
 # Correct move notation, assuming only valid stuff
 fix_move() {
-    args=($1)
-    if [ "${args[0]}" = "M" ]; then
-        result=""
-        # Annoying to compute direction
-        base_col=${args[1]:0:1}
-        base_row=${args[1]:1:1}
-        step_col=${args[2]:0:1}
-        step_row=${args[2]:1:1}
-        if [[ "$base_col" < "$step_col" ]]; then
-            result=">"
-        elif [[ "$base_col" > "$step_col" ]]; then
-            result="<"
-        elif [[ "$base_row" > "$step_row" ]]; then
-            result="-"
-        elif [[ "$base_row" < "$step_row" ]]; then
-            result="+"
-        fi
-        total=0
-        steps=${#args[@]}
-        for (( i = 3 ; i < steps ; i++ )); do
-            total=$(( total + ${args[i]} ))
-            result="$result${args[i]}"
-        done
-        echo "$total${args[1],}$result"
-    else
-        echo "$1"
-    fi;
+		args=($1)
+		if [ "${args[0]}" = "M" ]; then
+				result=""
+				# Annoying to compute direction
+				base_col=${args[1]:0:1}
+				base_row=${args[1]:1:1}
+				step_col=${args[2]:0:1}
+				step_row=${args[2]:1:1}
+				if [[ "$base_col" < "$step_col" ]]; then
+						result=">"
+				elif [[ "$base_col" > "$step_col" ]]; then
+						result="<"
+				elif [[ "$base_row" > "$step_row" ]]; then
+						result="-"
+				elif [[ "$base_row" < "$step_row" ]]; then
+						result="+"
+				fi
+				total=0
+				steps=${#args[@]}
+				for (( i = 3 ; i < steps ; i++ )); do
+						total=$(( total + ${args[i]} ))
+						result="$result${args[i]}"
+				done
+				echo "$total${args[1],}$result"
+		else
+				echo "$1"
+		fi;
 }
 
 # Use GNU sed to fix placement notation
@@ -52,45 +53,45 @@ s/P \(..\),/\L\1,/g"
 
 # Thiss is way too slow
 extract_ptn() {
-    extract "notation, result"
+		extract "notation, result"
 
-    size="$1"
-    dir="ptn-$size"
-    if [ ! -d "$dir" ]; then
-        mkdir "$dir";
-    fi;
+		size="$1"
+		dir="ptn-$size"
+		if [ ! -d "$dir" ]; then
+				mkdir "$dir";
+		fi;
 
-    # Separate each line into a file
-    count=0
-    while read game; do
-        file="$dir/$count.ptn"
-        echo "$game" | tr ',' '\n' | sed "$fix_place" > "$file"
-        # Prepare to reformat log as PTN
-        ply=1
-        prepend=""
-        new_contents=""
-        while read line; do
-            # PTN calls for numbers before every new turn in the game
-            if [ $(( $ply % 2 )) -eq 1 ]; then
-                prepend="$(( $ply / 2 + 1 )). "
-                if [ $ply -gt 1 ]; then
-                    prepend="\n$prepend"
-                fi
-            else
-                prepend=" "
-            fi
-            # We have to fix the move notation
-            line=$(fix_move "$line")
-            new_contents="$new_contents$prepend$line"
-            ply=$(( $ply + 1 ))
-        done < "$file"
-        echo -e "$new_contents" | tr '|' '\n' > "$file"
-        count=$(( $count + 1 ))
-    done < "playtak-$size"
+		# Separate each line into a file
+		count=0
+		while read game; do
+				file="$dir/$count.ptn"
+				echo "$game" | tr ',' '\n' | sed "$fix_place" > "$file"
+				# Prepare to reformat log as PTN
+				ply=1
+				prepend=""
+				new_contents=""
+				while read line; do
+						# PTN calls for numbers before every new turn in the game
+						if [ $(( $ply % 2 )) -eq 1 ]; then
+								prepend="$(( $ply / 2 + 1 )). "
+								if [ $ply -gt 1 ]; then
+										prepend="\n$prepend"
+								fi
+						else
+								prepend=" "
+						fi
+						# We have to fix the move notation
+						line=$(fix_move "$line")
+						new_contents="$new_contents$prepend$line"
+						ply=$(( $ply + 1 ))
+				done < "$file"
+				echo -e "$new_contents" | tr '|' '\n' > "$file"
+				count=$(( $count + 1 ))
+		done < "playtak-$size"
 }
 
 if [ ! -f "$db_file" ]; then
-    wget "https://www.playtak.com/games_anon.db"
+		wget "https://www.playtak.com/games_anon.db"
 fi
 
-extract notation
+extract notation,result
diff --git a/src/pptdb.c b/src/pptdb.c
index 3812ee4..c536637 100644
--- a/src/pptdb.c
+++ b/src/pptdb.c
@@ -5,10 +5,40 @@
 #include <tak.h>
 
 uint64_t heights[16];
+FILE *training_fh = NULL;
+int wc_col, bc_col, wc_row, bc_row, result;
+
+static void
+write_input(void) {
+	// Four numbers for capstone coords (col,row)
+	fprintf(training_fh,"%d,%d,%d,%d,",wc_col,wc_row,bc_col,bc_row);
+	// Two numbers for flats remaining
+	fprintf(training_fh,"%d,%d,",white_count & 127, black_count & 127);
+	// Two layers of board_size * board_size:
+	int h, t; uint16_t mask;
+	for (int k = 0; k < board_size * board_size; k++) {
+		// stacks encoded as balanced ternary without capstones and walls
+		t = 0; mask = 1;
+		h = COUNT_AT(k);
+		if (STONE_AT(k)) h--;
+		while (h-->0) {
+			t *= 3;
+			t += (colours[k] & mask) ? +1 : -1;
+			mask <<= 1;
+		}
+		fprintf(training_fh,"%d,",t);
+	}
+	for (int k = 0; k < board_size * board_size; k++) {
+		// walls with +- 1
+		if (COUNT_AT(k) && STONE_AT(k) == STONE_STANDING)
+			fprintf(training_fh,"%d,",(colours[k] & 1) ? +1 : -1);
+		else fputs("0,", training_fh);
+	}
+}
 
 // Warning: performs _no_ checks on input whatsoever
 static enum E_RESULT
-parse_line(char *pt, const ssize_t read) {
+parse_line(const char *pt, const ssize_t read) {
 	ssize_t idx = 0;
 	enum E_RESULT r;
 	for(;;) {
@@ -21,7 +51,17 @@ parse_line(char *pt, const ssize_t read) {
 			if (idx + 3 < read) {
 				switch (pt[idx+3]) {
 					case 'W': { stone = STONE_STANDING; break; }
-					case 'C': { stone = STONE_CAPSTONE; break; }
+					case 'C': {
+						if (current_colour == C_BLACK) {
+							bc_col = col + 1;
+							bc_row = row + 1;
+						} else {
+							wc_col = col + 1;
+							wc_row = row + 1;
+						}
+						stone = STONE_CAPSTONE;
+						break;
+					}
 					default:  { stone = STONE_FLAT;     break; }
 				}
 			} else {
@@ -38,7 +78,7 @@ parse_line(char *pt, const ssize_t read) {
 				d_col=pt[idx+3]-'A', d_row=pt[idx+4]-'1';
 			idx+=4;
 
-			enum MOVE_DIRECTION dir;
+			enum MOVE_DIRECTION dir = M_RIGHT;
 			if (s_col < d_col) dir=M_RIGHT;
 			else if (s_col > d_col) dir=M_LEFT;
 			else if (s_row < d_row) dir=M_UP;
@@ -59,6 +99,10 @@ parse_line(char *pt, const ssize_t read) {
 				if (COUNT_AT(k)>1) heights[COUNT_AT(k)]+=1;
 			}
 		}
+		// Generate training data
+		write_input();
+		fprintf(training_fh,"%d,%d\n", result, 1-result);
+		// Parse next action
 		while (idx<read && pt[idx++]!=',');
 		if (idx>=read) return ACT_OK;
 		next_ply();
@@ -66,7 +110,6 @@ parse_line(char *pt, const ssize_t read) {
 	return ACT_OK;
 }
 
-
 int
 main(int argc, char **argv) {
 	(void)(argc);
@@ -78,22 +121,35 @@ main(int argc, char **argv) {
 
 	for (int k = 0; k < 16; k++) heights[k] = 0;
 
-	FILE *playtak;
-	ssize_t read;
 	size_t len = 0;
-	char *line = NULL;
+	ssize_t read = 0;
+	FILE *playtak_fh = NULL;
+	char *line = NULL, td_fn[65];
 
 	const uint8_t size = argv[1][0]-'0';
-	playtak = fopen(argv[2], "r");
-	if (playtak == NULL) exit(EXIT_FAILURE);
 
-	while ((read = getline(&line, &len, playtak)) != -1) {
+	playtak_fh = fopen(argv[2], "r");
+	if (playtak_fh == NULL) exit(EXIT_FAILURE);
+
+	snprintf(td_fn, 64, "training-%d.csv",size);
+	training_fh = fopen(td_fn, "w");
+	if (training_fh == NULL) exit(EXIT_FAILURE);
+
+	while ((read = getline(&line, &len, playtak_fh)) != -1) {
+		// Reset everything
 		reset_state(size);
-		r = parse_line(line,read);
+		bc_col = 0; wc_col = 0;
+		bc_row = 0; wc_row = 0;
+		// Store the result of this game
+		if (line[read-4] == '0') result = 1;
+		else result = 0;
+		// Parse the line and adjust counts
+		r = parse_line(line,read-4);
 		if (r == ACT_ILLEGAL) {
 			illegal++;
-			printf("Culprit: (%ld) %s",read,line);
+			printf("Illegal:\n%s",line);
 		} else if (r == ACT_OVERFLOW) {
+			printf("Overflow:\n%s",line);
 			overflow++;
 		} else {
 			win = check_win();
@@ -110,9 +166,11 @@ main(int argc, char **argv) {
 		games++;
 	}
 
-	fclose(playtak);
+	fclose(playtak_fh);
+	fclose(training_fh);
 	if (line) free(line);
 
+	if (illegal || overflow) putchar('\n');
 	printf("Read %d games\n",games);
 	printf("Illegals:  %d\nOverflows: %d\nRoad wins: %d\nFlat wins: %d\n\
 Average turns to road win: %.3f\nAverage turns to flat win: %f\n",