Separate concerns in pptdb, remove legacy in extract.sh

2 files changed, 62 insertions, 105 deletions

diff --git a/data/extract.sh b/data/extract.sh
index b586429..285be26 100755
--- a/data/extract.sh
+++ b/data/extract.sh
@@ -10,88 +10,34 @@ and (result != '1/2-1/2');"
 
 extract() {
 		for size in 5 6; do
+				echo Extracing games of size "$size"...
 				sqlite3 "$db_file" "$(query $1 $size)" > "playtak-$size"
 		done
 }
 
-# Correct move notation, assuming only valid stuff
-fix_move() {
-		args=($1)
-		if [ "${args[0]}" = "M" ]; then
-				result=""
-				# Annoying to compute direction
-				base_col=${args[1]:0:1}
-				base_row=${args[1]:1:1}
-				step_col=${args[2]:0:1}
-				step_row=${args[2]:1:1}
-				if [[ "$base_col" < "$step_col" ]]; then
-						result=">"
-				elif [[ "$base_col" > "$step_col" ]]; then
-						result="<"
-				elif [[ "$base_row" > "$step_row" ]]; then
-						result="-"
-				elif [[ "$base_row" < "$step_row" ]]; then
-						result="+"
-				fi
-				total=0
-				steps=${#args[@]}
-				for (( i = 3 ; i < steps ; i++ )); do
-						total=$(( total + ${args[i]} ))
-						result="$result${args[i]}"
-				done
-				echo "$total${args[1],}$result"
-		else
-				echo "$1"
-		fi;
-}
-
-# Use GNU sed to fix placement notation
-fix_place="\
-s/P \(..\) C,/C\L\1,/g;\
-s/P \(..\) W,/S\L\1,/g;\
-s/P \(..\),/\L\1,/g"
-
-# Thiss is way too slow
-extract_ptn() {
-		extract "notation, result"
-
-		size="$1"
-		dir="ptn-$size"
-		if [ ! -d "$dir" ]; then
-				mkdir "$dir";
-		fi;
-
-		# Separate each line into a file
-		count=0
-		while read game; do
-				file="$dir/$count.ptn"
-				echo "$game" | tr ',' '\n' | sed "$fix_place" > "$file"
-				# Prepare to reformat log as PTN
-				ply=1
-				prepend=""
-				new_contents=""
-				while read line; do
-						# PTN calls for numbers before every new turn in the game
-						if [ $(( $ply % 2 )) -eq 1 ]; then
-								prepend="$(( $ply / 2 + 1 )). "
-								if [ $ply -gt 1 ]; then
-										prepend="\n$prepend"
-								fi
-						else
-								prepend=" "
-						fi
-						# We have to fix the move notation
-						line=$(fix_move "$line")
-						new_contents="$new_contents$prepend$line"
-						ply=$(( $ply + 1 ))
-				done < "$file"
-				echo -e "$new_contents" | tr '|' '\n' > "$file"
-				count=$(( $count + 1 ))
-		done < "playtak-$size"
-}
-
 if [ ! -f "$db_file" ]; then
 		wget "https://www.playtak.com/games_anon.db"
 fi
 
 extract notation,result
+
+echo Preparing pptdb
+cd ..
+if [ ! -f "pptdb" ]; then
+	 make pptdb
+fi
+echo -e "Beginning to process data\n"
+
+for i in 5 6; do
+		echo "Size $i..."
+		./pptdb "$i" "data/playtak-$i" > "data/check-$i"
+		tail -n21 "data/check-$i"
+		echo Stripping overflows and illegal games...
+		grep -Fvxf "data/check-$i" "data/playtak-$i" > "data/good-playtak-$i"
+		echo -n "Generating training data... "
+		./pptdb "$i" "data/good-playtak-$i" generate
+		echo "Shuffling data..."
+		shuf "data/training-$i.csv" > "data/shuf-$i.csv"
+		mv "data/shuf-$i.csv" "data/training-$i.csv"
+		echo
+done
diff --git a/src/pptdb.c b/src/pptdb.c
index eb39e25..34056c4 100644
--- a/src/pptdb.c
+++ b/src/pptdb.c
@@ -6,7 +6,7 @@
 
 uint64_t heights[16];
 FILE *training_fh = NULL;
-int wc_col, bc_col, wc_row, bc_row, result;
+int wc_col, bc_col, wc_row, bc_row, result, generate;
 
 static void
 write_input(void) {
@@ -100,8 +100,10 @@ parse_line(const char *pt, const ssize_t read) {
 			}
 		}
 		// Generate training data
-		write_input();
-		fprintf(training_fh,"%d,%d\n", result, 1-result);
+		if (generate) {
+			write_input();
+			fprintf(training_fh,"%d,%d\n", result, 1-result);
+		}
 		// Parse next action
 		while (idx<read && pt[idx++]!=',');
 		if (idx>=read) return ACT_OK;
@@ -131,9 +133,12 @@ main(int argc, char **argv) {
 	playtak_fh = fopen(argv[2], "r");
 	if (playtak_fh == NULL) exit(EXIT_FAILURE);
 
-	snprintf(td_fn, 64, "data/training-%d.csv",size);
-	training_fh = fopen(td_fn, "w");
-	if (training_fh == NULL) exit(EXIT_FAILURE);
+	if (argc > 3 && (!strncmp("generate", argv[3], 8))) {
+		generate=1;
+		snprintf(td_fn, 64, "data/training-%d.csv",size);
+		training_fh = fopen(td_fn, "w");
+		if (training_fh == NULL) exit(EXIT_FAILURE);
+	} else generate=0;
 
 	while ((read = getline(&line, &len, playtak_fh)) != -1) {
 		// Reset everything
@@ -143,42 +148,48 @@ main(int argc, char **argv) {
 		// Store the result of this game
 		if (line[read-4] == '0') result = 1;
 		else result = 0;
-		// Parse the line and adjust counts
+		// Parse the line
 		r = parse_line(line,read-4);
-		if (r == ACT_ILLEGAL) {
-			illegal++;
-			printf("Illegal:\n%s",line);
-		} else if (r == ACT_OVERFLOW) {
-			printf("Overflow:\n%s",line);
-			overflow++;
-		} else {
-			win = check_win();
-			if (win == WIN_FLAT_BLACK
-					|| win == WIN_FLAT_WHITE
-					|| win == WIN_DRAW) {
-				flat_wins++;
-				flat_turns += ply/2+1;
+		// Adjust counts if we're not generating training data
+		if (generate == 0) {
+			if (r == ACT_ILLEGAL) {
+				illegal++;
+				printf("Illegal:\n%s",line);
+			} else if (r == ACT_OVERFLOW) {
+				printf("Overflow:\n%s",line);
+				overflow++;
 			} else {
-				road_wins++;
-				road_turns += ply/2+1;
+				win = check_win();
+				if (win == WIN_FLAT_BLACK
+						|| win == WIN_FLAT_WHITE
+						|| win == WIN_DRAW) {
+					flat_wins++;
+					flat_turns += ply/2+1;
+				} else {
+					road_wins++;
+					road_turns += ply/2+1;
+				}
 			}
 		}
 		games++;
 	}
 
 	fclose(playtak_fh);
-	fclose(training_fh);
+	if (generate) fclose(training_fh);
 	if (line) free(line);
 
 	if (illegal || overflow) putchar('\n');
 	printf("Read %d games\n",games);
-	printf("Illegals:  %d\nOverflows: %d\nRoad wins: %d\nFlat wins: %d\n\
+
+	if (generate==0) {
+		printf("Illegals:  %d\nOverflows: %d\nRoad wins: %d\nFlat wins: %d\n\
 Average turns to road win: %.3f\nAverage turns to flat win: %f\n",
-				 illegal,overflow, road_wins, flat_wins,
-				 (double)(road_turns)/(double)(road_wins),
-				 (double)(flat_turns)/(double)(flat_wins));
-	for (int k = 2; k < 16; k++) {
-		printf("Height %2d: %7ld\n",k,heights[k]);
+					 illegal,overflow, road_wins, flat_wins,
+					 (double)(road_turns)/(double)(road_wins),
+					 (double)(flat_turns)/(double)(flat_wins));
+		for (int k = 2; k < 16; k++) {
+			printf("Height %2d: %7ld\n",k,heights[k]);
+		}
 	}
 
 	exit(EXIT_SUCCESS);