Separate concerns in pptdb, remove legacy in extract.sh

1 files changed, 22 insertions, 76 deletions

diff --git a/data/extract.sh b/data/extract.sh
index b586429..285be26 100755
--- a/data/extract.sh
+++ b/data/extract.sh
@@ -10,88 +10,34 @@ and (result != '1/2-1/2');"
 
 extract() {
 		for size in 5 6; do
+				echo Extracing games of size "$size"...
 				sqlite3 "$db_file" "$(query $1 $size)" > "playtak-$size"
 		done
 }
 
-# Correct move notation, assuming only valid stuff
-fix_move() {
-		args=($1)
-		if [ "${args[0]}" = "M" ]; then
-				result=""
-				# Annoying to compute direction
-				base_col=${args[1]:0:1}
-				base_row=${args[1]:1:1}
-				step_col=${args[2]:0:1}
-				step_row=${args[2]:1:1}
-				if [[ "$base_col" < "$step_col" ]]; then
-						result=">"
-				elif [[ "$base_col" > "$step_col" ]]; then
-						result="<"
-				elif [[ "$base_row" > "$step_row" ]]; then
-						result="-"
-				elif [[ "$base_row" < "$step_row" ]]; then
-						result="+"
-				fi
-				total=0
-				steps=${#args[@]}
-				for (( i = 3 ; i < steps ; i++ )); do
-						total=$(( total + ${args[i]} ))
-						result="$result${args[i]}"
-				done
-				echo "$total${args[1],}$result"
-		else
-				echo "$1"
-		fi;
-}
-
-# Use GNU sed to fix placement notation
-fix_place="\
-s/P \(..\) C,/C\L\1,/g;\
-s/P \(..\) W,/S\L\1,/g;\
-s/P \(..\),/\L\1,/g"
-
-# Thiss is way too slow
-extract_ptn() {
-		extract "notation, result"
-
-		size="$1"
-		dir="ptn-$size"
-		if [ ! -d "$dir" ]; then
-				mkdir "$dir";
-		fi;
-
-		# Separate each line into a file
-		count=0
-		while read game; do
-				file="$dir/$count.ptn"
-				echo "$game" | tr ',' '\n' | sed "$fix_place" > "$file"
-				# Prepare to reformat log as PTN
-				ply=1
-				prepend=""
-				new_contents=""
-				while read line; do
-						# PTN calls for numbers before every new turn in the game
-						if [ $(( $ply % 2 )) -eq 1 ]; then
-								prepend="$(( $ply / 2 + 1 )). "
-								if [ $ply -gt 1 ]; then
-										prepend="\n$prepend"
-								fi
-						else
-								prepend=" "
-						fi
-						# We have to fix the move notation
-						line=$(fix_move "$line")
-						new_contents="$new_contents$prepend$line"
-						ply=$(( $ply + 1 ))
-				done < "$file"
-				echo -e "$new_contents" | tr '|' '\n' > "$file"
-				count=$(( $count + 1 ))
-		done < "playtak-$size"
-}
-
 if [ ! -f "$db_file" ]; then
 		wget "https://www.playtak.com/games_anon.db"
 fi
 
 extract notation,result
+
+echo Preparing pptdb
+cd ..
+if [ ! -f "pptdb" ]; then
+	 make pptdb
+fi
+echo -e "Beginning to process data\n"
+
+for i in 5 6; do
+		echo "Size $i..."
+		./pptdb "$i" "data/playtak-$i" > "data/check-$i"
+		tail -n21 "data/check-$i"
+		echo Stripping overflows and illegal games...
+		grep -Fvxf "data/check-$i" "data/playtak-$i" > "data/good-playtak-$i"
+		echo -n "Generating training data... "
+		./pptdb "$i" "data/good-playtak-$i" generate
+		echo "Shuffling data..."
+		shuf "data/training-$i.csv" > "data/shuf-$i.csv"
+		mv "data/shuf-$i.csv" "data/training-$i.csv"
+		echo
+done