1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
|
# train5.py, train a small CNN to recognise winning Tak positions
#
# Copyright (C) 2021, tslil clingman
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
# Loading data
import pandas as pd
import numpy as np
# Output of weights
from tensorflow import transpose
# Custom activation function
from tensorflow import constant as k
from tensorflow import clip_by_value
from tensorflow.math import add, divide, multiply, square
# Computing size
from tensorflow.keras.backend import get_value
# Building models
from tensorflow.keras.models import Model
from tensorflow.keras.layers import Concatenate
from tensorflow.keras.layers import Input
from tensorflow.keras.layers import Dense
from tensorflow.keras.layers import Convolution2D
from tensorflow.keras.layers import Flatten
# We need something that's close to logistic, but cheaper to compute.
# (12.0+x+50.0*x/(x*x+10.0))/24.0, clipped between 0 and 1
# as it would otherwise exceed this range at +- 4.6 or so
def clipped_pade_logistic(x):
val = add(k(12.0),
add(x, multiply(k(50.0),
divide(x, add(square(x), k(10.0))))))
return clip_by_value(divide(val, k(24.0)), 0.0, 1.0)
# A cheaper version of tanh, x/6+25*x/(6*(2*x*x+5)),
# clipped between -1 and 1.
def clipped_pade_tanh(x):
val = add(divide(x, k(6.0)),
multiply(k(25.0),
divide(x, multiply(k(6.0), add(k(5.0),
multiply(k(2.0), square(x)))))))
return clip_by_value(val, -1.0, 1.0)
def make_model(size, magic=[12, 9, 9]):
# Our model for the stacks, a small CNN
stack_shape = (size, size, 6)
stack_input = Input(shape=stack_shape)
stack_model = Convolution2D(magic[0], kernel_size=(3, 3), strides=(1, 1),
padding='valid', activation="relu",
use_bias=True)(stack_input)
stack_model = Flatten()(stack_model)
stack_model = Model(inputs=stack_input, outputs=stack_model)
# The overall model
flats_input = Input(shape=(2,))
combn_input = Concatenate()([stack_model.output, flats_input])
model = Dense(magic[1], activation="relu", use_bias=True)(combn_input)
model = Dense(magic[2], activation="relu", use_bias=True)(model)
model = Dense(1, activation=clipped_pade_logistic, use_bias=True)(model)
model = Model(inputs=[stack_model.input, flats_input], outputs=model)
model.compile(optimizer='adam',
loss='mean_squared_error',
# loss='mean_absolute_error',
# loss='mean_absolute_percentage_error',
metrics=['accuracy'])
model.summary()
return model
def load_data(size):
shape = (-1, size, size, 6)
# Load training data
tr_fn = "training-"+str(size)+".csv"
training_csv = pd.read_csv(tr_fn) #.head(100000)
training_data = training_csv
training_stack_input = np.array(training_data.iloc[:, 2:-1]).reshape(shape, order='F')
training_flats_input = np.array(training_data.iloc[:, 0:2])
training_input = [training_stack_input, training_flats_input]
training_outcome = training_data.iloc[:, -1:]
# Load validation data
val_fn = "validation-"+str(size)+".csv"
val_data = pd.read_csv(val_fn).tail(20000)
val_stack_input = np.array(val_data.iloc[:, 2:-1]).reshape(shape, order='F')
val_flats_input = np.array(val_data.iloc[:, 0:2])
val_input = [val_stack_input, val_flats_input]
val_outcome = val_data.iloc[:, -1:]
return [(training_input, training_outcome), (val_input, val_outcome)]
def train(size, model, data, iterations=1, epochs=10):
(training_input, training_outcome), (val_input, val_outcome) = data
results = []
for i in range(0, iterations):
print("Iteration {0}/{1}".format(i+1, iterations))
model.fit(training_input, training_outcome, epochs=epochs,
validation_data=(val_input, val_outcome),
verbose=True)
v_loss, v_accuracy = model.evaluate(val_input, val_outcome,
verbose=False, batch_size=16)
t_loss, t_accuracy = model.evaluate(training_input, training_outcome,
verbose=False, batch_size=32)
results += [((v_loss, t_loss), (v_accuracy, t_accuracy))]
print("\nScores")
for i in range(len(results)):
print("Iteration {0}: {1}".format(i+1, results[i]))
return results
def model_size(model):
return sum([np.prod(get_value(w).shape) for w in model.trainable_weights])
def magic_search(data):
results = []
for width in range(9, 18):
for dense1 in range(9, 16):
for dense2 in range(9, max(32, dense1*2)):
m = make_model(5, [width, dense1, dense2])
if model_size(m) < 1990:
results += [([width, dense1, dense2],
train(5, m, data, iterations=1, epochs=20))]
print("\nSummary")
for r in results:
print("Parameters {0}: {1}".format(r[0], r[1][0]))
return results
def write_weights(model):
def fix(val):
string = str(np.array(val).tolist())
string = string.replace("[", "{").replace("]", "}")
return string
# Prepare everything in a sane memory order This isn't exactly in
# the correct order that tensorflow uses, because memory access
# out of order is an eyesore. Compared to tensorflow, the C
# implementation has the board reflected about the diagonal.
conv2d_weights = transpose(model.trainable_variables[0], perm=[3, 0, 1, 2])
conv2d_biases = model.trainable_variables[1]
dense1_weights = transpose(model.trainable_variables[2], perm=[1, 0])
dense1_biases = model.trainable_variables[3])
dense2_weights = transpose(model.trainable_variables[4], perm=[1, 0])
dense2_biases = model.trainable_variables[5])
output_weights = transpose(model.trainable_variables[6], perm=[1, 0])[0]
output_bias = model.trainable_variables[7][0]
# Prepare formatting
names = ["conv2d_weights[KERN_NUM][KERN_SIZE][KERN_SIZE][KERN_CHAN]",
"conv2d_biases[KERN_NUM]",
"dense1_weights[DENSE1_NUM][CONV_NUM+2]",
"dense1_biases[DENSE1_NUM]",
"dense2_weights[DENSE2_NUM][DENSE1_NUM]",
"dense2_biases[DENSE2_NUM]",
"output_weights[DENSE2_NUM]",
"output_bias"]
variables = [conv2d_weights, conv2d_biases,
dense1_weights, dense1_biases,
dense2_weights, dense2_biases,
output_weights, output_bias]
# Write to file
f = open("weights.txt", "w")
f.write("#include \"weights.h\"\n\n")
for (name, val) in zip(names, variables):
f.write("const float "+name+" =\n"+fix(val)+";\n\n")
f.close()
data = load_data(5)
model = make_model(5, [12, 11, 8])
results = train(5, model, data, iterations=5, epochs=10)
write_weights(model)
# results = magic_search(data)
|
