Trjtdtk

The data is in the format of a board with 'my'/'your' pieces and it is 'my' turn to move. Specifically the board is 8x4, and their are 4 piece types, and so the position is represented as a 4x8x4 with one-hot encoding. Then there is a layer of 2**13 hidden neurons, each of which is connected to every input on the 4x8x4 tensor with ReLU activation. Then there is an output layer of 3 neurons (L/D/W) are each connected to every hidden neuron with no activation function. The cost of a prediction is (L-is_lost)**2 + (D-is_drawn)**2 + (W-is_won)**2

I have implemented this in numpy. I am new to machine learning and I wanted to start at a low level of abstraction. This is the code:

import numpy as np

def relu(x):
 if x < 0:
 return 0
 return x

def relu_prime(x):
 if x < 0:
 return 0
 return 1

class NN():
def __init__(self, hidden_weights, hidden_biases, output_weights,output_biases):

 self.hidden_weights = hidden_weights
 self.hidden_biases = hidden_biases
 self.output_weights = output_weights
 self.output_biases = output_biases

def forward(self, tensor):
 hidden_neuron_count = 2**13 #not a parameter

 hidden_neuron_activations = []
 for f in range(hidden_neuron_count):
 activation = np.sum(self.hidden_weights[f,:,:]*tensor)+self.hidden_biases[f]
 hidden_neuron_activations.append(activation)

 hidden_neuron_activations = np.array(hidden_neuron_activations)
 output_activations = [np.sum(list(map(relu, hidden_neuron_activations))*self.output_weights[:,result_output])+self.output_biases[result_output] for result_output in range(3)]

 return (hidden_neuron_activations, output_activations)

def train_epoch(self, tensors, labels, lr_weights, lr_biases):
 hidden_neuron_count = 2**13 #not a parameter

 d_output_weights = np.zeros((hidden_neuron_count,3))
 d_output_biases = np.zeros(3)
 d_hidden_weights = np.zeros((hidden_neuron_count,4,8,4))
 d_hidden_biases = np.zeros(hidden_neuron_count)

 batch_size = len(tensors)

 for tensor, label in zip(tensors,labels):
 results = self.forward(tensor)
 hidden_neuron_activations = results[0]
 output_activations = results[1]

 Eo = np.array([ 2.0/batch_size*((output_activations[result_output])-float(int(result_output-1) == int(label))) for result_output in range(3)])

 d_output_biases += Eo
 d_output_weights += np.array([ np.multiply(Eo,relu(hidden_neuron_activations[neuron])) for neuron in range(hidden_neuron_count)])

 d_hidden_biases += np.array([ sum( np.multiply(Eo*self.output_weights[neuron], relu_prime(hidden_neuron_activations[neuron])) ) for neuron in range(hidden_neuron_count)])
 d_hidden_weights += np.array([ sum([np.multiply(tensor, Eo[j]*self.output_weights[neuron][j]*relu_prime(hidden_neuron_activations[neuron])) for j in range(3)]) for neuron in range(hidden_neuron_count)])

 self.output_weights -= lr_weights*d_output_weights
 self.output_biases -= lr_biases*d_output_biases
 self.hidden_weights -= lr_weights*d_hidden_weights
 self.hidden_biases -= lr_biases*d_hidden_biases

def show(self, tensors, labels):

 total_cost = 0
 batch_size = len(tensors)

 for tensor, label in zip(tensors,labels):

 activations = self.forward(tensor)
 output_activations = activations[1]
 cost = sum([((float(int(result_output-1) == int(label))-(output_activations[result_output])) )**2 for result_output in range(3)])
 total_cost += cost
 print( [(output_activations[result_output]) for result_output in range(3)] , cost, label )

 total_cost /= batch_size
 print('Average-cost: '.format(total_cost))

I am feeding it endgame positions. I've generated ~8,000 positions to feed it. THe error has decreased in 40 epochs (each epoch feeds it about 1/15 of the data) from 6000 to about 1500. I am doubtful at this point that tailoring the learning rate (I use double for biases) or giving it more data will improve this score. I also believe that I would have more success with sigmoid activations on both layers. Basically, should I press on or is this problem intractable on this small of a network. I have a GPU which supports an old CUDA, so I will soon do something more auspicious in pytorch.