Commit c3e1db3e authored by Jan Kovář's avatar Jan Kovář
Browse files

Few changes for the purpose of testing

parent 33510b6e

Data.py

+8 −4
Original line number Diff line number Diff line
@@ -9,11 +9,15 @@ def additional_input(k, x, y): 
        return y*y

    elif k == 2:

        return x*y

    elif k == 3:

        return np.sin(x)*np.cos(y)

    elif k == 4:

        return np.sin(y)*np.cos(x)

    else:

        return x*y



class Data:

    input_data_dimension = 4 

    input_data_dimension = 2 

    input_coord_dimension = 2

    output_data_dimension = 1

    data_size = 1
@@ -156,7 +160,7 @@ class Data: 
            self.labelled_data[2*i+1] = 1

            

            # Increase the angle and distance for the next iteration

            angle += 6*math.pi/self.data_size

            angle += 4*math.pi/self.data_size

            distance += 1.5/self.data_size



    def createTwoElipses(self):
@@ -165,10 +169,10 @@ class Data: 


        center_x1 = 0.4

        center_y1 = 0.4

        range1 = 0.4

        range1 = 0.5

        center_x2 = -0.8

        center_y2 = -0.8

        range2 = 0.5

        range2 = 0.4



        for i in range(self.data_size):

            if self.norm((self.all_data[i][0]-center_x1)/1, (self.all_data[i][1]-center_y1)/1) < range1:

LearningController.py

+110 −103
Original line number Diff line number Diff line
@@ -13,7 +13,7 @@ class TrainingController: 
    folder_path = "model/"

    learning_rate = 0.1

    l2_regularization = 0.0

    batch_size = 30

    batch_size = 10

    max_epochs = 1000

    train_loss = 1.0

    train_loss_baseline = 1.0
@@ -21,6 +21,7 @@ class TrainingController: 
    test_loss_baseline = 1.0

    total_loss = 1.0

    total_loss_baseline = 1.0

    epoch = 0

    

    train_loss_history = []

    test_loss_history = []
@@ -29,6 +30,9 @@ class TrainingController: 
        self.network = network

        self.network.setLearningParameters(self.learning_rate, self.l2_regularization)



        plt.ion()

        self.fig, (self.state,self.cost) = plt.subplots(1,2, figsize=(10, 5))

        

    def trainOnBatch(self, batch, labels):

        for j in range(len(batch)):

            self.network.activation(batch[j])
@@ -84,113 +88,67 @@ class TrainingController: 
                self.trainOnBatch(batch, labels)

                self.network.updateWeights()



    def trainOnDataset(self,dataset,number_of_epochs):

        for i in range(number_of_epochs):

            self.trainOneEpoch(dataset)

            # do functions for graphics

    

    def saveTrainedModel(self):

        pass

        #TODO saving plots, history in txt, weights in txt and numpy for network loading





data_size = 1000

training_data_size = 300



data = Data(data_size, training_data_size)

if(data.initializeDataSet("spiral") == False):

    print("Dataset not defined")

    sys.exit(1)



structure = [data.input_data_dimension,8,8,8,data.output_data_dimension]

network = NeuralNetwork(structure)



training_controller = TrainingController(network)



os.makedirs(training_controller.folder_path, exist_ok=True)



plt.ion()

fig, (state,cost) = plt.subplots(1,2, figsize=(10, 5))



test_points_x = np.linspace(-1,1,101)

test_points_y = np.linspace(1,-1,101)



labelled_data_A = np.zeros((data_size, data.input_data_dimension))

labelled_data_B = np.zeros((data_size, data.input_data_dimension))



for k in range(data_size):

        if data.labelled_data[k] > 0:

            labelled_data_A[k] = data.all_data[k]

        else:

            labelled_data_B[k] = data.all_data[k]



labelled_data_A = labelled_data_A[~np.all(labelled_data_A == 0, axis=1)]

labelled_data_B = labelled_data_B[~np.all(labelled_data_B == 0, axis=1)]



epoch = 0

epsilon = 0.05

costFunction = 1.0

costFunction_training_value = 1.0



while epoch < training_controller.max_epochs and training_controller.train_loss >= epsilon:

    print("Epoch "+str(epoch)+":")



    training_controller.trainOneEpochSGD(data,10)

    #training_controller.trainOneEpochMiniBGD(data)



            

    training_controller.train_loss = training_controller.network.updateLossFunction(data.all_data[data.training_indices], data.labelled_data[data.training_indices])

    training_controller.test_loss = training_controller.network.updateLossFunction(data.all_data[data.test_indices], data.labelled_data[data.test_indices])

    training_controller.total_loss = training_controller.train_loss+training_controller.test_loss

    if epoch == 0:

        training_controller.train_loss_baseline = training_controller.train_loss

        training_controller.test_loss_baseline = training_controller.test_loss

        training_controller.total_loss_baseline = training_controller.train_loss_baseline + training_controller.test_loss_baseline

    def updateLossFunctionAfterEpoch(self):

        self.train_loss = self.network.updateLossFunction(data.all_data[data.training_indices], data.labelled_data[data.training_indices])

        self.test_loss = self.network.updateLossFunction(data.all_data[data.test_indices], data.labelled_data[data.test_indices])

        self.total_loss = self.train_loss+self.test_loss

        if self.epoch == 0:

            self.train_loss_baseline = self.train_loss

            self.test_loss_baseline = self.test_loss

            self.total_loss_baseline = self.train_loss_baseline + self.test_loss_baseline



    training_controller.train_loss /= training_controller.train_loss_baseline

    training_controller.test_loss /= training_controller.test_loss_baseline

    training_controller.total_loss /= training_controller.total_loss_baseline

        self.train_loss /= self.train_loss_baseline

        self.test_loss /= self.test_loss_baseline

        self.total_loss /= self.total_loss_baseline



    training_controller.train_loss_history.append((epoch, training_controller.train_loss))

    training_controller.test_loss_history.append((epoch, (training_controller.test_loss)))

        self.train_loss_history.append((self.epoch, self.train_loss))

        self.test_loss_history.append((self.epoch, (self.test_loss)))



    print("Cost function: "+str(training_controller.total_loss))

    print("Training cost function: "+str(training_controller.train_loss))

        print("Cost function: "+str(self.total_loss))

        print("Training cost function: "+str(self.train_loss))



    cost.plot(*zip(*training_controller.train_loss_history), 'k-', label='Training data')

    cost.plot(*zip(*training_controller.test_loss_history), 'b-', label='Testing data')

    def plotAfterEpoch(self):

        if self.epoch % 10 == 0:

            self.cost.plot(*zip(*self.train_loss_history), 'k-', label='Training data')

            self.cost.plot(*zip(*self.test_loss_history), 'b-', label='Testing data')



    if epoch % 20 == 0:

        colormap = training_controller.network.computeHeatMap(test_points_x, test_points_y)

            colormap = self.network.computeHeatMap2(test_points_x, test_points_y)



            #heatmap = proc.imshow(colormap, cmap='RdYlGn', extent=([-1, 1, -1, 1]), interpolation='bilinear', vmin=-1, vmax=1)

        state.imshow(colormap, cmap='RdYlGn', extent=([-1, 1, -1, 1]), interpolation='bilinear', vmin=-1, vmax=1)

        state.plot(labelled_data_A[:,0], labelled_data_A[:,1], 'go', label="Class A")

        state.plot(labelled_data_B[:,0], labelled_data_B[:,1], 'ro', label="Class B")

        state.plot(data.training_data[:,0], data.training_data[:,1], 'k.', label="Training data")

            self.state.imshow(colormap, cmap='RdYlGn', extent=([-1, 1, -1, 1]), interpolation='bilinear', vmin=-1, vmax=1)

            self.state.plot(labelled_data_A[:,0], labelled_data_A[:,1], 'go', label="Class A")

            self.state.plot(labelled_data_B[:,0], labelled_data_B[:,1], 'ro', label="Class B")

            self.state.plot(data.training_data[:,0], data.training_data[:,1], 'k.', label="Training data")

            

        if(epoch == 0):

            state.legend(loc='lower center', bbox_to_anchor=(0.5, 1.0), ncols=3)

            if(self.epoch == 0):

                self.state.legend(loc='lower center', bbox_to_anchor=(0.5, 1.0), ncols=3)

                #fig.colorbar(heatmap, orientation="horizontal")

            cost.legend(loc='lower center', bbox_to_anchor=(0.5, 1.0), ncols=2)

                self.cost.legend(loc='lower center', bbox_to_anchor=(0.5, 1.0), ncols=2)



        fig.canvas.draw()

        fig.canvas.flush_events()

            self.fig.canvas.draw()

            self.fig.canvas.flush_events()



    epoch += 1

    def trainOnDataset(self,dataset,number_of_epochs):

        for i in range(number_of_epochs):

            self.trainOneEpoch(dataset)

            # do functions for graphics

    

colormap = training_controller.network.computeHeatMap(test_points_x, test_points_y)



    def saveTrainedModel(self):

        #TODO saving plots, history in txt, weights in txt and numpy for network loading

        colormap = self.network.computeHeatMap(test_points_x, test_points_y)



        #heatmap = proc.imshow(colormap, cmap='RdYlGn', extent=([-1, 1, -1, 1]), interpolation='bilinear', vmin=-1, vmax=1)

state.imshow(colormap, cmap='RdYlGn', extent=([-1, 1, -1, 1]), interpolation='bilinear', vmin=-1, vmax=1)

state.plot(labelled_data_A[:,0], labelled_data_A[:,1], 'go', label="Class A")

state.plot(labelled_data_B[:,0], labelled_data_B[:,1], 'ro', label="Class B")

state.plot(data.training_data[:,0], data.training_data[:,1], 'k.', label="Training data")

        self.state.imshow(colormap, cmap='RdYlGn', extent=([-1, 1, -1, 1]), interpolation='bilinear', vmin=-1, vmax=1)

        self.state.plot(labelled_data_A[:,0], labelled_data_A[:,1], 'go', label="Class A")

        self.state.plot(labelled_data_B[:,0], labelled_data_B[:,1], 'ro', label="Class B")

        self.state.plot(data.training_data[:,0], data.training_data[:,1], 'k.', label="Training data")



numberOfNeurons = np.sum(structure)

        #numberOfNeurons = np.sum(structure)

        maxNeuronsInLayer = np.max(structure)



networkMap = training_controller.network.computeWholeNeuralNetworkHeatMap(test_points_x, test_points_y, len(structure), maxNeuronsInLayer)

        networkMap = self.network.computeWholeNeuralNetworkHeatMap(test_points_x, test_points_y, len(structure), maxNeuronsInLayer)



        #fig2, subfigs = plt.subplots(maxNeuronsInLayer, len(structure), figsize=(4*maxNeuronsInLayer, 4*maxNeuronsInLayer))

        subfigs = []
@@ -198,13 +156,17 @@ fig2 = plt.figure(figsize=(4*len(structure), 4*maxNeuronsInLayer)) 


        max_weights_in_layer = []

        for i in range(len(structure)):

    max_weights_in_layer.append(np.max(training_controller.network.layer[i].weights))

            max_weights_in_layer.append(np.max(self.network.layer[i].weights))

        max_weight = np.max(np.array(max_weights_in_layer))



        for i in range(len(structure)):

            for j in range(structure[i]):

                subfig = fig2.add_subplot(maxNeuronsInLayer,  len(structure), i+len(structure)*j+1)

                subfig.imshow(networkMap[i+len(structure)*j], cmap='RdYlGn', extent=([-1, 1, -1, 1]), vmin=-1, vmax=1)

                if(i > 0):

                    subfig.set_xticks([])  # Remove x-axis tick labels

                    subfig.set_yticks([])  # Remove y-axis tick labels



                subfigs.append(subfig)

                

                index_of_first_neuron_in_previous_layer = int(np.sum(structure[:i-1]))
@@ -213,7 +175,7 @@ for i in range(len(structure)): 
                    xy2=[-1,0]



                    for index in range(structure[i-1]):

                weight = training_controller.network.layer[i].weights[j][index]/max_weight

                        weight = self.network.layer[i].weights[j][index]/max_weight

                        if weight >= 0:

                            con = ConnectionPatch(xyA=xy2, xyB=xy1, coordsA="data", coordsB="data", axesA=subfig, axesB=subfigs[index_of_first_neuron_in_previous_layer+index], color="red", linestyle='dashed', lw=3.0*weight)

                        else:
@@ -225,11 +187,56 @@ for i in range(len(structure)): 
        fig2.canvas.draw()

        fig2.canvas.flush_events()



        fig2.savefig(self.folder_path+"heatmap", dpi=500)

        self.fig.savefig(self.folder_path+"final_state", dpi=300)



        plt.show(block='false')



training_controller.network.printWeights()



training_controller.network.printBiases()

data_size = 1000

training_data_size = 300



data = Data(data_size, training_data_size)

if(data.initializeDataSet("two_elipses") == False):

    print("Dataset not defined")

    sys.exit(1)



structure = [data.input_data_dimension,5,5,5,data.output_data_dimension]

network = NeuralNetwork(structure)



training_controller = TrainingController(network)



os.makedirs(training_controller.folder_path, exist_ok=True)



test_points_x = np.linspace(-1,1,101)

test_points_y = np.linspace(1,-1,101)



labelled_data_A = np.zeros((data_size, data.input_data_dimension))

labelled_data_B = np.zeros((data_size, data.input_data_dimension))



for k in range(data_size):

        if data.labelled_data[k] > 0:

            labelled_data_A[k] = data.all_data[k]

        else:

            labelled_data_B[k] = data.all_data[k]



labelled_data_A = labelled_data_A[~np.all(labelled_data_A == 0, axis=1)]

labelled_data_B = labelled_data_B[~np.all(labelled_data_B == 0, axis=1)]



epsilon = 0.05

costFunction = 1.0

costFunction_training_value = 1.0



while training_controller.epoch < training_controller.max_epochs and training_controller.train_loss >= epsilon:

    print("Epoch "+str(training_controller.epoch)+":")



    training_controller.trainOneEpochSGD(data,10)

    #training_controller.trainOneEpochMiniBGD(data)

    #training_controller.trainOneEpochBGD(data)



    training_controller.updateLossFunctionAfterEpoch()

    training_controller.plotAfterEpoch()

    

    training_controller.epoch += 1



fig2.savefig(training_controller.folder_path+"heatmap", dpi=500)

fig.savefig(training_controller.folder_path+"final_state", dpi=500)
 
 No newline at end of file
 
training_controller.saveTrainedModel()
 
 No newline at end of file

NeuralNetwork.py

+15 −0
Original line number Diff line number Diff line
@@ -104,6 +104,21 @@ class NeuralNetwork: 
                heatmap[i][j] = self.getResult(point)

        return heatmap

    

    def computeHeatMap2(self, map_coord_x, map_coord_y):

        

        x = len(map_coord_x)

        y = len(map_coord_y)

        heatmap = np.zeros((x, y))

        neuron_count = len(self.layer[0].neurons) - 2



        for i in range(x):

            for j in range(y):

                add = np.array([additional_input(k, map_coord_x[j], map_coord_y[i]) for k in range(neuron_count)])

                point = np.hstack(([map_coord_x[j], map_coord_y[i]], add))

                heatmap[i][j] = self.getResult(point)



        return heatmap



    def getResultFromNeuron(self, layer_index, neuron_index):

        return self.layer[layer_index].neurons[neuron_index]

model/final_state.png

−475 KiB (502 KiB)
Loading image diff...

model/heatmap.png

−2.54 MiB (1020 KiB)
Loading image diff...