Back to Topics

1 Hidden Layer : 3 neurons, 1 Output Layer

Class + Explicit network definition

from IPython.display import Image
Image(filename = "nn_img/Python_Pytorch_nn_Sequential_i2_H1_n3_o1_sigmoid_01a.png", width=500)

from IPython.display import Image
Image(filename = "nn_img/Python_Pytorch_nn_Sequential_i2_H1_n3_o1_sigmoid_01.png", width=500)

Prerequisites

import torch
from torch import nn
from torch import optim
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
#from draw import display_solution

#predictive
torch.manual_seed(1);

device = torch.device('cpu')
if torch.cuda.is_available():
    device = torch.device('cuda')  
    
#sigmoid explicit
def sigmoid(x):
    return 1/(1+torch.exp(-x))

#sigmoid pre-defined
activation = torch.nn.Sigmoid()

# Loss (Binary Cross Entropy) error function, explicit def
def bce_err(output, target):
    return -target * torch.log(output) - (1-target) * torch.log(1-output)

#sigmoid + BCELoss (Binary Cross Entropy)
criterion = torch.nn.BCEWithLogitsLoss()

def state(interval,i,loss,out,y):
    if(i%interval == 0):
        print("\n========== Epoch", i,"==========")
        print("loss=",loss)
        accuracy = np.mean( ((out > 0.5)==y).numpy() )
        print("accuracy=",accuracy)
        
#array data points: x1, x2
data = np.array([
    [1,10,1],
    [3,10,0],
    [1.8,2.0,0],
    [-1,-1,1],
    [-2,10,1],
])         

#df = pd.read_csv('test.csv', header=None)
#df = pd.read_csv('data2.csv', header=None)
#data = df.to_numpy()        

Class + Explicit network definition

#features
x = torch.from_numpy(data[:, [0,1]]).double()
#target/labels
y = torch.from_numpy(data[:, [2]]).double()

torch.manual_seed(1);

class NN():    
    def __init__(self, n_input, n_hidden, n_output):

        # Weights for inputs to hidden layer
        self.w1 = torch.randn(n_input, n_hidden, dtype=torch.double, requires_grad=True)
        # Weights for hidden layer to output layer
        self.w2 = torch.randn(n_hidden, n_output, dtype=torch.double, requires_grad=True)

        # and bias terms for hidden and output layers
        self.b1 = torch.randn(1, n_hidden, dtype=torch.double, requires_grad=True)
        self.b2 = torch.randn(1, n_output, dtype=torch.double, requires_grad=True)  
        
        self.activation = torch.nn.Sigmoid()
        
    def forward(self,x):        
        o = activation(torch.mm(x,self.w1) + self.b1)
        o = activation(torch.mm(o,self.w2) + self.b2)        
        return o
        
net = NN(2,3,1)
print(net.w1)
print(net.b1)
print(net.w2)
print(net.b2)
print(net.forward(x))         

tensor([[ 0.6614,  0.2669,  0.0617],
        [ 0.6213, -0.4519, -0.1661]], dtype=torch.float64, requires_grad=True)
tensor([[-0.5631, -0.8923, -0.0583]], dtype=torch.float64, requires_grad=True)
tensor([[-1.5228],
        [ 0.3817],
        [-1.0276]], dtype=torch.float64, requires_grad=True)
tensor([[-0.1955]], dtype=torch.float64, requires_grad=True)
tensor([[0.1326],
        [0.1306],
        [0.1328],
        [0.3094],
        [0.1373]], dtype=torch.float64, grad_fn=<SigmoidBackward>)

#learn rate
alpha = 0.01
#iterations
epochs = 1001
#display state
fv = 100

lossHistory = []
predictionHistory = []

for i in range(epochs):
    
    #forward: output/prediction
    out = net.forward(x)
    
    loss = torch.mean(bce_err(out,y))
    
    #alternative
    #out = torch.mm(X, W.view(2,1))+b
    #loss = criterion(out, y)  
      
    #backward: compute gradients
    loss.backward()   

    #update weights (disable gradients when update weights)
    with torch.no_grad():
        net.w1 -= alpha * net.w1.grad
        net.b1 -= alpha * net.b1.grad
        net.w2 -= alpha * net.w2.grad
        net.b2 -= alpha * net.b2.grad
        
        # Manually zero the gradients after updating weights
        net.w1.grad.zero_()
        net.b1.grad.zero_()  
        net.w2.grad.zero_()
        net.b2.grad.zero_()  
                
    lossHistory.append(loss)    

    state(fv,i,loss,out,y)

========== Epoch 0 ==========
loss= tensor(1.0922, dtype=torch.float64, grad_fn=<MeanBackward0>)
accuracy= 0.4

========== Epoch 100 ==========
loss= tensor(0.8401, dtype=torch.float64, grad_fn=<MeanBackward0>)
accuracy= 0.4

========== Epoch 200 ==========
loss= tensor(0.7213, dtype=torch.float64, grad_fn=<MeanBackward0>)
accuracy= 0.6

========== Epoch 300 ==========
loss= tensor(0.6621, dtype=torch.float64, grad_fn=<MeanBackward0>)
accuracy= 0.6

========== Epoch 400 ==========
loss= tensor(0.6286, dtype=torch.float64, grad_fn=<MeanBackward0>)
accuracy= 0.6

========== Epoch 500 ==========
loss= tensor(0.6024, dtype=torch.float64, grad_fn=<MeanBackward0>)
accuracy= 0.8

========== Epoch 600 ==========
loss= tensor(0.5779, dtype=torch.float64, grad_fn=<MeanBackward0>)
accuracy= 0.8

========== Epoch 700 ==========
loss= tensor(0.5566, dtype=torch.float64, grad_fn=<MeanBackward0>)
accuracy= 1.0

========== Epoch 800 ==========
loss= tensor(0.5373, dtype=torch.float64, grad_fn=<MeanBackward0>)
accuracy= 1.0

========== Epoch 900 ==========
loss= tensor(0.5189, dtype=torch.float64, grad_fn=<MeanBackward0>)
accuracy= 1.0

========== Epoch 1000 ==========
loss= tensor(0.5009, dtype=torch.float64, grad_fn=<MeanBackward0>)
accuracy= 1.0

#loss evolution
graph_x = np.arange(0, epochs)
graph_y = lossHistory
plt.plot(graph_x, graph_y)
plt.show()

Setting custom weights

net.w1 = torch.tensor( [[1, -1, 1],
                    [1, -1, 1]], dtype=torch.float64)
net.b1 = torch.tensor( [[1, -1, 1]], dtype=torch.float64)

net.w2 = torch.tensor( [[ 1],
                    [-1],
                    [ 1]], dtype=torch.float64)
net.b2 = torch.tensor( [[ 1]], dtype=torch.float64)

print(x)
out = net.forward(x)
print("---\n",out,"\n---")

tensor([[ 1.0000, 10.0000],
        [ 3.0000, 10.0000],
        [ 1.8000,  2.0000],
        [-1.0000, -1.0000],
        [-2.0000, 10.0000]], dtype=torch.float64)
---
 tensor([[0.9526],
        [0.9526],
        [0.9515],
        [0.6914],
        [0.9526]], dtype=torch.float64) 
---

net.w1 = torch.tensor( [[1, 1, 1],
                    [1, 1, 1]], dtype=torch.float64)
net.b1 = torch.tensor( [[1, 1, 1]], dtype=torch.float64)

net.w2 = torch.tensor( [[ 1],
                    [ 1],
                    [ 1]], dtype=torch.float64)
net.b2 = torch.tensor( [[ 1]], dtype=torch.float64)

print(x)
out = net.forward(x)
print("---\n",out,"\n---")

tensor([[ 1.0000, 10.0000],
        [ 3.0000, 10.0000],
        [ 1.8000,  2.0000],
        [-1.0000, -1.0000],
        [-2.0000, 10.0000]], dtype=torch.float64)
---
 tensor([[0.9820],
        [0.9820],
        [0.9816],
        [0.8590],
        [0.9820]], dtype=torch.float64) 
---

Corciu Mihai - 2021 - mc200520052005@yahoo.com