In [1]:
from IPython.display import Image
Image(filename = "nn_img/Python_Pytorch_nn_Sequential_i2_o2_sigmoid_01a.png", width=500)
Out[1]:
In [2]:
from IPython.display import Image
Image(filename = "nn_img/Python_Pytorch_nn_Sequential_i2_o2_sigmoid_01.png", width=500)
Out[2]:
Prerequisites¶
In [3]:
import torch
from torch import nn
from torch import optim
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
#predictive
torch.manual_seed(1);
device = torch.device('cpu')
if torch.cuda.is_available():
device = torch.device('cuda')
#sigmoid explicit
def sigmoid(x):
return 1/(1+torch.exp(-x))
#sigmoid pre-defined
activation = torch.nn.Sigmoid()
# Loss (Binary Cross Entropy) error function, explicit def
def bce_err(output, target):
return -target * torch.log(output) - (1-target) * torch.log(1-output)
#sigmoid + BCELoss (Binary Cross Entropy)
criterion = torch.nn.BCEWithLogitsLoss()
def state(interval,i,loss,out,y):
if(i%interval == 0):
print("\n========== Epoch", i,"==========")
print("loss=",loss)
#.clone().detach()
#to numpy
out1 = out.data.numpy()
y3 = y.numpy()
#all values less than 0.5 to < 0
out2 = out1-0.5
# <0.5 turned to False, >= 0.5 turned to True
out3 = (out2>=0)
#boolean to decimal 0/1
out4=out3*1
#boolean results
rez = (out4 == y3)
#procent of True's(exact classification: y_hat=y, prediction = label/target)
accuracy = np.min(np.mean(rez,axis=0))
print("accuracy=",accuracy)
# quadrants
# 0,1 | 0,0
# --------------
# 1,1 | 1,0
#training points
data = np.array([
[0.1,0.1,0,0],
[0.2,0.5,0,0],
[-0.2,0.4,0,1],
[-0.8,0.2,0,1],
[-0.5,-0.1,1,1],
[-0.1,-0.24,1,1],
[0.2,-0.2,1,0],
])
#df = pd.read_csv('test.csv', header=None)
#df = pd.read_csv('data2.csv', header=None)
#data = df.to_numpy()
Explicit network definition¶
In [4]:
torch.manual_seed(1);
#features
x = torch.from_numpy(data[:, [0,1]]).double()
#target/labels
y = torch.from_numpy(data[:, [2,3]]).double()
# Define the size of each layer in our network
n_input = 2 # Number of input units, must match number of input features
n_hidden = 2 # Number of hidden units
n_output = 2 # Number of output units
# Weights for inputs to hidden layer
w = torch.randn(n_input, n_hidden, dtype=torch.double, requires_grad=True)
# and bias terms for hidden and output layers
b = torch.randn(1, n_hidden, dtype=torch.double, requires_grad=True)
out = torch.nn.Sigmoid()(torch.mm(x,w)+(b))
print(x)
print(w)
print(b)
print(out)
tensor([[ 0.1000, 0.1000],
[ 0.2000, 0.5000],
[-0.2000, 0.4000],
[-0.8000, 0.2000],
[-0.5000, -0.1000],
[-0.1000, -0.2400],
[ 0.2000, -0.2000]], dtype=torch.float64)
tensor([[0.6614, 0.2669],
[0.0617, 0.6213]], dtype=torch.float64, requires_grad=True)
tensor([[-0.4519, -0.1661]], dtype=torch.float64, requires_grad=True)
tensor([[0.4062, 0.4807],
[0.4283, 0.5493],
[0.3637, 0.5073],
[0.2751, 0.4365],
[0.3124, 0.4105],
[0.3699, 0.4153],
[0.4178, 0.4410]], dtype=torch.float64, grad_fn=<SigmoidBackward>)
In [ ]:
In [5]:
#learn rate
alpha = 0.1
#iterations
epochs = 700
#display state
fv = 50
lossHistory = []
for i in range(epochs):
#forward: output/prediction
out = torch.nn.Sigmoid()(torch.mm(x,w)+(b))
loss = torch.mean(bce_err(out,y))
#backward: compute gradients
loss.backward()
#update weights
with torch.no_grad():
w -= alpha * w.grad
b -= alpha * b.grad
# Manually zero the gradients after updating weights
w.grad.zero_()
b.grad.zero_()
lossHistory.append(loss)
state(fv,i,loss,out,y)
========== Epoch 0 ========== loss= tensor(0.7283, dtype=torch.float64, grad_fn=<MeanBackward0>) accuracy= 0.42857142857142855 ========== Epoch 50 ========== loss= tensor(0.6721, dtype=torch.float64, grad_fn=<MeanBackward0>) accuracy= 0.5714285714285714 ========== Epoch 100 ========== loss= tensor(0.6311, dtype=torch.float64, grad_fn=<MeanBackward0>) accuracy= 0.5714285714285714 ========== Epoch 150 ========== loss= tensor(0.5971, dtype=torch.float64, grad_fn=<MeanBackward0>) accuracy= 0.7142857142857143 ========== Epoch 200 ========== loss= tensor(0.5674, dtype=torch.float64, grad_fn=<MeanBackward0>) accuracy= 0.7142857142857143 ========== Epoch 250 ========== loss= tensor(0.5410, dtype=torch.float64, grad_fn=<MeanBackward0>) accuracy= 0.7142857142857143 ========== Epoch 300 ========== loss= tensor(0.5172, dtype=torch.float64, grad_fn=<MeanBackward0>) accuracy= 0.8571428571428571 ========== Epoch 350 ========== loss= tensor(0.4958, dtype=torch.float64, grad_fn=<MeanBackward0>) accuracy= 0.8571428571428571 ========== Epoch 400 ========== loss= tensor(0.4765, dtype=torch.float64, grad_fn=<MeanBackward0>) accuracy= 0.8571428571428571 ========== Epoch 450 ========== loss= tensor(0.4588, dtype=torch.float64, grad_fn=<MeanBackward0>) accuracy= 0.8571428571428571 ========== Epoch 500 ========== loss= tensor(0.4427, dtype=torch.float64, grad_fn=<MeanBackward0>) accuracy= 0.8571428571428571 ========== Epoch 550 ========== loss= tensor(0.4280, dtype=torch.float64, grad_fn=<MeanBackward0>) accuracy= 0.8571428571428571 ========== Epoch 600 ========== loss= tensor(0.4144, dtype=torch.float64, grad_fn=<MeanBackward0>) accuracy= 1.0 ========== Epoch 650 ========== loss= tensor(0.4019, dtype=torch.float64, grad_fn=<MeanBackward0>) accuracy= 1.0
In [6]:
#loss evolution
graph_x = np.arange(0, epochs)
graph_y = lossHistory
plt.plot(graph_x, graph_y)
plt.show()
In [ ]:
In [7]:
#final weights
print(w.data)
print(b.data)
tensor([[ 0.4308, -3.0967],
[-3.0529, -0.0708]], dtype=torch.float64)
tensor([[-0.0008, -0.0438]], dtype=torch.float64)
In [8]:
#new testing data
test_data = np.array([
[0.11,0.45,0,0],
[-0.0,0.25,0,1],
[-0.5,-0.6,1,1],
[0.91,-0.3,1,0],
])
x_test = torch.from_numpy(test_data[:, [0,1]]).double()
y_test = torch.from_numpy(test_data[:, [2,3]]).double()
#print(x_test)
test_out = torch.nn.Sigmoid()(torch.mm(x_test,w)+(b))
print(test_out)
tensor([[0.2096, 0.3974],
[0.3178, 0.4846],
[0.8342, 0.8245],
[0.7870, 0.0552]], dtype=torch.float64, grad_fn=<SigmoidBackward>)
In [ ]:
In [9]:
def accuracy(out,y):
#to numpy
out1 = out.data.numpy()
y3 = y.numpy()
#all values less than 0.5 to < 0
out2 = out1-0.5
# <0.5 turned to False, >= 0.5 turned to True
out3 = (out2>=0)
#boolean to decimal 0/1
out4=out3*1
#boolean results
rez = (out4 == y3)
print(rez)
#procent of True's(exact classification: y_hat=y, prediction = label/target)
accuracy = np.min(np.mean(rez,axis=0))
print("accuracy=",accuracy)
In [10]:
accuracy(test_out,y_test)
[[ True True] [ True False] [ True True] [ True True]] accuracy= 0.75
In [ ]:
Setting custom weights¶
In [15]:
w = torch.tensor( [[1, -1],
[1, -1]], dtype=torch.float64, requires_grad=True)
b = torch.tensor( [[1, -1]], dtype=torch.float64, requires_grad=True)
print(x)
print(w)
print(b)
out = torch.nn.Sigmoid()(torch.mm(x,w)+(b))
print("---\n",out,"\n---")
tensor([[ 0.1000, 0.1000],
[ 0.2000, 0.5000],
[-0.2000, 0.4000],
[-0.8000, 0.2000],
[-0.5000, -0.1000],
[-0.1000, -0.2400],
[ 0.2000, -0.2000]], dtype=torch.float64)
tensor([[ 1., -1.],
[ 1., -1.]], dtype=torch.float64, requires_grad=True)
tensor([[ 1., -1.]], dtype=torch.float64, requires_grad=True)
---
tensor([[0.7685, 0.2315],
[0.8455, 0.1545],
[0.7685, 0.2315],
[0.5987, 0.4013],
[0.5987, 0.4013],
[0.6593, 0.3407],
[0.7311, 0.2689]], dtype=torch.float64, grad_fn=<SigmoidBackward>)
---
In [16]:
w = torch.tensor( [[1, 1],
[1, 1]], dtype=torch.float64, requires_grad=True)
b = torch.tensor( [[1, 1]], dtype=torch.float64, requires_grad=True)
print(x)
print(w)
print(b)
out = torch.nn.Sigmoid()(torch.mm(x,w)+(b))
print("---\n",out,"\n---")
tensor([[ 0.1000, 0.1000],
[ 0.2000, 0.5000],
[-0.2000, 0.4000],
[-0.8000, 0.2000],
[-0.5000, -0.1000],
[-0.1000, -0.2400],
[ 0.2000, -0.2000]], dtype=torch.float64)
tensor([[1., 1.],
[1., 1.]], dtype=torch.float64, requires_grad=True)
tensor([[1., 1.]], dtype=torch.float64, requires_grad=True)
---
tensor([[0.7685, 0.7685],
[0.8455, 0.8455],
[0.7685, 0.7685],
[0.5987, 0.5987],
[0.5987, 0.5987],
[0.6593, 0.6593],
[0.7311, 0.7311]], dtype=torch.float64, grad_fn=<SigmoidBackward>)
---
In [ ]: