In [1]:
from IPython.display import Image
Image(filename = "nn_img/Python_Pytorch_nn_Sequential_i2_o2_sigmoid_01a.png", width=500)
Out[1]:
In [2]:
from IPython.display import Image
Image(filename = "nn_img/Python_Pytorch_nn_Sequential_i2_o2_sigmoid_01.png", width=500)
Out[2]:
Prerequisites¶
In [3]:
import torch
from torch import nn
from torch import optim
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
#predictive
torch.manual_seed(1);
device = torch.device('cpu')
if torch.cuda.is_available():
device = torch.device('cuda')
#sigmoid explicit
def sigmoid(x):
return 1/(1+torch.exp(-x))
#sigmoid pre-defined
activation = torch.nn.Sigmoid()
# Loss (Binary Cross Entropy) error function, explicit def
def bce_err(output, target):
return -target * torch.log(output) - (1-target) * torch.log(1-output)
#sigmoid + BCELoss (Binary Cross Entropy)
criterion = torch.nn.BCEWithLogitsLoss()
def state(interval,i,loss,out,y):
if(i%interval == 0):
print("\n========== Epoch", i,"==========")
print("loss=",loss)
#.clone().detach()
#to numpy
out1 = out.data.numpy()
y3 = y.numpy()
#all values less than 0.5 to < 0
out2 = out1-0.5
# <0.5 turned to False, >= 0.5 turned to True
out3 = (out2>=0)
#boolean to decimal 0/1
out4=out3*1
#boolean results
rez = (out4 == y3)
#procent of True's(exact classification: y_hat=y, prediction = label/target)
accuracy = np.min(np.mean(rez,axis=0))
print("accuracy=",accuracy)
# quadrants
# 0,1 | 0,0
# --------------
# 1,1 | 1,0
#training points
data = np.array([
[0.1,0.1,0,0],
[0.2,0.5,0,0],
[-0.2,0.4,0,1],
[-0.8,0.2,0,1],
[-0.5,-0.1,1,1],
[-0.1,-0.24,1,1],
[0.2,-0.2,1,0],
])
#df = pd.read_csv('test.csv', header=None)
#df = pd.read_csv('data2.csv', header=None)
#data = df.to_numpy()
Class + Linear network definition¶
In [4]:
# nn.Linear(input,neurons)
linear = nn.Linear(2, 2)
linear.weight
Out[4]:
Parameter containing:
tensor([[ 0.3643, -0.3121],
[-0.1371, 0.3319]], requires_grad=True)
In [5]:
torch.manual_seed(1);
#features
x = torch.from_numpy(data[:, [0,1]]).float()
#target/labels
y = torch.from_numpy(data[:, [2,3]]).float()
class Network(nn.Module):
def __init__(self):
super().__init__()
self.linear = nn.Linear(2,2)
self.activation = nn.Sigmoid()
def forward(self,x):
o = self.linear(x)
o = self.activation(o)
return o
net = Network()
print(net)
print(net.linear.weight.data)
print(net.linear.bias.data)
print(net.forward(x))
Network(
(linear): Linear(in_features=2, out_features=2, bias=True)
(activation): Sigmoid()
)
tensor([[ 0.3643, -0.3121],
[-0.1371, 0.3319]])
tensor([-0.6657, 0.4241])
tensor([[0.3406, 0.6091],
[0.3211, 0.6371],
[0.2966, 0.6420],
[0.2651, 0.6457],
[0.3065, 0.6129],
[0.3481, 0.5886],
[0.3704, 0.5818]], grad_fn=<SigmoidBackward>)
In [6]:
#learn rate
alpha = 0.1
#iterations
epochs = 700
#display state
fv = 50
#Stochastic/Batch gradient descent
optimizer = optim.SGD(net.parameters(), lr=alpha)
lossHistory = []
predictionHistory = []
#optional manually set weights: gives error, maybe a bug ?!
#net.linear.weight.data.fill_(1)
#net.linear.bias.data.fill_(1)
#net.linear.weight.data = torch.tensor([[1, -1]])
#net.linear.bias.data = torch.tensor([1])
for i in range(epochs):
#forward
out = net.forward(x)
#error function
loss = torch.mean(bce_err(out,y))
#loss = torch.mean(nn.BCELoss(out,y))
#loss = criterion(out, y)
lossHistory.append(loss)
optimizer.zero_grad()
#process gradients
loss.backward()
#update weights
optimizer.step()
state(fv,i,loss,out,y)
print("")
w = linear.weight.data.numpy()
b = net.linear.bias.data.numpy()
print("Final result:\n", w, b)
========== Epoch 0 ========== loss= tensor(0.6728, grad_fn=<MeanBackward0>) accuracy= 0.5714285714285714 ========== Epoch 50 ========== loss= tensor(0.6284, grad_fn=<MeanBackward0>) accuracy= 0.5714285714285714 ========== Epoch 100 ========== loss= tensor(0.5935, grad_fn=<MeanBackward0>) accuracy= 0.5714285714285714 ========== Epoch 150 ========== loss= tensor(0.5637, grad_fn=<MeanBackward0>) accuracy= 0.7142857142857143 ========== Epoch 200 ========== loss= tensor(0.5374, grad_fn=<MeanBackward0>) accuracy= 0.8571428571428571 ========== Epoch 250 ========== loss= tensor(0.5140, grad_fn=<MeanBackward0>) accuracy= 0.8571428571428571 ========== Epoch 300 ========== loss= tensor(0.4929, grad_fn=<MeanBackward0>) accuracy= 0.8571428571428571 ========== Epoch 350 ========== loss= tensor(0.4737, grad_fn=<MeanBackward0>) accuracy= 0.8571428571428571 ========== Epoch 400 ========== loss= tensor(0.4563, grad_fn=<MeanBackward0>) accuracy= 0.8571428571428571 ========== Epoch 450 ========== loss= tensor(0.4404, grad_fn=<MeanBackward0>) accuracy= 1.0 ========== Epoch 500 ========== loss= tensor(0.4259, grad_fn=<MeanBackward0>) accuracy= 1.0 ========== Epoch 550 ========== loss= tensor(0.4125, grad_fn=<MeanBackward0>) accuracy= 1.0 ========== Epoch 600 ========== loss= tensor(0.4001, grad_fn=<MeanBackward0>) accuracy= 1.0 ========== Epoch 650 ========== loss= tensor(0.3886, grad_fn=<MeanBackward0>) accuracy= 1.0 Final result: [[ 0.3643461 -0.31210154] [-0.13708077 0.33189395]] [-0.01134609 -0.03631671]
In [ ]:
In [7]:
#loss evolution
graph_x = np.arange(0, epochs)
graph_y = lossHistory
plt.plot(graph_x, graph_y)
plt.show()
In [ ]:
In [8]:
w = net.linear.weight.data.numpy()
b = net.linear.bias.data.numpy()
print(w)
print(b)
[[ 0.30137005 -3.2584748 ] [-3.2315242 -0.2577798 ]] [-0.01134609 -0.03631671]
In [ ]:
In [9]:
#new testing data
test_data = np.array([
[0.11,0.45,0,0],
[-0.0,0.25,0,1],
[-0.5,-0.6,1,1],
[0.91,-0.3,1,0],
])
x_test = torch.from_numpy(test_data[:, [0,1]]).float()
y_test = torch.from_numpy(test_data[:, [2,3]]).float()
#print(x_test)
test_out = net.forward(x_test)
print(test_out)
tensor([[0.1908, 0.3757],
[0.3045, 0.4748],
[0.8573, 0.8499],
[0.7756, 0.0522]], grad_fn=<SigmoidBackward>)
In [ ]:
In [10]:
def accuracy(out,y):
#to numpy
out1 = out.data.numpy()
y3 = y.numpy()
#all values less than 0.5 to < 0
out2 = out1-0.5
# <0.5 turned to False, >= 0.5 turned to True
out3 = (out2>=0)
#boolean to decimal 0/1
out4=out3*1
#boolean results
rez = (out4 == y3)
print(rez)
#procent of True's(exact classification: y_hat=y, prediction = label/target)
accuracy = np.min(np.mean(rez,axis=0))
print("accuracy=",accuracy)
In [11]:
accuracy(test_out,y_test)
[[ True True] [ True False] [ True True] [ True True]] accuracy= 0.75
In [ ]:
Setting custom weights¶
In [12]:
w = torch.tensor( [[1, -1],
[1, -1]], dtype=torch.float64)
b = torch.tensor( [[1, -1]], dtype=torch.float64)
net.linear.load_state_dict( {'weight': w.T, 'bias': b[0]})
print(net.state_dict())
print(x)
print(net.linear.weight)
print(net.linear.bias)
out = net(x)
print("---\n",out,"\n---")
OrderedDict([('linear.weight', tensor([[ 1., 1.],
[-1., -1.]])), ('linear.bias', tensor([ 1., -1.]))])
tensor([[ 0.1000, 0.1000],
[ 0.2000, 0.5000],
[-0.2000, 0.4000],
[-0.8000, 0.2000],
[-0.5000, -0.1000],
[-0.1000, -0.2400],
[ 0.2000, -0.2000]])
Parameter containing:
tensor([[ 1., 1.],
[-1., -1.]], requires_grad=True)
Parameter containing:
tensor([ 1., -1.], requires_grad=True)
---
tensor([[0.7685, 0.2315],
[0.8455, 0.1545],
[0.7685, 0.2315],
[0.5987, 0.4013],
[0.5987, 0.4013],
[0.6593, 0.3407],
[0.7311, 0.2689]], grad_fn=<SigmoidBackward>)
---
In [13]:
net.linear.weight.data.fill_(1)
net.linear.bias.data.fill_(1)
print(net.state_dict())
print(x)
print(net.linear.weight)
print(net.linear.bias)
out = net(x)
print("---\n",out,"\n---")
OrderedDict([('linear.weight', tensor([[1., 1.],
[1., 1.]])), ('linear.bias', tensor([1., 1.]))])
tensor([[ 0.1000, 0.1000],
[ 0.2000, 0.5000],
[-0.2000, 0.4000],
[-0.8000, 0.2000],
[-0.5000, -0.1000],
[-0.1000, -0.2400],
[ 0.2000, -0.2000]])
Parameter containing:
tensor([[1., 1.],
[1., 1.]], requires_grad=True)
Parameter containing:
tensor([1., 1.], requires_grad=True)
---
tensor([[0.7685, 0.7685],
[0.8455, 0.8455],
[0.7685, 0.7685],
[0.5987, 0.5987],
[0.5987, 0.5987],
[0.6593, 0.6593],
[0.7311, 0.7311]], grad_fn=<SigmoidBackward>)
---
In [ ]: