Deep neural network keras

In [23]:

import numpy as np
import matplotlib.pyplot as plt
# sklearn contains all of prepared dataset
from sklearn import datasets
# import keras to run on tf
import keras
# choose linear model
from keras.models import Sequential
# for hidden layers
from keras.layers import Dense
# gradient optimizer function
from keras.optimizers import Adam

import numpy as np import matplotlib.pyplot as plt # sklearn contains all of prepared dataset from sklearn import datasets # import keras to run on tf import keras # choose linear model from keras.models import Sequential # for hidden layers from keras.layers import Dense # gradient optimizer function from keras.optimizers import Adam

In [24]:

# seed points to have the same random number eachtime the program runs
np.random.seed(0)

# seed points to have the same random number eachtime the program runs np.random.seed(0)

In [25]:

# number of points
n_pts = 500
"""
# create circle dataset labels
# '0' large circle (-)
# '1' small circle (+)
make_circles() doc
- n_samples : numbers of sample points 
- random_state : random numbers to seed the numbers generator
- noise : gaussian noise : low value less convoluted shape and easy to classify
- factor : diameter of the small circle based on large circle diameter size (%)
"""
X, y = datasets.make_circles(n_samples=n_pts, random_state = 123, noise=0.1, factor=0.2)
# print(X) 
# print(y)

# plot all X coordinators with label of zero (y=0)
plt.scatter(X[y==0, 0], X[y==0, 1])
# plot all X coordinators with label of one (y=1)
plt.scatter(X[y==1, 0], X[y==1, 1])

# number of points n_pts = 500 """ # create circle dataset labels # '0' large circle (-) # '1' small circle (+) make_circles() doc - n_samples : numbers of sample points - random_state : random numbers to seed the numbers generator - noise : gaussian noise : low value less convoluted shape and easy to classify - factor : diameter of the small circle based on large circle diameter size (%) """ X, y = datasets.make_circles(n_samples=n_pts, random_state = 123, noise=0.1, factor=0.2) # print(X) # print(y) # plot all X coordinators with label of zero (y=0) plt.scatter(X[y==0, 0], X[y==0, 1]) # plot all X coordinators with label of one (y=1) plt.scatter(X[y==1, 0], X[y==1, 1])

Out[25]:

<matplotlib.collections.PathCollection at 0x7fa86a361190>

In [26]:

# create linear/sequential  DNN with keras
model = Sequential()
# four hidden layers; 2 input nodes: X1,X2 ; with sigmoid activation function 
model.add(Dense(4, input_shape=(2,), activation='sigmoid'))
# output layers with sigmoid activation function
model.add(Dense(1, activation='sigmoid'))
# configure the model with learning rate : crossyentropy : metrics "similar to loss function" (accuracy)
model.compile(Adam(lr = 0.01), 'binary_crossentropy', metrics=['accuracy'])

# https://playground.tensorflow.org/#activation=sigmoid&batchSize=10&dataset=circle&regDataset=reg-plane&learningRate=0.03&regularizationRate=0&noise=0&networkShape=4,1&seed=0.87706&showTestData=false&discretize=false&percTrainData=50&x=true&y=true&xTimesY=false&xSquared=false&ySquared=false&cosX=false&sinX=false&cosY=false&sinY=false&collectStats=false&problem=classification&initZero=false&hideText=false

# create linear/sequential DNN with keras model = Sequential() # four hidden layers; 2 input nodes: X1,X2 ; with sigmoid activation function model.add(Dense(4, input_shape=(2,), activation='sigmoid')) # output layers with sigmoid activation function model.add(Dense(1, activation='sigmoid')) # configure the model with learning rate : crossyentropy : metrics "similar to loss function" (accuracy) model.compile(Adam(lr = 0.01), 'binary_crossentropy', metrics=['accuracy']) # https://playground.tensorflow.org/#activation=sigmoid&batchSize=10&dataset=circle&regDataset=reg-plane&learningRate=0.03&regularizationRate=0&noise=0&networkShape=4,1&seed=0.87706&showTestData=false&discretize=false&percTrainData=50&x=true&y=true&xTimesY=false&xSquared=false&ySquared=false&cosX=false&sinX=false&cosY=false&sinY=false&collectStats=false&problem=classification&initZero=false&hideText=false

In [27]:

# train the model : Trains the model for a fixed number of epochs (iterations on a dataset).

h = model.fit(x=X, y=y, verbose=1, batch_size=20, epochs=100, shuffle='true')

# train the model : Trains the model for a fixed number of epochs (iterations on a dataset). h = model.fit(x=X, y=y, verbose=1, batch_size=20, epochs=100, shuffle='true')

Epoch 1/100
25/25 [==============================] - 0s 2ms/step - loss: 0.6948 - accuracy: 0.3342
Epoch 2/100
25/25 [==============================] - 0s 2ms/step - loss: 0.6943 - accuracy: 0.6304
Epoch 3/100
25/25 [==============================] - 0s 2ms/step - loss: 0.6923 - accuracy: 0.5714
Epoch 4/100
25/25 [==============================] - 0s 2ms/step - loss: 0.6935 - accuracy: 0.4365
Epoch 5/100
25/25 [==============================] - 0s 1ms/step - loss: 0.6909 - accuracy: 0.6780
Epoch 6/100
25/25 [==============================] - 0s 1ms/step - loss: 0.6905 - accuracy: 0.5595
Epoch 7/100
25/25 [==============================] - 0s 1ms/step - loss: 0.6900 - accuracy: 0.6791
Epoch 8/100
25/25 [==============================] - 0s 2ms/step - loss: 0.6897 - accuracy: 0.5512
Epoch 9/100
25/25 [==============================] - 0s 1ms/step - loss: 0.6862 - accuracy: 0.6709
Epoch 10/100
25/25 [==============================] - 0s 1ms/step - loss: 0.6852 - accuracy: 0.7630
Epoch 11/100
25/25 [==============================] - 0s 2ms/step - loss: 0.6816 - accuracy: 0.6887
Epoch 12/100
25/25 [==============================] - 0s 2ms/step - loss: 0.6778 - accuracy: 0.6873
Epoch 13/100
25/25 [==============================] - 0s 2ms/step - loss: 0.6715 - accuracy: 0.8132
Epoch 14/100
25/25 [==============================] - 0s 2ms/step - loss: 0.6618 - accuracy: 0.8229
Epoch 15/100
25/25 [==============================] - 0s 2ms/step - loss: 0.6545 - accuracy: 0.8025
Epoch 16/100
25/25 [==============================] - 0s 2ms/step - loss: 0.6421 - accuracy: 0.8315
Epoch 17/100
25/25 [==============================] - 0s 2ms/step - loss: 0.6249 - accuracy: 0.8502
Epoch 18/100
25/25 [==============================] - 0s 2ms/step - loss: 0.6071 - accuracy: 0.8551
Epoch 19/100
25/25 [==============================] - 0s 2ms/step - loss: 0.5844 - accuracy: 0.8603
Epoch 20/100
25/25 [==============================] - 0s 1ms/step - loss: 0.5583 - accuracy: 0.9130
Epoch 21/100
25/25 [==============================] - 0s 1ms/step - loss: 0.5280 - accuracy: 0.9221
Epoch 22/100
25/25 [==============================] - 0s 1ms/step - loss: 0.5115 - accuracy: 0.9176
Epoch 23/100
25/25 [==============================] - 0s 2ms/step - loss: 0.4750 - accuracy: 0.9385
Epoch 24/100
25/25 [==============================] - 0s 2ms/step - loss: 0.4464 - accuracy: 0.9434
Epoch 25/100
25/25 [==============================] - 0s 2ms/step - loss: 0.4225 - accuracy: 0.9594
Epoch 26/100
25/25 [==============================] - 0s 2ms/step - loss: 0.3994 - accuracy: 0.9551
Epoch 27/100
25/25 [==============================] - 0s 2ms/step - loss: 0.3831 - accuracy: 0.9505
Epoch 28/100
25/25 [==============================] - 0s 2ms/step - loss: 0.3535 - accuracy: 0.9686
Epoch 29/100
25/25 [==============================] - 0s 2ms/step - loss: 0.3403 - accuracy: 0.9763
Epoch 30/100
25/25 [==============================] - 0s 2ms/step - loss: 0.3156 - accuracy: 0.9837
Epoch 31/100
25/25 [==============================] - 0s 2ms/step - loss: 0.2892 - accuracy: 0.9921
Epoch 32/100
25/25 [==============================] - 0s 1ms/step - loss: 0.2710 - accuracy: 0.9931
Epoch 33/100
25/25 [==============================] - 0s 2ms/step - loss: 0.2502 - accuracy: 0.9958
Epoch 34/100
25/25 [==============================] - 0s 1ms/step - loss: 0.2374 - accuracy: 1.0000
Epoch 35/100
25/25 [==============================] - 0s 2ms/step - loss: 0.2181 - accuracy: 0.9996
Epoch 36/100
25/25 [==============================] - 0s 2ms/step - loss: 0.2034 - accuracy: 1.0000
Epoch 37/100
25/25 [==============================] - 0s 2ms/step - loss: 0.1927 - accuracy: 0.9995
Epoch 38/100
25/25 [==============================] - 0s 2ms/step - loss: 0.1817 - accuracy: 1.0000
Epoch 39/100
25/25 [==============================] - 0s 2ms/step - loss: 0.1743 - accuracy: 0.9982
Epoch 40/100
25/25 [==============================] - 0s 2ms/step - loss: 0.1556 - accuracy: 1.0000
Epoch 41/100
25/25 [==============================] - 0s 2ms/step - loss: 0.1505 - accuracy: 1.0000
Epoch 42/100
25/25 [==============================] - 0s 2ms/step - loss: 0.1413 - accuracy: 0.9995
Epoch 43/100
25/25 [==============================] - 0s 2ms/step - loss: 0.1336 - accuracy: 0.9945
Epoch 44/100
25/25 [==============================] - 0s 2ms/step - loss: 0.1333 - accuracy: 1.0000
Epoch 45/100
25/25 [==============================] - 0s 1ms/step - loss: 0.1246 - accuracy: 0.9990
Epoch 46/100
25/25 [==============================] - 0s 2ms/step - loss: 0.1148 - accuracy: 1.0000
Epoch 47/100
25/25 [==============================] - 0s 2ms/step - loss: 0.1121 - accuracy: 1.0000
Epoch 48/100
25/25 [==============================] - 0s 2ms/step - loss: 0.1096 - accuracy: 0.9986
Epoch 49/100
25/25 [==============================] - 0s 2ms/step - loss: 0.0991 - accuracy: 1.0000
Epoch 50/100
25/25 [==============================] - 0s 2ms/step - loss: 0.0978 - accuracy: 1.0000
Epoch 51/100
25/25 [==============================] - 0s 2ms/step - loss: 0.0902 - accuracy: 0.9984
Epoch 52/100
25/25 [==============================] - 0s 2ms/step - loss: 0.0877 - accuracy: 0.9984
Epoch 53/100
25/25 [==============================] - 0s 2ms/step - loss: 0.0876 - accuracy: 1.0000
Epoch 54/100
25/25 [==============================] - 0s 2ms/step - loss: 0.0797 - accuracy: 1.0000
Epoch 55/100
25/25 [==============================] - 0s 2ms/step - loss: 0.0809 - accuracy: 0.9973
Epoch 56/100
25/25 [==============================] - 0s 2ms/step - loss: 0.0720 - accuracy: 0.9987
Epoch 57/100
25/25 [==============================] - 0s 2ms/step - loss: 0.0733 - accuracy: 1.0000
Epoch 58/100
25/25 [==============================] - 0s 2ms/step - loss: 0.0675 - accuracy: 1.0000
Epoch 59/100
25/25 [==============================] - 0s 2ms/step - loss: 0.0669 - accuracy: 1.0000
Epoch 60/100
25/25 [==============================] - 0s 2ms/step - loss: 0.0635 - accuracy: 1.0000
Epoch 61/100
25/25 [==============================] - 0s 1ms/step - loss: 0.0620 - accuracy: 1.0000
Epoch 62/100
25/25 [==============================] - 0s 2ms/step - loss: 0.0590 - accuracy: 1.0000
Epoch 63/100
25/25 [==============================] - 0s 2ms/step - loss: 0.0564 - accuracy: 0.9986
Epoch 64/100
25/25 [==============================] - 0s 2ms/step - loss: 0.0552 - accuracy: 1.0000
Epoch 65/100
25/25 [==============================] - 0s 2ms/step - loss: 0.0577 - accuracy: 1.0000
Epoch 66/100
25/25 [==============================] - 0s 2ms/step - loss: 0.0496 - accuracy: 1.0000
Epoch 67/100
25/25 [==============================] - 0s 2ms/step - loss: 0.0495 - accuracy: 1.0000
Epoch 68/100
25/25 [==============================] - 0s 2ms/step - loss: 0.0482 - accuracy: 1.0000
Epoch 69/100
25/25 [==============================] - 0s 2ms/step - loss: 0.0482 - accuracy: 1.0000
Epoch 70/100
25/25 [==============================] - 0s 2ms/step - loss: 0.0456 - accuracy: 1.0000
Epoch 71/100
25/25 [==============================] - 0s 2ms/step - loss: 0.0435 - accuracy: 1.0000
Epoch 72/100
25/25 [==============================] - 0s 2ms/step - loss: 0.0428 - accuracy: 1.0000
Epoch 73/100
25/25 [==============================] - 0s 2ms/step - loss: 0.0411 - accuracy: 1.0000
Epoch 74/100
25/25 [==============================] - 0s 2ms/step - loss: 0.0417 - accuracy: 1.0000
Epoch 75/100
25/25 [==============================] - 0s 2ms/step - loss: 0.0409 - accuracy: 1.0000
Epoch 76/100
25/25 [==============================] - 0s 2ms/step - loss: 0.0368 - accuracy: 1.0000
Epoch 77/100
25/25 [==============================] - 0s 1ms/step - loss: 0.0367 - accuracy: 1.0000
Epoch 78/100
25/25 [==============================] - 0s 2ms/step - loss: 0.0380 - accuracy: 1.0000
Epoch 79/100
25/25 [==============================] - 0s 2ms/step - loss: 0.0359 - accuracy: 1.0000
Epoch 80/100
25/25 [==============================] - 0s 2ms/step - loss: 0.0364 - accuracy: 1.0000
Epoch 81/100
25/25 [==============================] - 0s 2ms/step - loss: 0.0345 - accuracy: 1.0000
Epoch 82/100
25/25 [==============================] - 0s 2ms/step - loss: 0.0322 - accuracy: 1.0000
Epoch 83/100
25/25 [==============================] - 0s 2ms/step - loss: 0.0315 - accuracy: 1.0000
Epoch 84/100
25/25 [==============================] - 0s 1ms/step - loss: 0.0333 - accuracy: 1.0000
Epoch 85/100
25/25 [==============================] - 0s 2ms/step - loss: 0.0321 - accuracy: 1.0000
Epoch 86/100
25/25 [==============================] - 0s 2ms/step - loss: 0.0267 - accuracy: 1.0000
Epoch 87/100
25/25 [==============================] - 0s 2ms/step - loss: 0.0285 - accuracy: 1.0000
Epoch 88/100
25/25 [==============================] - 0s 2ms/step - loss: 0.0282 - accuracy: 1.0000
Epoch 89/100
25/25 [==============================] - 0s 2ms/step - loss: 0.0270 - accuracy: 1.0000
Epoch 90/100
25/25 [==============================] - 0s 1ms/step - loss: 0.0262 - accuracy: 1.0000
Epoch 91/100
25/25 [==============================] - 0s 2ms/step - loss: 0.0267 - accuracy: 1.0000
Epoch 92/100
25/25 [==============================] - 0s 1ms/step - loss: 0.0260 - accuracy: 1.0000
Epoch 93/100
25/25 [==============================] - 0s 2ms/step - loss: 0.0271 - accuracy: 1.0000
Epoch 94/100
25/25 [==============================] - 0s 1ms/step - loss: 0.0234 - accuracy: 1.0000
Epoch 95/100
25/25 [==============================] - 0s 1ms/step - loss: 0.0228 - accuracy: 1.0000
Epoch 96/100
25/25 [==============================] - 0s 2ms/step - loss: 0.0232 - accuracy: 1.0000
Epoch 97/100
25/25 [==============================] - 0s 2ms/step - loss: 0.0232 - accuracy: 1.0000
Epoch 98/100
25/25 [==============================] - 0s 2ms/step - loss: 0.0220 - accuracy: 1.0000
Epoch 99/100
25/25 [==============================] - 0s 2ms/step - loss: 0.0220 - accuracy: 1.0000
Epoch 100/100
25/25 [==============================] - 0s 2ms/step - loss: 0.0222 - accuracy: 1.0000

In [28]:

plt.plot(h.history['accuracy'])
plt.legend(['accuracy'])
plt.ylabel('accuracy')
plt.xlabel('epoch')
 
plt.plot(h.history['loss'])
plt.legend(['loss'])
plt.title('loss')
plt.xlabel('epoch')

plt.plot(h.history['accuracy']) plt.legend(['accuracy']) plt.ylabel('accuracy') plt.xlabel('epoch') plt.plot(h.history['loss']) plt.legend(['loss']) plt.title('loss') plt.xlabel('epoch')

Out[28]:

Text(0.5, 0, 'epoch')

In [29]:

# show the final probability zone based on model prediction during each training
def plot_decision_boundary(X, y, model):
  """
  - X : input nodes
  - y : probabilities output
  - model : pretrained model
  """
  x_span = np.linspace(min(X[:,0]) - 0.25, max(X[:,0]) + 0.25, 50)
  y_span = np.linspace(min(X[:,1]) - 0.25, max(X[:,1]) + 0.25, 50)
  xx, yy = np.meshgrid(x_span, y_span)
  grid = np.c_[xx.ravel(), yy.ravel()]
  pred_func = model.predict(grid)
  # display distinct zone prediction probability level
  z = pred_func.reshape(xx.shape)
  plt.contourf(xx, yy, z)

# show the final probability zone based on model prediction during each training def plot_decision_boundary(X, y, model): """ - X : input nodes - y : probabilities output - model : pretrained model """ x_span = np.linspace(min(X[:,0]) - 0.25, max(X[:,0]) + 0.25, 50) y_span = np.linspace(min(X[:,1]) - 0.25, max(X[:,1]) + 0.25, 50) xx, yy = np.meshgrid(x_span, y_span) grid = np.c_[xx.ravel(), yy.ravel()] pred_func = model.predict(grid) # display distinct zone prediction probability level z = pred_func.reshape(xx.shape) plt.contourf(xx, yy, z)

In [30]:

plot_decision_boundary(X, y, model)
plt.scatter(X[y==0, 0], X[y==0, 1])
plt.scatter(X[y==1, 0], X[y==1, 1])

# display a new predicted point
plot_decision_boundary(X, y, model)
plt.scatter(X[y==0, 0], X[y==0, 1])
plt.scatter(X[y==1, 0], X[y==1, 1])

# point coordinators
x = 0.1
y = 0.75

# give the new point to the trained model
point = np.array([[x, y]])
predict = model.predict(point)
plt.plot([x], [y], marker='o', markersize=10, color="red")
print("Prediction is: ", predict)

# every point outside the zone has the proba = 0
# every point inside the classfied zone the de model are = 1
# so the new point is outside the classification ZONE!!! which show that the model is working great!!!

plot_decision_boundary(X, y, model) plt.scatter(X[y==0, 0], X[y==0, 1]) plt.scatter(X[y==1, 0], X[y==1, 1]) # display a new predicted point plot_decision_boundary(X, y, model) plt.scatter(X[y==0, 0], X[y==0, 1]) plt.scatter(X[y==1, 0], X[y==1, 1]) # point coordinators x = 0.1 y = 0.75 # give the new point to the trained model point = np.array([[x, y]]) predict = model.predict(point) plt.plot([x], [y], marker='o', markersize=10, color="red") print("Prediction is: ", predict) # every point outside the zone has the proba = 0 # every point inside the classfied zone the de model are = 1 # so the new point is outside the classification ZONE!!! which show that the model is working great!!!

Prediction is:  [[0.02737104]]

In [30]: