Browse Source

Modified ex3 so it could be imported as module. Minor changes to ex3. Finished ex3_nn.

master
wchen342 6 years ago
parent
commit
8d2257af78
  1. 86
      ex3/ex3.py
  2. 96
      ex3/ex3_nn.py

86
ex3/ex3.py

@ -55,6 +55,7 @@ def displaydata(X, example_width=None):
# Do not show axis
plt.axis('off')
plt.pause(0.0001)
# SIGMOID Compute sigmoid function
@ -70,19 +71,20 @@ def sigmoid(z):
# theta as the parameter for regularized logistic regression and the
# gradient of the cost w.r.t. to the parameters.
def lr_cost_function(theta, X, y, ld):
# Shape variables
m = y.shape[0]
d1 = X.shape[1]
d2 = y.shape[1]
theta = theta.reshape((d1, d2))
h = sigmoid(X.dot(theta))
if y.dtype == bool:
y = y.astype(int)
h = sigmoid(X.dot(theta))
theta[0, :] = 0
J = np.sum(-y * np.log(h) - (1 - y) * np.log(1 - h)) / m + ld / m / 2 * np.sum(theta ** 2)
grad = X.T.dot((h - y)) / m + ld / m * theta
print(J)
return J, grad.flatten()
@ -100,7 +102,7 @@ def one_vs_all(X, y, num_labels, ld):
X = np.hstack((np.ones((m, 1)), X))
initial_theta = np.zeros((n + 1, num_labels)) + 0.01
ret = scipy.optimize.minimize(lr_cost_function, initial_theta, args=(X, y_map, ld), jac=True,
options={'maxiter': 50, 'disp': True}, method='CG')
options={'maxiter': 500, 'disp': True}, method='CG') # BFGS is too slow here
return ret['x'].reshape((n + 1, num_labels))
@ -113,7 +115,7 @@ def one_vs_all2(X, y, num_labels, ld):
for i in range(0, num_labels):
initial_theta = np.zeros((n + 1, 1))
ret = scipy.optimize.minimize(lr_cost_function, initial_theta, args=(X, (y == i + 1), ld), jac=True,
options={'maxiter': 1000, 'disp': True}, method='CG')
options={'maxiter': 50, 'disp': True}, method='CG') # BFGS is too slow here
all_theta[:, i:i + 1] = ret['x'].reshape(-1, 1)
return all_theta
@ -138,52 +140,54 @@ def predict_one_vs_all(all_theta, X):
return p
np.set_printoptions(formatter={'float': '{: 0.5f}'.format}, edgeitems=20, linewidth=150)
## Setup the parameters you will use for this part of the exercise
input_layer_size = 400 # 20x20 Input Images of Digits
num_labels = 10 # 10 labels, from 1 to 10
# (note that we have mapped "0" to label 10)
## ===================== Start Main ======================
if __name__ == "__main__":
np.set_printoptions(formatter={'float': '{: 0.5f}'.format}, edgeitems=20, linewidth=150)
## Setup the parameters you will use for this part of the exercise
input_layer_size = 400 # 20x20 Input Images of Digits
num_labels = 10 # 10 labels, from 1 to 10
# (note that we have mapped "0" to label 10)
## =========== Part 1: Loading and Visualizing Data =============
# We start the exercise by first loading and visualizing the dataset.
# You will be working with a dataset that contains handwritten digits.
## =========== Part 1: Loading and Visualizing Data =============
# We start the exercise by first loading and visualizing the dataset.
# You will be working with a dataset that contains handwritten digits.
# Load Training Data
print('Loading and Visualizing Data ...\n')
# Load Training Data
print('Loading and Visualizing Data ...\n')
data = scipy.io.loadmat('mat/ex3data1.mat', matlab_compatible=True) # training data stored in arrays X, y
X = data['X']
y = data['y']
m = X.shape[0]
data = scipy.io.loadmat('mat/ex3data1.mat', matlab_compatible=True) # training data stored in arrays X, y
X = data['X']
y = data['y']
m = X.shape[0]
# Randomly select 100 data points to display
rand_indices = np.random.permutation(m)
sel = X[rand_indices[:100], :]
# Randomly select 100 data points to display
rand_indices = np.random.permutation(m)
sel = X[rand_indices[:100], :]
displaydata(sel)
plt.show()
displaydata(sel)
plt.show()
print('Program paused. Press enter to continue.\n')
input()
print('Program paused. Press enter to continue.\n')
input()
## ============ Part 2: Vectorize Logistic Regression ============
# In this part of the exercise, you will reuse your logistic regression
# code from the last exercise. You task here is to make sure that your
# regularized logistic regression implementation is vectorized. After
# that, you will implement one-vs-all classification for the handwritten
# digit dataset.
## ============ Part 2: Vectorize Logistic Regression ============
# In this part of the exercise, you will reuse your logistic regression
# code from the last exercise. You task here is to make sure that your
# regularized logistic regression implementation is vectorized. After
# that, you will implement one-vs-all classification for the handwritten
# digit dataset.
print('\nTraining One-vs-All Logistic Regression...\n')
print('\nTraining One-vs-All Logistic Regression...\n')
ld = 0.1
ld = 0.1
all_theta = one_vs_all(X, y, num_labels, ld)
all_theta = one_vs_all(X, y, num_labels, ld)
print('Program paused. Press enter to continue.\n')
input()
print('Program paused. Press enter to continue.\n')
input()
## ================ Part 3: Predict for One-Vs-All ================
# After ...
pred = predict_one_vs_all(all_theta, X)
## ================ Part 3: Predict for One-Vs-All ================
# After ...
pred = predict_one_vs_all(all_theta, X)
#
print('\nTraining Set Accuracy: {0:f}\n'.format(np.mean(pred == y.flatten()) * 100))
print('\nTraining Set Accuracy: {0:f}\n'.format(np.mean(pred == y.flatten()) * 100))

96
ex3/ex3_nn.py

@ -0,0 +1,96 @@
## Machine Learning Online Class - Exercise 3 | Part 2: Neural Networks
import numpy as np
import matplotlib.pyplot as plt
import scipy, scipy.io, scipy.optimize
import math
from ex3 import displaydata, sigmoid
# PREDICT Predict the label of an input given a trained neural network
# p = PREDICT(Theta1, Theta2, X) outputs the predicted label of X given the
# trained weights of a neural network (Theta1, Theta2)
def predict(Theta1, Theta2, X):
# Useful values
m = X.shape[0]
num_labels = Theta2.shape[0]
# Add ones to the X data matrix
X = np.hstack((np.ones((m, 1)), X))
# First hidden layer
temp = sigmoid(X.dot(Theta1.T))
m2 = temp.shape[0]
# Add ones to hidden layer units
temp = np.hstack((np.ones((m2, 1)), temp))
# Output layer
temp = sigmoid(temp.dot(Theta2.T))
p = temp.argmax(axis=1) + 1
return p
## Setup the parameters you will use for this exercise
input_layer_size = 400 # 20x20 Input Images of Digits
hidden_layer_size = 25 # 25 hidden units
num_labels = 10 # 10 labels, from 1 to 10
## =========== Part 1: Loading and Visualizing Data =============
# Load Training Data
print('Loading and Visualizing Data ...\n')
data = scipy.io.loadmat('mat/ex3data1.mat', matlab_compatible=True) # training data stored in arrays X, y
X = data['X']
y = data['y']
m = X.shape[0]
# Randomly select 100 data points to display
sel = np.random.permutation(m)
sel = X[sel[:100], :]
displaydata(sel)
plt.show()
print('Program paused. Press enter to continue.\n')
input()
## ================ Part 2: Loading Pameters ================
print('\nLoading Saved Neural Network Parameters ...\n')
# Load the weights into variables Theta1 and Theta2
data = scipy.io.loadmat('mat/ex3weights.mat', matlab_compatible=True)
Theta1 = data['Theta1']
Theta2 = data['Theta2']
## ================= Part 3: Implement Predict =================
pred = predict(Theta1, Theta2, X)
print('\nTraining Set Accuracy: {0:f}\n'.format(np.mean(pred == y.flatten()) * 100))
print('Program paused. Press enter to continue.\n')
input()
# To give you an idea of the network's output, you can also run
# through the examples one at the a time to see what it is predicting.
# Randomly permute examples
rp = np.random.permutation(m)
for i in range(0, m):
ind = rp[i] # Chosen index in X
# Display
print('\nDisplaying Example Image\n')
displaydata(X[ind:ind+1, :])
plt.show(block=False)
pred = predict(Theta1, Theta2, X[ind:ind+1,:])
print('\nNeural Network Prediction: {0:d} (digit {1:d})\n'.format(pred[0], np.mod(pred, 10)[0]))
# Pause
print('Program paused. Press enter to continue.\n')
input()
Loading…
Cancel
Save