|
|
@ -1,9 +1,17 @@ |
|
|
|
## Machine Learning Online Class |
|
|
|
# Exercise 6 | Support Vector Machines |
|
|
|
import numpy as np |
|
|
|
import scipy.io |
|
|
|
import matplotlib.pyplot as plt |
|
|
|
import sys |
|
|
|
from sklearn.svm import SVC |
|
|
|
|
|
|
|
|
|
|
|
# PLOTDATA Plots the data points X and y into a new figure |
|
|
|
# PLOTDATA(x,y) plots the data points with + for the positive examples |
|
|
|
# and o for the negative examples. X is assumed to be a Mx2 matrix. |
|
|
|
# |
|
|
|
# Note: This was slightly modified such that it expects y = 1 or y = 0 |
|
|
|
def plotdata(X, y): |
|
|
|
pos = y == 1 |
|
|
|
neg = y == 0 |
|
|
@ -11,6 +19,10 @@ def plotdata(X, y): |
|
|
|
plt.plot(X[:, :1][neg], X[:, 1:2][neg], 'ko', markerfacecolor='y', markersize=7) |
|
|
|
|
|
|
|
|
|
|
|
# VISUALIZEBOUNDARYLINEAR plots a linear decision boundary learned by the |
|
|
|
# SVM |
|
|
|
# VISUALIZEBOUNDARYLINEAR(X, y, model) plots a linear decision boundary |
|
|
|
# learned by the SVM and overlays the data on it |
|
|
|
def visualize_boundary_linear(X, y, clf): |
|
|
|
w = clf.coef_ |
|
|
|
b = clf.intercept_ |
|
|
@ -20,33 +32,70 @@ def visualize_boundary_linear(X, y, clf): |
|
|
|
plt.plot(xp, yp, '-b') |
|
|
|
|
|
|
|
|
|
|
|
# VISUALIZEBOUNDARY plots a non-linear decision boundary learned by the SVM |
|
|
|
# VISUALIZEBOUNDARYLINEAR(X, y, model) plots a non-linear decision |
|
|
|
# boundary learned by the SVM and overlays the data on it |
|
|
|
def visualize_boundary(X, y, clf): |
|
|
|
plotdata(X, y) |
|
|
|
x1plot = np.linspace(X[:, :1].min(), X[:, :1].max(), 100).T |
|
|
|
x2plot = np.linspace(X[:, 1:2].min(), X[:, 1:2].max(), 100).T |
|
|
|
X1, X2 = np.meshgrid(x1plot, x2plot) |
|
|
|
vals = np.zeros(X1.shape) |
|
|
|
for i in range(1, X1.shape[1]+1): |
|
|
|
this_X = np.hstack((X1[:, i-1:i], X2[:, i-1:i])) |
|
|
|
vals[:, i-1] = clf.predict(this_X) |
|
|
|
|
|
|
|
for i in range(1, X1.shape[1] + 1): |
|
|
|
this_X = np.hstack((X1[:, i - 1:i], X2[:, i - 1:i])) |
|
|
|
vals[:, i - 1] = clf.predict(this_X) |
|
|
|
plt.contour(X1, X2, vals, [0], colors='b') |
|
|
|
# plt.contour(X1, X2, vals, colors='b') |
|
|
|
|
|
|
|
|
|
|
|
# EX6PARAMS returns your choice of C and sigma for Part 3 of the exercise |
|
|
|
# where you select the optimal (C, sigma) learning parameters to use for SVM |
|
|
|
# with RBF kernel |
|
|
|
# C, sigma = EX6PARAMS(X, y, Xval, yval) returns your choice of C and |
|
|
|
# sigma. You should complete this function to return the optimal C and |
|
|
|
# sigma based on a cross-validation set. |
|
|
|
def dataset3_params(X, y, Xval, yval): |
|
|
|
# You need to return the following variables correctly. |
|
|
|
C = [0.01, 0.03, 0.1, 0.3, 1, 3, 10, 30] |
|
|
|
sigma = [0.01, 0.03, 0.1, 0.3, 1, 3, 10, 30] |
|
|
|
|
|
|
|
minError = sys.maxsize |
|
|
|
finalC = 0 |
|
|
|
finalSigma = 0 |
|
|
|
|
|
|
|
clf = SVC(kernel='rbf') |
|
|
|
|
|
|
|
for i in C: |
|
|
|
for j in sigma: |
|
|
|
clf = clf.set_params(C=i, gamma=1 / (2 * j * j)) |
|
|
|
clf.fit(X, y.ravel()) |
|
|
|
predictions = clf.predict(Xval) |
|
|
|
error = np.mean(predictions.reshape(-1, 1) != yval) |
|
|
|
if error <= minError: |
|
|
|
minError = error |
|
|
|
finalC = i |
|
|
|
finalSigma = j |
|
|
|
return finalC, finalSigma |
|
|
|
|
|
|
|
|
|
|
|
## =============== Part 1: Loading and Visualizing Data ================ |
|
|
|
print('Loading and Visualizing Data ...\n') |
|
|
|
|
|
|
|
# Load from ex6data1: |
|
|
|
# You will have X, y in your environment |
|
|
|
data = scipy.io.loadmat('mat/ex6data1.mat', matlab_compatible=True) |
|
|
|
|
|
|
|
X = data['X'] |
|
|
|
y = data['y'] |
|
|
|
|
|
|
|
# Plot training data |
|
|
|
plotdata(X, y) |
|
|
|
plt.show() |
|
|
|
|
|
|
|
print('Program paused. Press enter to continue.\n') |
|
|
|
input() |
|
|
|
|
|
|
|
## ==================== Part 2: Training Linear SVM ==================== |
|
|
|
print('\nTraining Linear SVM ...\n') |
|
|
|
|
|
|
|
C = 1 |
|
|
|
clf = SVC(C=C, kernel='linear') |
|
|
|
|
|
|
@ -59,22 +108,30 @@ input() |
|
|
|
|
|
|
|
print('Loading and Visualizing Data ...\n') |
|
|
|
|
|
|
|
## =============== Part 4: Visualizing Dataset 2 ================ |
|
|
|
# Load from ex6data2: |
|
|
|
# You will have X, y in your environment |
|
|
|
data = scipy.io.loadmat('mat/ex6data2.mat', matlab_compatible=True) |
|
|
|
|
|
|
|
X = data['X'] |
|
|
|
y = data['y'] |
|
|
|
|
|
|
|
# Plot training data |
|
|
|
plotdata(X, y) |
|
|
|
plt.show() |
|
|
|
|
|
|
|
print('Program paused. Press enter to continue.\n') |
|
|
|
input() |
|
|
|
|
|
|
|
## ========== Part 5: Training SVM with RBF Kernel (Dataset 2) ========== |
|
|
|
print('\nTraining SVM with RBF Kernel (this may take 1 to 2 minutes) ...\n') |
|
|
|
|
|
|
|
# SVM Parameters |
|
|
|
C = 1 |
|
|
|
sigma = 0.1 |
|
|
|
gamma = 1 / (2 * sigma * sigma) |
|
|
|
|
|
|
|
# Train SVM Model |
|
|
|
clf = SVC(C=C, gamma=gamma, kernel='rbf') |
|
|
|
clf.fit(X, y.ravel()) |
|
|
|
|
|
|
@ -84,16 +141,31 @@ plt.show() |
|
|
|
print('Program paused. Press enter to continue.\n') |
|
|
|
input() |
|
|
|
|
|
|
|
## =============== Part 6: Visualizing Dataset 3 ================ |
|
|
|
print('Loading and Visualizing Data ...\n') |
|
|
|
|
|
|
|
# Load from ex6data3: |
|
|
|
# You will have X, y in your environment |
|
|
|
data = scipy.io.loadmat('mat/ex6data3.mat', matlab_compatible=True) |
|
|
|
|
|
|
|
X = data['X'] |
|
|
|
y = data['y'] |
|
|
|
Xval = data['Xval'] |
|
|
|
yval = data['yval'] |
|
|
|
|
|
|
|
# Plot training data |
|
|
|
plotdata(X, y) |
|
|
|
plt.show() |
|
|
|
|
|
|
|
print('Program paused. Press enter to continue.\n') |
|
|
|
input() |
|
|
|
|
|
|
|
## ========== Part 7: Training SVM with RBF Kernel (Dataset 3) ========== |
|
|
|
# Try different SVM Parameters here |
|
|
|
C, sigma = dataset3_params(X, y, Xval, yval) |
|
|
|
|
|
|
|
# Train the SVM |
|
|
|
clf = clf.set_params(C=C, gamma = 1 / (2 * sigma * sigma)) |
|
|
|
clf.fit(X, y.ravel()) |
|
|
|
|
|
|
|
visualize_boundary(X, y, clf) |
|
|
|