1 概述
两层全连接网络。
代码实现环境：python3
2 数据处理
2.1 加载数据集
将原始数据集放入“data/cifar10/”文件夹下。
### 加载cifar10数据集
import os
import pickle
import random
import numpy as np
import matplotlib.pyplot as plt
def load_CIFAR_batch(filename):
"""
cifar-10数据集是分batch存储的，这是载入单个batch
@参数 filename: cifar文件名
@r返回值: X, Y: cifar batch中的 data 和 labels
"""
with open(filename,'rb') as f:
datadict=pickle.load(f,encoding='bytes')
X=datadict[b'data']
Y=datadict[b'labels']
X=X.reshape(10000, 3, 32, 32).transpose(0,2,3,1).astype("float")
Y=np.array(Y)
return X, Y
def load_CIFAR10(ROOT):
"""
读取载入整个 CIFAR-10 数据集
@参数 ROOT: 根目录名
@return: X_train, Y_train: 训练集 data 和 labels
X_test, Y_test: 测试集 data 和 labels
"""
xs=[]
ys=[]
for b in range(1,6):
f=os.path.join(ROOT, "data_batch_%d" % (b, ))
X, Y=load_CIFAR_batch(f)
xs.append(X)
ys.append(Y)
X_train=np.concatenate(xs)
Y_train=np.concatenate(ys)
del X, Y
X_test, Y_test=load_CIFAR_batch(os.path.join(ROOT, "test_batch"))
return X_train, Y_train, X_test, Y_test
X_train, y_train, X_test, y_test = load_CIFAR10('data/cifar10/')
print(X_train.shape)
print(y_train.shape)
print(X_test.shape)
print( y_test.shape)
运行结果如下：
(50000, 32, 32, 3)
(50000,)
(10000, 32, 32, 3)
(10000,)
2.2 划分数据集
将加载好的数据集划分为训练集，验证集，以及测试集。
## 划分训练集，验证集，测试集
num_train = 49000
num_val = 1000
num_test = 1000
# Validation set
mask = range(num_train, num_train + num_val)
X_val = X_train[mask]
y_val = y_train[mask]
# Train set
mask = range(num_train)
X_train = X_train[mask]
y_train = y_train[mask]
# Test set
mask = range(num_test)
X_test = X_test[mask]
y_test = y_test[mask]
# Preprocessing: reshape the images data into rows
X_train = np.reshape(X_train, (X_train.shape[0], -1))
X_val = np.reshape(X_val, (X_val.shape[0], -1))
X_test = np.reshape(X_test, (X_test.shape[0], -1))
print('Train data shape: ', X_train.shape)
print('Validation data shape: ', X_val.shape)
print('Test data shape: ', X_test.shape)
运行结果如下：
Train data shape: (49000, 3072)
Validation data shape: (1000, 3072)
Test data shape: (1000, 3072)
2.3 归一化
将划分好的数据集归一化，即：所有划分好的数据集减去均值图像。
# Processing: subtract the mean images
mean_image = np.mean(X_train, axis=0)
X_train -= mean_image
X_val -= mean_image
X_test -= mean_image
3 两层神经网络的分类器
3.1 定义两层的神经网络
class TwoLayerNet(object):
def __init__(self, input_size, hidden_size, output_size, std = 1e-4):
"""
Initialize the model weights
W1: First layer weights; has shape (D, H)
b1: First layer biases; has shape (H,)
W2: Second layer weights; has shape (H, C)
b2: Second layer biases; has shape (C,)
Inputs:
- input_size: The dimension D of the input data.
- hidden_size: The number of neurons H in the hidden layer.
- output_size: The number of classes C.
"""
self.params = {}
self.params['W1'] = std * np.random.randn(input_size, hidden_size)
self.params['b1'] = np.zeros(hidden_size)
self.params['W2'] = std * np.random.randn(hidden_size, output_size)
self.params['b2'] = np.zeros(output_size)
def loss(self, X, y, reg = 0.0):
"""
Two-layer network loss function, vectorized implementation (without loops).
Inputs:
- X: A numpy array of shape (num_train, D) contain the training data
consisting of num_train samples each of dimension D
- y: A numpy array of shape (num_train,) contain the training labels,
where y[i] is the label of X[i]
- reg: float, regularization strength
Return:
- loss: the loss value between predict value and ground truth
- grads: Dictionary mapping parameter names to gradients of those parameters
with respect to the loss function; has the same keys as self.params.
Contain 'dW1', 'db1', 'dW2', 'db2'
"""
N, dim = X.shape
grads = {}
# input layer ==> hidden layer ==> ReLU ==> output layer ==> Softmax
W1 = self.params['W1']
b1 = self.params['b1']
W2 = self.params['W2']
b2 = self.params['b2']
# input layer==> hidden layer
Z1 = np.dot(X, W1) + b1
# hidden layer ==> ReLU
A1 = np.maximum(0, Z1) # ReLU function
# ReLU ==> output layer
scores = np.dot(A1, W2) + b2
# output layer ==> Softmax
scores_shift = scores - np.max(scores, axis=1).reshape(-1, 1)
Softmax_output = np.exp(scores_shift) / np.sum(np.exp(scores_shift), axis=1).reshape(-1, 1)
loss = -np.sum(np.log(Softmax_output[range(N), list(y)]))
loss /= N
loss += 0.5 * reg * np.sum(W1 * W1) + 0.5 * reg * np.sum(W2 * W2)
# grads
dS = Softmax_output.copy()
dS[range(N), list(y)] += -1
dS /= N
dW2 = np.dot(A1.T, dS)
db2 = np.sum(dS, axis=0)
dA1 = np.dot(dS, W2.T)
dZ1 = dA1 * (A1 > 0)
dW1 = np.dot(X.T, dZ1)
db1 = np.sum(dZ1, axis=0)
dW2 += reg * W2
dW1 += reg * W1
grads['W1'] = dW1
grads['b1'] = db1
grads['W2'] = dW2
grads['b2'] = db2
return loss, grads
def predict(self, X):
"""
Use the trained weights to predict data labels
Inputs:
- X: A numpy array of shape (num_test, D) contain the test data
Outputs:
- y_pred: A numpy array, predicted labels for the data in X
"""
W1 = self.params['W1']
b1 = self.params['b1']
W2 = self.params['W2']
b2 = self.params['b2']
Z1 = np.dot(X, W1) + b1
A1 = np.maximum(0, Z1) # ReLU function
scores = np.dot(A1, W2) + b2
y_pred = np.argmax(scores, axis=1)
return y_pred
def train(self, X, y, X_val, y_val, learning_rate=1e-3, learning_rate_decay=0.95,
reg=5e-6, num_iters=100, batch_size=200, print_flag=False):
"""
Train Two-layer neural network classifier using SGD
Inputs:
- X: A numpy array of shape (num_train, D) contain the training data
consisting of num_train samples each of dimension D
- y: A numpy array of shape (num_train,) contain the training labels,
where y[i] is the label of X[i], y[i] = c, 0 <= c <= C
- X_val: A numpy array of shape (num_val, D) contain the validation data
consisting of num_val samples each of dimension D
- y_val: A numpy array of shape (num_val,) contain the validation labels,
where y_val[i] is the label of X_val[i], y_val[i] = c, 0 <= c <= C
- learning rate: (float) learning rate for optimization
- learning_rate_decay: Scalar giving factor used to decay the learning rate
after each epoch.
- reg: (float) regularization strength
- num_iters: (integer) numbers of steps to take when optimization
- batch_size: (integer) number of training examples to use at each step
- print_flag: (boolean) If true, print the progress during optimization
Outputs:
- a dictionary contains the loss_history, train_accuracy_history and val_accuracy_history
"""
num_train = X.shape[0]
iterations_per_epoch = max(num_train / batch_size, 1)
loss_history = []
train_accuracy_history = []
val_accuracy_history = []
for t in range(num_iters):
idx_batch = np.random.choice(num_train, batch_size, replace=True)
X_batch = X[idx_batch]
y_batch = y[idx_batch]
loss, grads = self.loss(X_batch, y_batch, reg)
loss_history.append(loss)
self.params['W1'] += -learning_rate * grads['W1']
self.params['b1'] += -learning_rate * grads['b1']
self.params['W2'] += -learning_rate * grads['W2']
self.params['b2'] += -learning_rate * grads['b2']
# Every epoch, check train and val accuracy and decay learning rate.
if t % iterations_per_epoch == 0:
train_accuracy = np.mean(self.predict(X_batch) == y_batch)
val_accuracy = np.mean(self.predict(X_val) == y_val)
train_accuracy_history.append(train_accuracy)
val_accuracy_history.append(val_accuracy)
# Decay learning rate
learning_rate *= learning_rate_decay
# print the progress during optimization
if print_flag and t%100 == 0:
print('iteration %d / %d: loss %f' % (t, num_iters, loss))
return {
'loss_history': loss_history,
'train_accuracy_history': train_accuracy_history,
'val_accuracy_history': val_accuracy_history,
}
使用随机梯度下降训练模型，并在验证集上测试效果。
input_size = 32 * 32 * 3
hidden_size = 50
num_classes = 10
net = TwoLayerNet(input_size, hidden_size, num_classes)
# Train the network
stats = net.train(X_train, y_train, X_val, y_val,
num_iters=1000, batch_size=200,
learning_rate=1e-4, learning_rate_decay=0.95,
reg=0.25, print_flag=True)
# Predict on the validation set
val_acc = (net.predict(X_val) == y_val).mean()
print('Validation accuracy: ', val_acc)
运行结果为：
iteration 0 / 1000: loss 2.302776
iteration 100 / 1000: loss 2.302100
iteration 200 / 1000: loss 2.296930
iteration 300 / 1000: loss 2.261319
iteration 400 / 1000: loss 2.174613
iteration 500 / 1000: loss 2.075674
iteration 600 / 1000: loss 2.103124
iteration 700 / 1000: loss 2.036040
iteration 800 / 1000: loss 1.931442
iteration 900 / 1000: loss 1.906379
Validation accuracy: 0.281
3.2 微调超参数
# 微调超参数
# Hyperparameters
learning_rates = [1e-4, 5e-4, 9e-4, 13e-4, 15e-4]
regularization_strengths = [0.25, 0.5, 0.75, 1.0]
num_iters = 3000
batch_size = 200
learning_rate_decay = 0.98
# Net structure
input_size = 32 * 32 * 3
hidden_size = [50, 100, 150]
num_classes = 10
# Initialization
best_net = None
best_hidden_size = None
best_val = -1
best_lr = None
best_reg = None
results = {}
# Train the two layers network
for i in range(len(hidden_size)):
for lr in learning_rates:
for reg in regularization_strengths:
net = TwoLayerNet(input_size, hidden_size[i], num_classes)
stats = net.train(X_train, y_train, X_val, y_val,
num_iters=num_iters, batch_size=batch_size,
learning_rate=lr, learning_rate_decay=learning_rate_decay,
reg=reg, print_flag=False)
train_accuracy = stats['train_accuracy_history'][-1]
val_accuracy = stats['val_accuracy_history'][-1]
if val_accuracy > best_val:
best_lr = lr
best_reg = reg
best_val = val_accuracy
best_net = net
best_hidden_size = hidden_size[i]
results[(lr, reg)] = train_accuracy, val_accuracy
print('hidden_size: %d lr: %e reg: %e train accuracy: %f val accuracy: %f' %
(hidden_size[i], lr, reg, results[(lr, reg)][0], results[(lr, reg)][1]))
print('Best hidden_size: %d
Best lr: %e
Best reg: %e
train accuracy: %f
val accuracy: %f' %
(hidden_size[i], best_lr, best_reg, results[(best_lr, best_reg)][0], results[(best_lr, best_reg)][1]))
运行结果如下：
hidden_size: 50 lr: 1.000000e-04 reg: 2.500000e-01 train accuracy: 0.410000 val accuracy: 0.407000
hidden_size: 50 lr: 1.000000e-04 reg: 5.000000e-01 train accuracy: 0.415000 val accuracy: 0.399000
hidden_size: 50 lr: 1.000000e-04 reg: 7.500000e-01 train accuracy: 0.370000 val accuracy: 0.400000
hidden_size: 50 lr: 1.000000e-04 reg: 1.000000e+00 train accuracy: 0.420000 val accuracy: 0.403000
hidden_size: 50 lr: 5.000000e-04 reg: 2.500000e-01 train accuracy: 0.595000 val accuracy: 0.494000
hidden_size: 50 lr: 5.000000e-04 reg: 5.000000e-01 train accuracy: 0.595000 val accuracy: 0.505000
hidden_size: 50 lr: 5.000000e-04 reg: 7.500000e-01 train accuracy: 0.495000 val accuracy: 0.483000
hidden_size: 50 lr: 5.000000e-04 reg: 1.000000e+00 train accuracy: 0.600000 val accuracy: 0.475000
hidden_size: 50 lr: 9.000000e-04 reg: 2.500000e-01 train accuracy: 0.670000 val accuracy: 0.496000
hidden_size: 50 lr: 9.000000e-04 reg: 5.000000e-01 train accuracy: 0.675000 val accuracy: 0.510000
hidden_size: 50 lr: 9.000000e-04 reg: 7.500000e-01 train accuracy: 0.610000 val accuracy: 0.485000
hidden_size: 50 lr: 9.000000e-04 reg: 1.000000e+00 train accuracy: 0.635000 val accuracy: 0.487000
hidden_size: 50 lr: 1.300000e-03 reg: 2.500000e-01 train accuracy: 0.700000 val accuracy: 0.486000
hidden_size: 50 lr: 1.300000e-03 reg: 5.000000e-01 train accuracy: 0.590000 val accuracy: 0.500000
hidden_size: 50 lr: 1.300000e-03 reg: 7.500000e-01 train accuracy: 0.615000 val accuracy: 0.466000
hidden_size: 50 lr: 1.300000e-03 reg: 1.000000e+00 train accuracy: 0.620000 val accuracy: 0.481000
hidden_size: 50 lr: 1.500000e-03 reg: 2.500000e-01 train accuracy: 0.660000 val accuracy: 0.485000
hidden_size: 50 lr: 1.500000e-03 reg: 5.000000e-01 train accuracy: 0.585000 val accuracy: 0.466000
hidden_size: 50 lr: 1.500000e-03 reg: 7.500000e-01 train accuracy: 0.655000 val accuracy: 0.483000
hidden_size: 50 lr: 1.500000e-03 reg: 1.000000e+00 train accuracy: 0.630000 val accuracy: 0.484000
hidden_size: 100 lr: 1.000000e-04 reg: 2.500000e-01 train accuracy: 0.325000 val accuracy: 0.413000
hidden_size: 100 lr: 1.000000e-04 reg: 5.000000e-01 train accuracy: 0.340000 val accuracy: 0.416000
hidden_size: 100 lr: 1.000000e-04 reg: 7.500000e-01 train accuracy: 0.375000 val accuracy: 0.421000
hidden_size: 100 lr: 1.000000e-04 reg: 1.000000e+00 train accuracy: 0.480000 val accuracy: 0.409000
hidden_size: 100 lr: 5.000000e-04 reg: 2.500000e-01 train accuracy: 0.605000 val accuracy: 0.496000
hidden_size: 100 lr: 5.000000e-04 reg: 5.000000e-01 train accuracy: 0.580000 val accuracy: 0.510000
hidden_size: 100 lr: 5.000000e-04 reg: 7.500000e-01 train accuracy: 0.605000 val accuracy: 0.496000
hidden_size: 100 lr: 5.000000e-04 reg: 1.000000e+00 train accuracy: 0.540000 val accuracy: 0.508000
hidden_size: 100 lr: 9.000000e-04 reg: 2.500000e-01 train accuracy: 0.720000 val accuracy: 0.509000
hidden_size: 100 lr: 9.000000e-04 reg: 5.000000e-01 train accuracy: 0.665000 val accuracy: 0.507000
hidden_size: 100 lr: 9.000000e-04 reg: 7.500000e-01 train accuracy: 0.630000 val accuracy: 0.512000
hidden_size: 100 lr: 9.000000e-04 reg: 1.000000e+00 train accuracy: 0.610000 val accuracy: 0.497000
hidden_size: 100 lr: 1.300000e-03 reg: 2.500000e-01 train accuracy: 0.720000 val accuracy: 0.495000
hidden_size: 100 lr: 1.300000e-03 reg: 5.000000e-01 train accuracy: 0.775000 val accuracy: 0.524000
hidden_size: 100 lr: 1.300000e-03 reg: 7.500000e-01 train accuracy: 0.640000 val accuracy: 0.503000
hidden_size: 100 lr: 1.300000e-03 reg: 1.000000e+00 train accuracy: 0.665000 val accuracy: 0.478000
hidden_size: 100 lr: 1.500000e-03 reg: 2.500000e-01 train accuracy: 0.650000 val accuracy: 0.516000
hidden_size: 100 lr: 1.500000e-03 reg: 5.000000e-01 train accuracy: 0.675000 val accuracy: 0.499000
hidden_size: 100 lr: 1.500000e-03 reg: 7.500000e-01 train accuracy: 0.635000 val accuracy: 0.493000
hidden_size: 100 lr: 1.500000e-03 reg: 1.000000e+00 train accuracy: 0.600000 val accuracy: 0.493000
hidden_size: 150 lr: 1.000000e-04 reg: 2.500000e-01 train accuracy: 0.410000 val accuracy: 0.420000
hidden_size: 150 lr: 1.000000e-04 reg: 5.000000e-01 train accuracy: 0.475000 val accuracy: 0.415000
hidden_size: 150 lr: 1.000000e-04 reg: 7.500000e-01 train accuracy: 0.385000 val accuracy: 0.422000
hidden_size: 150 lr: 1.000000e-04 reg: 1.000000e+00 train accuracy: 0.375000 val accuracy: 0.425000
hidden_size: 150 lr: 5.000000e-04 reg: 2.500000e-01 train accuracy: 0.605000 val accuracy: 0.524000
hidden_size: 150 lr: 5.000000e-04 reg: 5.000000e-01 train accuracy: 0.550000 val accuracy: 0.512000
hidden_size: 150 lr: 5.000000e-04 reg: 7.500000e-01 train accuracy: 0.565000 val accuracy: 0.511000
hidden_size: 150 lr: 5.000000e-04 reg: 1.000000e+00 train accuracy: 0.580000 val accuracy: 0.501000
hidden_size: 150 lr: 9.000000e-04 reg: 2.500000e-01 train accuracy: 0.635000 val accuracy: 0.526000
hidden_size: 150 lr: 9.000000e-04 reg: 5.000000e-01 train accuracy: 0.615000 val accuracy: 0.512000
hidden_size: 150 lr: 9.000000e-04 reg: 7.500000e-01 train accuracy: 0.655000 val accuracy: 0.523000
hidden_size: 150 lr: 9.000000e-04 reg: 1.000000e+00 train accuracy: 0.585000 val accuracy: 0.507000
hidden_size: 150 lr: 1.300000e-03 reg: 2.500000e-01 train accuracy: 0.745000 val accuracy: 0.519000
hidden_size: 150 lr: 1.300000e-03 reg: 5.000000e-01 train accuracy: 0.655000 val accuracy: 0.501000
hidden_size: 150 lr: 1.300000e-03 reg: 7.500000e-01 train accuracy: 0.625000 val accuracy: 0.530000
hidden_size: 150 lr: 1.300000e-03 reg: 1.000000e+00 train accuracy: 0.645000 val accuracy: 0.506000
hidden_size: 150 lr: 1.500000e-03 reg: 2.500000e-01 train accuracy: 0.660000 val accuracy: 0.478000
hidden_size: 150 lr: 1.500000e-03 reg: 5.000000e-01 train accuracy: 0.675000 val accuracy: 0.485000
hidden_size: 150 lr: 1.500000e-03 reg: 7.500000e-01 train accuracy: 0.670000 val accuracy: 0.512000
hidden_size: 150 lr: 1.500000e-03 reg: 1.000000e+00 train accuracy: 0.655000 val accuracy: 0.495000
Best hidden_size: 150
Best lr: 1.300000e-03
Best reg: 7.500000e-01
train accuracy: 0.625000
val accuracy: 0.530000
在测试集上测试其性能：
test_acc = (best_net.predict(X_test) == y_test).mean()
print('Test accuracy: ', test_acc)
结果如下：
Test accuracy: 0.504