一、cifar10数据集
（http://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz）源格式是数据文件，因为训练需要转换成图片格式
转换代码：
注意文件路径改成自己的文件路径，train文件夹需要自己建，等待转换完成
from scipy.misc import imsave import numpy as np # 解压 返回解压后的字典 def unpickle(file): import pickle as pk fo = open(file, 'rb') dict = pk.load(fo,encoding='iso-8859-1') fo.close() return dict # 生成训练集图片 for j in range(1, 6): dataName = "cifar-10-python/cifar-10-batches-py/data_batch_" + str(j) # 读取当前目录下的data_batch1~5文件。 Xtr = unpickle(dataName) print (dataName + " is loading...") for i in range(0, 10000): img = np.reshape(Xtr['data'][i], (3, 32, 32)) # Xtr['data']为图片二进制数据 img = img.transpose(1, 2, 0) # 读取image picName = 'train/' + str(Xtr['labels'][i]) + '_' + str(i + (j - 1)*10000) + '.jpg' # Xtr['labels']为图片的标签，值范围0-9，本文中，train文件夹需要存在，并与脚本文件在同一目录下。 imsave(picName, img) print (dataName + " loaded.") print ("test_batch is loading...") # 生成测试集图片 testXtr = unpickle("test_batch") for i in range(0, 10000): img = np.reshape(testXtr['data'][i], (3, 32, 32)) img = img.transpose(1, 2, 0) picName = 'test/' + str(testXtr['labels'][i]) + '_' + str(i) + '.jpg' imsave(picName, img) print ("test_batch loaded.")
二、mnist数据集的转化
1、先解压出二进制文件，再运行
import numpy as np import struct from PIL import Image import os data_file = 'MNIST_data/train-images.idx3-ubyte' #需要修改的路径 # It's 47040016B, but we should set to 47040000B data_file_size = 47040016 data_file_size = str(data_file_size - 16) + 'B' data_buf = open(data_file, 'rb').read() magic, numImages, numRows, numColumns = struct.unpack_from( '>IIII', data_buf, 0) datas = struct.unpack_from( '>' + data_file_size, data_buf, struct.calcsize('>IIII')) datas = np.array(datas).astype(np.uint8).reshape( numImages, 1, numRows, numColumns) label_file = 'MNIST_data/train-labels.idx1-ubyte' #需要修改的路径 # It's 60008B, but we should set to 60000B label_file_size = 60008 label_file_size = str(label_file_size - 8) + 'B' label_buf = open(label_file, 'rb').read() magic, numLabels = struct.unpack_from('>II', label_buf, 0) labels = struct.unpack_from( '>' + label_file_size, label_buf, struct.calcsize('>II')) labels = np.array(labels).astype(np.int64) datas_root = 'MNIST_data/mnist_train' #需要修改的路径 if not os.path.exists(datas_root): os.mkdir(datas_root) for i in range(10): file_name = datas_root + os.sep + str(i) if not os.path.exists(file_name): os.mkdir(file_name) for ii in range(numLabels): img = Image.fromarray(datas[ii, 0, 0:28, 0:28]) label = labels[ii] file_name = datas_root + os.sep + str(label) + os.sep + 'mnist_train_' + str(ii) + '.png' img.save(file_name)
import numpy as np import struct from PIL import Image import os data_file = 'MNIST_data/t10k-images.idx3-ubyte' #需要修改的路径 # It's 7840016B, but we should set to 7840000B data_file_size = 7840016 data_file_size = str(data_file_size - 16) + 'B' data_buf = open(data_file, 'rb').read() magic, numImages, numRows, numColumns = struct.unpack_from( '>IIII', data_buf, 0) datas = struct.unpack_from( '>' + data_file_size, data_buf, struct.calcsize('>IIII')) datas = np.array(datas).astype(np.uint8).reshape( numImages, 1, numRows, numColumns) label_file = 'MNIST_data/t10k-labels.idx1-ubyte'#需要修改的路径 # It's 10008B, but we should set to 10000B label_file_size = 10008 label_file_size = str(label_file_size - 8) + 'B' label_buf = open(label_file, 'rb').read() magic, numLabels = struct.unpack_from('>II', label_buf, 0) labels = struct.unpack_from( '>' + label_file_size, label_buf, struct.calcsize('>II')) labels = np.array(labels).astype(np.int64) datas_root = 'MNIST_data/mnist_test' #需要修改的路径 if not os.path.exists(datas_root): os.mkdir(datas_root) for i in range(10): file_name = datas_root + os.sep + str(i) if not os.path.exists(file_name): os.mkdir(file_name) for ii in range(numLabels): img = Image.fromarray(datas[ii, 0, 0:28, 0:28]) label = labels[ii] file_name = datas_root + os.sep + str(label) + os.sep + 'mnist_test_' + str(ii) + '.png' img.save(file_name)
2、接着构造出图片集noisy_test和noisy_train
这两个图片集是加了椒盐噪声的集合（可用作图像去噪）
import numpy as np import struct import numpy as np from PIL import Image import os data_file = 'MNIST_data/train-images.idx3-ubyte' #需要修改的路径 # It's 47040016B, but we should set to 47040000B data_file_size = 47040016 data_file_size = str(data_file_size - 16) + 'B' data_buf = open(data_file, 'rb').read() magic, numImages, numRows, numColumns = struct.unpack_from( '>IIII', data_buf, 0) datas = struct.unpack_from( '>' + data_file_size, data_buf, struct.calcsize('>IIII')) datas = np.array(datas).astype(np.uint8).reshape( numImages, 1, numRows, numColumns) label_file = 'MNIST_data/train-labels.idx1-ubyte' #需要修改的路径 # It's 60008B, but we should set to 60000B label_file_size = 60008 label_file_size = str(label_file_size - 8) + 'B' label_buf = open(label_file, 'rb').read() magic, numLabels = struct.unpack_from('>II', label_buf, 0) labels = struct.unpack_from( '>' + label_file_size, label_buf, struct.calcsize('>II')) labels = np.array(labels).astype(np.int64) datas_root = 'MNIST_data/noisy_train' #需要修改的路径 if not os.path.exists(datas_root): os.mkdir(datas_root) for i in range(10): file_name = datas_root + os.sep + str(i) if not os.path.exists(file_name): os.mkdir(file_name) for ii in range(numLabels): img = Image.fromarray(datas[ii, 0, 0:28, 0:28]) label = labels[ii] file_name = datas_root + os.sep + str(label) + os.sep + 'mnist_train_' + str(ii) + '.png' x_train_noisy = np.array(img) noise_factor = 0.5 x_train_noisy = x_train_noisy.astype('float32') / 255. x_train_noisy = x_train_noisy + noise_factor * np.random.normal(loc=0.0, scale=1.0, size=x_train_noisy.shape) x_train_noisy = np.clip(x_train_noisy, 0., 1.) x_train_noisy = x_train_noisy.astype(np.float) x_train_noisy = x_train_noisy.astype('float32') * 255 x_train_noisy = x_train_noisy.astype(np.uint8) #print(x_train_noisy) #os._exit(0) img=Image.fromarray(x_train_noisy) img.save(file_name)
import numpy as np import struct from PIL import Image import os data_file = 'MNIST_data/t10k-images.idx3-ubyte' #需要修改的路径 # It's 7840016B, but we should set to 7840000B data_file_size = 7840016 data_file_size = str(data_file_size - 16) + 'B' data_buf = open(data_file, 'rb').read() magic, numImages, numRows, numColumns = struct.unpack_from( '>IIII', data_buf, 0) datas = struct.unpack_from( '>' + data_file_size, data_buf, struct.calcsize('>IIII')) datas = np.array(datas).astype(np.uint8).reshape( numImages, 1, numRows, numColumns) label_file = 'MNIST_data/t10k-labels.idx1-ubyte'#需要修改的路径 # It's 10008B, but we should set to 10000B label_file_size = 10008 label_file_size = str(label_file_size - 8) + 'B' label_buf = open(label_file, 'rb').read() magic, numLabels = struct.unpack_from('>II', label_buf, 0) labels = struct.unpack_from( '>' + label_file_size, label_buf, struct.calcsize('>II')) labels = np.array(labels).astype(np.int64) datas_root = 'MNIST_data/noisy_test' #需要修改的路径 if not os.path.exists(datas_root): os.mkdir(datas_root) for i in range(10): file_name = datas_root + os.sep + str(i) if not os.path.exists(file_name): os.mkdir(file_name) for ii in range(numLabels): img = Image.fromarray(datas[ii, 0, 0:28, 0:28]) label = labels[ii] file_name = datas_root + os.sep + str(label) + os.sep + 'mnist_test_' + str(ii) + '.png' x_train_noisy = np.array(img) noise_factor = 0.5 x_train_noisy = x_train_noisy.astype('float32') / 255. x_train_noisy = x_train_noisy + noise_factor * np.random.normal(loc=0.0, scale=1.0, size=x_train_noisy.shape) x_train_noisy = np.clip(x_train_noisy, 0., 1.) x_train_noisy = x_train_noisy.astype(np.float) x_train_noisy = x_train_noisy.astype('float32') * 255 x_train_noisy = x_train_noisy.astype(np.uint8) #print(x_train_noisy) #os._exit(0) img=Image.fromarray(x_train_noisy) img.save(file_name)