基本数据类型
5月16号更新
---------------------------------------------
- 保存模型
print("Saving state, iter:", str(epoch))
torch.save(model.state_dict(), f'logs/Epoch{epoch}-acc{acc}.pth')
- 加载模型 / 用于预训练
model = ResNet18().to(device)
# ----------------------------#
model_path = r"logs/Epoch2-acc0.6816.pth"
print('Loading weights into state dict...')
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
state_dict = torch.load(model_path, map_location=device)
model.load_state_dict(state_dict, strict=True)
- 使用部分预训练权重(5月21号更新)
# -----使用部分预训练权重------------------#
model_path = r"logs/Epoch2-acc0.6831.pth"
print('Loading weights into state dict...')
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
model_dict = model.state_dict()
pretrained_dict = torch.load(model_path, map_location=device)
pretrained_dict = {k: v for k, v in pretrained_dict.items() if np.shape(model_dict[k]) == np.shape(v)}
# pretrained_dict = {k: v for k, v in pretrained_dict.items() if k in model_dict} # 上面不能用时用下面这个
model_dict.update(pretrained_dict)
model.load_state_dict(model_dict)
print('Finished!')
# ---------------------------------------#
---------------------------------------------
Data type
type check
a = torch.randn(2, 3) # 随机生成2行3列的矩阵
print(a.shape)
print(a.size(1)) # 返回shape的第2个元素
print(a.shape[1]) # 3
# cpu上
print(a.type()) # torch.FloatTensor
print(type(a))
print(isinstance(a, torch.FloatTensor))
# Gpu上
data = a.cuda()
print(isinstance(data, torch.cuda.FloatTensor))
"""
在pytorch0.3的版本中dimention为0的tensor是不存在的，如果表达是标量返回[0.3]
在之后的版本中，标量返回0.3 (为了语义更加清晰，使用长度为0的标量)
区分dim/size/shape/tensor
[2, 2]
dim: 2 rank
size/shape: [2, 2]
tensor: 具体数字 [1, 3 ]
[2, 4]
"""
Dimension 0/1/2
# Dim=0,用于loss
a = torch.tensor(2.2)
print(a.shape) # torch.Size([])
print(len(a.shape)) # 0
print(a.size()) # torch.Size([])
# Dim=1,用于Bias/Linear input
b = torch.tensor([2]) # 直接这样写，里面的数据类型跟着里面数据变化
print(b)
print(b.type())
c = torch.tensor([1.1, 2.2])
print(c)
print(c.type())
d = torch.FloatTensor(2)
print(d)
e = torch.IntTensor([2.2])
print(e)
data = np.ones(3)
print(data)
f = torch.from_numpy(data) # 将numpy转换成tensor
print(f)
# Dim=2,Linear input/batch
g = torch.randn(2, 3) # 随机正太分布
print(g)
print(g.shape)
print(g.size())
print(g.size(0))
print(g.size(1))
print(g.shape[1])
# Dim=3 RNN input/Batch
h = torch.rand(3, 2, 3) # 随机均匀分布
print(h)
print(h.shape)
print(h[0])
print(h[1])
print(list(h.shape))
# Dim=4 CNN:[b,c,h,w]
# 下面解释为2张照片，每张照片通道数为3，长宽为28×28
i = torch.rand(2, 3, 28, 28) # 照片数 通道数(彩色图片为3) 图片长 图片宽
print(i)
创建Tensor
import from numpy
import torch
import numpy as np
# 从numpy中导入
a = np.array([2, 3.3])
data = torch.from_numpy(a)
print(data)
b = np.ones([3, 4])
dd = torch.from_numpy(b)
print(dd)
import from list
# 从list中导入
# 大写的Tensor():与FloatTensor类似，接受shape作为参数，小写的接受现有的数据
c = torch.tensor([2., 3.2])
d = torch.FloatTensor([2., 3.2]) # 也可接受现有数据，但是数据必须用一个list来表示。如果接受shape：（2, 3）
e = torch.tensor([[2., 3.2], [1., 22.3]])
print(c)
print(d)
print(e)
uninitialized
# 生成未初始化数据:只是作为一个容器，后面会把数据写进来
# torch.empty() : 给shape
# torch.FloatTensor(d1, d2, d3)
# torch.IntTensor(d1, d2, d3)
f = torch.empty(2, 3)
print(f)
print(torch.Tensor(2, 3)) # 数据大小相差大，记得覆盖否则可能出现torch.not number或torch.infinity
print(torch.IntTensor(2, 3))
print(torch.FloatTensor(2, 3))
set default type
# set default type: torch中默认的类型是torch.FloatTensor
print(torch.tensor([1.2, 3]).type())
torch.set_default_tensor_type(torch.DoubleTensor)
print(torch.tensor([1.2, 3]).type())
rand/rand_like, randint
# rand/rand_like, randint
# rand : [0, 1] 均匀分布
# rand_like: [min, max) 最大值不包含在里面
# randint *_like
print(torch.rand(3, 3)) # 比较均匀的采样出来
a = torch.rand(3, 3)
print(torch.rand_like(a)) # rand_like接受的参数是一个tensor,相当于把a.shape读出来再送给rand函数
print(torch.randint(1, 10, [3, 3]))
randn
# randn: 正态分布
# N(0, 1) 用在bias比较多
# N(u, std)
print(torch.randn(3, 3))
# full函数生成长度为10都为0的list 反差从1到0慢慢减小
print(torch.normal(mean=torch.full([10], 0), std=torch.arange(1, 0, -0.1)))
full
# full
print(torch.full([2, 3], 7))
print(torch.full([], 7)) # dim=0
print(torch.full([1], 7)) # dim=1
arange/range
print(torch.arange(0, 10)) # 不包括10
print(torch.arange(0, 10, 2))
linspace/logspace
print(torch.linspace(0, 10, steps=4)) # 等分的切，包括10
print(torch.logspace(0, 1, steps=10)) # 切10等分，每个取指数0**10~1**10
Ones/zeros/eye
# ones:生成全是0的，直接给出shape
# zeros:生成全是1的
# eye: 生成对角线全是1的,只接受1个参数或2个参数
print(torch.ones(3, 3))
print(torch.zeros(3, 3))
print(torch.eye(3, 4))
data = torch.ones(3, 3)
print(torch.ones_like(data))
randperm:随机打散
# randperm:随机打散
print(torch.randperm(10))
a = torch.rand(2, 3)
b = torch.rand(2, 2)
idx = torch.randperm(2)
print(idx)
print(a)
print(b)
print(a[idx]) # 达到协同shuffle的功能
print(b[idx])
索引与切片
indexing
a = torch.rand(4, 3, 28, 28)
print(a[0])
print(a[0].shape) # torch.Size([3, 28, 28]) ：索引第一个维度 ：取第0张图片
print(a[0, 0].shape) # torch.Size([28, 28])：第二个维度：第0张图片的第0个通道
print(a[0, 0, 2])
print(a[0, 0, 2, 4]) # tensor(0.9441) : 第0张图片第0个通道第二行第4列
select first/last N
# select first/last N
a = torch.rand(4, 3, 28, 28)
print(a.shape) # torch.Size([4, 3, 28, 28])
print(a[:2].shape) # torch.Size([2, 3, 28, 28])
print(a[:2, :1, :, :].shape) # torch.Size([2, 1, 28, 28])
print(a[:2, 1:, :, :].shape) # torch.Size([2, 2, 28, 28])
print(a[:2, -1:, :, :].shape) # torch.Size([2, 1, 28, 28])
select by steps
# select by steps
a = torch.rand(4, 3, 28, 28)
print(a[:, :, 0:28:2, 0:28:2].shape) # torch.Size([4, 3, 14, 14])
print(a[:, :, ::2, ::2].shape) # torch.Size([4, 3, 14, 14])
select by specific index
# select by specific index
a = torch.rand(4, 3, 28, 28)
print(a)
print(a.index_select(0, torch.tensor([0, 2])).shape) # 第1个参数的第0和第1个
print(a.index_select(2, torch.arange(20)).shape)
... 任意多的维度
# ... 任意多的维度
a = torch.rand(4, 3, 28, 28)
print(a[...].shape) # torch.Size([4, 3, 28, 28])
print(a[:, 1, ...].shape) # torch.Size([4, 28, 28])
print(a[..., :2].shape) # torch.Size([4, 3, 28, 2])
select by mask
# select by mask
x = torch.randn(3, 4)
y = torch.randn(3, 4)
print(x)
mask = x.ge(0.5) # >=0.5的位置为True
print(mask)
print(torch.masked_select(y, mask)) # 为True的位置选出来
select by flatten index
# select by flatten index
src = torch.IntTensor(3, 4)
print(src)
print(torch.take(src, torch.tensor([0, 2, 5])))
Tensor维度变换
- view # 将一个shape转换成例一个shape
- squeeze(减少维度)/unsqueeze(增加维度)
- transpose(单维变换)/t(转置)/repeat(多维变换)
- expand(改变理解方式)/repeat(实实在在增加数据 memory copied)
view: lost dim information
# view: lost dim information
a = torch.rand(4, 1, 28, 28)
print(a)
print(a.shape)
print(a.view(4, 28 * 28).shape)
print(a.view(4 * 28, 28).shape)
print(a.view(4*1, 28, 28).shape)
b = a.view(4, 784)
b.view(4, 28, 28, 1) # logic bug
# flexible but prone to corrupt, 维度不匹配
print(a.view(4, 783)) # RuntimeError: shape '[4, 783]' is invalid for input of size 3136
squeeze / unsqueeze
unsqueeze
"""
范围:
[-a.dim()-1, a.dim()+1]
[-5, 5)
"""
a = torch.rand(4, 1, 28, 28)
print(a.shape)
print(a.unsqueeze(0).shape)
print(a.unsqueeze(-1).shape)
print(a.unsqueeze(4).shape)
print(a.unsqueeze(-5).shape)
print(a.unsqueeze(5).shape) # IndexError: Dimension out of range (expected to be in range of [-5, 4], but got 5)
a = torch.tensor([1.2, 2.3])
print(a)
print(a.unsqueeze(-1))
print(a.unsqueeze(0))
# 案例:
b = torch.rand(32)
f = torch.rand(4, 32, 14, 14)
b = b.unsqueeze(1).unsqueeze(2).unsqueeze(0)
print(b.shape)
squeeze
# squeeze
b = torch.rand(1, 32, 1, 1)
print(b.squeeze()) # 能压缩的都压缩
print(b.squeeze(0).shape) # 压缩第0个元素
print(b.squeeze(-1).shape)
print(b.squeeze(1).shape) # 32不能压缩就不压缩
print(b.squeeze(-4).shape)
expand/repeat
# expand/repeat
# expand: broadcasting 改变理解方式
# repeat: memory copied 实实在在的增加数据
a = torch.rand(4, 32, 14, 14)
b = torch.rand(1, 32, 1, 1)
print(b)
print(b.expand(4, 32, 14, 14)) # torch.Size([4, 32, 14, 14])
print(b.expand(-1, 32, -1, -1).shape) # -1表示该维度不变
print(b.expand(-1, 32, -1, -4).shape) # 写-4变-4 RuntimeError: invalid shape dimension -128
# repeat:不建议使用
print(b.repeat(4, 32, 1, 1).shape) # 第二个拷贝32次
print(b.repeat(4, 1, 1, 1).shape)
print(b.repeat(4, 1, 32, 32).shape) #
t():转置 只适合2D tensor
# t():转置 只适合2D tensor
a = torch.randn(3, 4)
print(a)
print(a.t())
Transpose: 维度变换
# Transpose: 维度变换
a = torch.rand(4, 3, 32, 32)
print(a.shape)
"""
RuntimeError: view size is not compatible with input tensor's size and stride
(at least one dimension spans across two contiguous subspaces). Use .reshape(...) instead.
"""
a1 = a.transpose(1, 3).contiguous().view(4, 3*32*32).view(4, 3, 32, 32) # 要加contigous
a2 = a.transpose(1, 3).contiguous().view(4, 3*32*32).view(4, 3, 32, 32).transpose(1, 3)
print(a1.shape)
print(a2.shape)
permute:可以直接排位置，可以使用任意多次的transpose来达到他的目的
# permute:可以直接排位置，可以使用任意多次的transpose来达到他的目的
a = torch.rand(4, 3, 28, 28)
print(a.transpose(1, 3).shape) # torch.Size([4, 28, 28, 3])
b = torch.rand(4, 3, 28, 32)
print(b.transpose(1, 3).shape) # torch.Size([4, 32, 28, 3])
print(b.transpose(1, 3).transpose(1, 3).shape) # torch.Size([4, 3, 28, 32])
print(b.permute(0, 2, 3, 1).shape) # torch.Size([4, 28, 32, 3])
Broadcast自动扩展
"""
expand
without copying data
insert 1 dim ahead
expand dims with size 1 to same size
feature maps:[4, 32, 14, 14]
bias:[32, 1, 1] => [1, 32, 1, 1] => [4, 32, 14, 14] bias的扩张
"""
broadcast
# situation 1
# [4, 32, 14, 14]
# [1, 32, 1, 1] => [4, 32, 14, 14]
# situation2
# [4, 32, 14, 14]
# [14, 14] => [1, 1, 14, 14] => [4, 32, 14, 14] # 可以先unsqueeze再expand
# situation3（不符合）
# [4, 32, 14, 14]
# [2, 32, 14, 14]
# a = torch.tensor([2, 32, 14, 14])
# # print(a)
# # print(a[:])
# a = torch.IntTensor(4, 3)
# b = torch.IntTensor(3)
# print(a)
# print(b)
"""
match from last dim
1. no dim
2. dim of size 1
"""
拼接与拆分
"""
Merge or split
合并:
cat
stack
分割:
split
chunk
"""
cat
# cat
a = torch.rand(4, 32, 8)
b = torch.rand(5, 32, 8)
print(torch.cat([a, b], dim=0).shape) # torch.Size([9, 32, 8])
a1 = torch.rand(4, 3, 32, 32)
a2 = torch.rand(4, 1, 32, 32)
print(torch.cat([a1, a2], dim=0).shape) # RuntimeError: invalid argument 0 其他维度要一致
print(torch.cat([a1, a2], dim=1).shape) # torch.Size([4, 4, 32, 32])
stack: create a new dim: 需求 维度完全一致
# stack: create a new dim: 需求 维度完全一致
a1 = torch.rand(4, 3, 16, 32)
a2 = torch.rand(4, 3, 16, 32)
print(torch.cat([a1, a2], dim=2).shape) # torch.Size([4, 3, 32, 32])
print(torch.stack([a1, a2], dim=2).shape) # torch.Size([4, 3, 2, 16, 32])
a = torch.rand(32, 8)
b = torch.rand(32, 8)
print(torch.stack([a, b], dim=0).shape) # torch.Size([2, 32, 8])
split: by len: 根据长度来分
# split: by len: 根据长度来分
b = torch.rand(32, 8)
a = torch.rand(32, 8)
# print(a.shape) # torch.Size([32, 8])
c = torch.stack([a, b], dim=0)
# print(c.shape) # torch.Size([2, 32, 8])
aa, bb = c.split([4, 4], dim=2)
print(aa.shape, bb.shape) # torch.Size([1, 32, 8]) torch.Size([1, 32, 8])
# aa, bb = c.split(2, dim=0) # ValueError: not enough values to unpack (expected 2, got 1)
print(c.shape) # torch.Size([2, 32, 8])
chunk: by num: 根据数量来分
# chunk: by num: 根据数量来分
aa, bb = c.chunk(2, dim=2) # torch.Size([2, 32, 8])
print(aa.shape, bb.shape) # torch.Size([1, 32, 8]) torch.Size([1, 32, 8])
数学运算
"""
Math operation
1. add/minus/multiply/divide
2. matmul
3. pow
4. sqrt/rsqrt
5. round
"""
基础部分
# 基础部分
a = torch.rand(3, 4)
b = torch.rand(4)
print(a)
print(b)
print(a + b) # b会被广播
# all()函数的功能: 如果张量tensor中所有元素都是True, 才返回True; 否则返回False
b = torch.tensor([1, 1, 1, 1])
print(torch.all(torch.eq(a-b, torch.sub(a, b))))
matmul
# matmul
# torch.mm
# only for 2d
# torch.matmul
# @
a = torch.tensor([[3., 3.],
[3., 3.]])
print(a)
b = torch.ones(2, 2)
print(b)
print(torch.mm(a, b)) # 只针对2d矩阵
print(torch.matmul(a, b))
print(a@b)
# 案例:
# ==2d的tensor运算
a = torch.rand(4, 784)
x = torch.rand(4, 784)
w = torch.rand(512, 784) # 分别为ch-out ch-in
print((x.@w.t()).shape) # torch.Size([4, 512]) ×时第一个元素为out，所以需要转置
print(torch.matmul(x, w.t()).shape) # torch.Size([4, 512])
# >2d的tensor运算
a = torch.rand(4, 3, 28, 64)
b = torch.rand(4, 3, 64, 32)
print(torch.matmul(a, b).shape) # torch.Size([4, 3, 28, 32])
b = torch.rand(4, 1, 64, 32)
print(torch.matmul(a, b).shape) # torch.Size([4, 3, 28, 32]), 这种情况会先使用broadcast,再使用矩阵相乘
power
# power
a = torch.full([2, 2], 3)
print(a.pow(2))
print(a**2)
aa = a**2
print(aa.sqrt())
print(aa.rsqrt())
print(aa**(0.5))
exp/log
# exp/log
a = torch.exp(torch.ones(2, 2))
print(a)
print(torch.log(a))
approximation
# approximation
a = torch.tensor(3.14)
print(a.floor(), a.ceil(), a.trunc(), a.frac()) # tensor(3.) tensor(4.) tensor(3.) tensor(0.1400)
# 往下取整 往上取整 截取,保留整数 截取,保留小数
a = torch.tensor(3.499)
print(a.round()) # tensor(3.) 四舍五入
a = torch.tensor(3.5)
print(a.round()) # tensor(4.)
clamp：裁剪
# clamp：裁剪
"""
gradient clipping
(min)
(min, max)
"""
grad = torch.rand(2, 3)*15
print(grad)
print(grad.max())
print(grad.median())
print(grad.clamp(10)) # 里面的元素小于10的全部变成10
print(grad.clamp(2, 10)) # 小于2的裁剪成2， 大于10的裁剪成10
Tensor统计
"""
statistics
norm：范数
mean sum
prod
max, min, argmin(最小值的位置), argmax
kthvalue(第几个值 默认是小的: 比如第8个小的), topk(top多少)
"""
norm:
# norm:
a = torch.full([8], 1)
b = a.view(2, 4)
c = a.view(2, 2, 2)
print(a)
print(b)
print(c)
print(a.norm(1), b.norm(1), c.norm(1)) # nsor(8.) tensor(8.) tensor(8.)
print(a.norm(2), b.norm(2), c.norm(2)) # tensor(2.8284) tensor(2.8284) tensor(2.8284)
print(b.norm(1, dim=1)) # dim=1：将dim=1的部分取范数，同时二维向量变成一维向量 tensor([4., 4.])
print(b.norm(2, dim=1)) # tensor([2., 2.])
print(c.norm(1, dim=0))
print(c.norm(2, dim=0))
mean sum min max prod(阶乘)
# mean sum min max prod(阶乘)
a = torch.arange(8).view(2, 4).float()
print(a)
"""
tensor([[0., 1., 2., 3.],
[4., 5., 6., 7.]])
"""
print(a.min(), a.max(), a.mean(), a.prod()) # tensor(0.) tensor(7.) tensor(3.5000) tensor(0.)
print(a.sum()) # tensor(28.)
print(a.argmin(), a.argmax()) # tensor(0) tensor(7)
argmin/argmax在指定维度的表示
# argmin/argmax在指定维度的表示
a = torch.rand(4, 5)
print(a)
print(a.argmax())
print(a.argmax(dim=1)) # 在dim=1即取每个维度中最大值的位置
keepdim
# keepdim
a = torch.rand(4, 10)
print(a)
# print(a.max(dim=1))
print(a.argmax(dim=1))
print(a.max(dim=1, keepdim=True)) # 这个会返回他在dim=1的最大值和最大值的位置
top-k or k-th
# top-k or k-th
a = torch.rand(4, 10)
print(a.topk(3, dim=1))
print(a.topk(3, dim=1, largest=False))
print(a.kthvalue(8, dim=1)) # 返回dim=1的第8大的值
"""
torch.return_types.kthvalue(
values=tensor([0.7363, 0.8011, 0.6856, 0.6297]),
indices=tensor([4, 0, 7, 8]))
"""
compare
# compare
"""
> >= < <= != ==
torch.eq(a, b)
"""
a = torch.rand(4, 10)
print(a > 5) # 里面的每个元素都要比较
print(torch.gt(a, 0))
print(a != 0)
a = torch.ones(2, 3)
b = torch.randn(2, 3)
"""
疑问: torch.rand()和torch.randn()的区分?
答：rand()是均匀分布，randn()是标准正太分布
"""
print(a)
print(b)
print(torch.eq(a,b))
print(torch.eq(a, a)) # 返回每个元素
"""
tensor([[True, True, True],
[True, True, True]])
"""
print(torch.equal(a, a)) # True 所有都为True才为True
Tensor高阶
"""
tensor 高级操作
where
gather: 收集，gather语句类似于查表的过程. 设计目的:使用GPU实现CPU的功能
"""
where
# where
# torch.where(condition,x,y) --> Tensor
# 案例:
cond = torch.tensor([[0.6769, 0.7271],
[0.8884, 0.4163]])
print(cond)
a = torch.zeros(2, 2)
print(a)
b = torch.ones(2, 2)
print(b)
print(torch.where(cond > 0.5, a, b)) # 如果cond成立，选取a中的元素，否则选择b中的元素
gather
# 案例:检索 retrieve label
prob = torch.randn(4, 10)
# print(prob)
idx = prob.topk(dim=1, k=3)
# print(idx)
idx = idx[1]
# print(idx)
label = torch.arange(10) + 100
# print(label)
label_expand = label.expand(4, 10)
print(label_expand)
print(idx) # 这是索引
print('------------------')
# print(idx.long()) # 转换成Longtensor数据格式
print(torch.gather(label_expand, dim=1, index=idx.long())) # 按照index索引进行取数据
梯度
"""
1. len： 可以表示变化的层度
2. dir：表示变化的方向
"""
激活函数
Sigmoid / Logistic梯度推导
# 激活函数
z = torch.linspace(-100, 100, 10)
# sigmoid激活函数
print(z)
print(torch.sigmoid(z)) # 范围在0-1
Tanh
# tanh激活函数: 在rnn中用的比较多 取值范围为-1-1
a = torch.linspace(-1, 1, 10)
print(torch.tanh(a))
Relu
# Relu激活函数
# 在pytorch中的两种实现：1.从torch.nn中 2. 从torch.relu中
from torch.nn import functional as F
a = torch.linspace(-1, 1, 10)
print(torch.relu(a))
print(F.relu(a))
LOSS及其梯度
"""
1. Mean Squared Error
2. Cross Entropy Loss
1. binary
2. multi-class
"""
MSE
一：autograd.grad
# 一：autograd.grad
# Mean Squared Error
# 这里注意MSE于2范数相比，2范数有开根号但是这里没有开根号
# 使用pytorch进行简单的求导
# 这里pred = w * x + b
from torch.nn import functional as F
x = torch.ones(1)
w = torch.full([1], 2)
mse = F.mse_loss(torch.ones(1), x*w) # 第一个参数pred 第二个参数label
print(torch.autograd.grad(mse, [w])) # 第一个参数loss 第二个参数w1, w2, w3
"""
RuntimeError: element 0 of tensors does not require grad and does not have a grad_fn?
w函数在初始化的时候没有设置他需要导数信息，pytorch在建图的时候标注torch不需要求导信息
"""
# 改变如下：告诉pytorch w需要梯度信息
w.requires_grad_()
print(torch.autograd.grad(mse, [w]))
"""
RuntimeError: element 0 of tensors does not require grad and does not have a grad_fn
更新之后还是会报错，因为pytorch是一个动态图
这里更新了w但是图还没有更新
因为pytorch是做一步计算一次图
"""
# 必须经过计算图的过程重新更新一遍
mse = F.mse_loss(torch.ones(1), x*w) # 动态图的建图
print(torch.autograd.grad(mse, [w])) # (tensor([2.]),) 图重新更新后可以计算出结果
print(mse)
二：loss.backward
# 二：loss.backward
from torch.nn import functional as F
x = torch.ones(1)
w = torch.full([1], 2)
mse = F.mse_loss(torch.ones(1), x*w)
# torch.autograd.grad(mse, [w])
w.requires_grad_() # 使w获取梯度
mse = F.mse_loss(torch.ones(1), x*w) # 再次计算获取动态图
# torch.autograd.grad(mse, [w]) # 1. 自动计算 再次计算梯度
mse.backward() # 2. 手动计算tensor([2.])
print(w.grad)
总结
"""
Gradient API
1. 手动求导torch.autograd.grad(loss, [w1, w2, ...])
[w1 grad, w2 grad...]
2. 自动求导loss.backward() # 他返回的梯度信息不会返回而是附在每个梯度信息上面
w1.grad
w2.grad
"""
Softmax
"""
softmax求导:
pi(1-pj) if i=j
-pj*pi if i!=j
1 if i=j
0 if i!=j
"""
import torch
from torch.nn import functional as F
a = torch.rand(3) # tensor([0.4207, 0.2955, 0.8440])
print(a.requires_grad_()) # 这样之后就可以求梯度 tensor([0.5424, 0.1913, 0.9416], requires_grad=True)
p = F.softmax(a, dim=0) # 自动完成建图操作 tensor([0.2489, 0.3556, 0.3954], grad_fn=<SoftmaxBackward>)
# 当你调用一次backward之后除了完成一次反向传播以外，还会把这个图的梯度信息清除掉
print(torch.autograd.grad(p[1], [a],retain_graph=True)) # (tensor([-0.0755, 0.1879, -0.1125]),) i=1为正的其他的为负的
# #
# #
print(torch.autograd.grad(p[2], [a])) # (tensor([-0.1349, -0.1125, 0.2473]),) # i=2为正的其他的为负的
感知机
单一输出感知机
import torch
from torch.nn import functional as F
x = torch.randn(1, 10)
w = torch.randn(1, 10, requires_grad=True)
print(x)
print(w)
o = torch.sigmoid(x@w.t()) # 这里没有写bias
print(o)
print(torch.ones(1, 1))
loss = F.mse_loss(torch.ones(1, 1), o)
print(loss)
loss.backward()
print(w.grad)
多输出感知机
import torch
from torch.nn import functional as F
x = torch.randn(1, 10)
w = torch.randn(2, 10, requires_grad=True)
print(x)
print(w)
o = torch.sigmoid(x@w.t())
loss = F.mse_loss(torch.ones(1, 2), o)
loss.backward()
print(w.grad)
链式法则
import torch
x = torch.tensor(1.)
w1 = torch.tensor(2., requires_grad=True)
b1 = torch.tensor(1.)
w2 = torch.tensor(2., requires_grad=True)
b2 = torch.tensor(1.)
y1 = x*w1 + b1
y2 = y1*w2 + b2
dy2_dy1 = torch.autograd.grad(y2, [y1], retain_graph=True)[0]
dy1_dw1 = torch.autograd.grad(y1, [w1], retain_graph=True)[0]
dy2_dw1 = torch.autograd.grad(y2, w1, retain_graph=True)[0] # 这里的w1加不加[]都行？？
print(dy2_dy1*dy1_dw1)
print(dy2_dw1)
优化实例
import numpy as np
from mpl_toolkits.mplot3d import Axes3D
from matplotlib import pyplot as plt
import torch
def himmelblau(x):
return (x[0] ** 2 + x[1] - 11) ** 2 + (x[0] + x[1] ** 2 - 7) ** 2
# 画图
x = np.arange(-6, 6, 0.1)
y = np.arange(-6, 6, 0.1)
print('x,y range:', x.shape, y.shape)
X, Y = np.meshgrid(x, y) # 将x这个图片和y这个图片拼接到一起
print('X,Y maps:', X.shape, Y.shape)
Z = himmelblau([X, Y])
fig = plt.figure('himmelblau')
ax = fig.gca(projection='3d')
ax.plot_surface(X, Y, Z) # 把x, y的坐标送入Z函数里面得到z的坐标
ax.view_init(60, -30)
ax.set_xlabel('x')
ax.set_ylabel('y')
plt.show()
# 找最小值--初始点不同找的也不同
# [1., 0.], [-4, 0.], [4, 0.]
x = torch.tensor([4., 0.], requires_grad=True) # 在这里不同的初始化权重更新的速率和最后得到的结果都不太同。所以说梯度下降法的初始化很关键
optimizer = torch.optim.Adam([x], lr=1e-3)
for step in range(20000):
pred = himmelblau(x) # x送进来得到预测值，目的是min这个预测值
optimizer.zero_grad() # 将梯度信息进行清零
pred.backward() # 生成x.grad和y.grad即：x和y的梯度信息
optimizer.step() # 将x,y的梯度进行更新
if step % 2000 == 0:
print('step {}: x = {}, f(x) = {}'
.format(step, x.tolist(), pred.item()))
Logistic Regression
Cross Entropy
熵
import torch
a = torch.full([4], 1/4)
print(a)
print(a*torch.log2(a))
print(-(a*torch.log2(a)).sum()) # tensor(2.) 熵越高代表越稳定，没有惊喜度
b = torch.tensor([0.1, 0.1, 0.1, 0.7])
print(-(b*torch.log2(b)).sum()) # tensor(1.3568) higher uncertainty 惊喜度较高
c = torch.tensor([0.001, 0.001, 0.001, 0.999])
print(-(c*torch.log2(c)).sum()) # tensor(0.0313) 极度不稳定
numerical stability
import torch
from torch.nn import functional as F
x = torch.randn(1, 784)
w = torch.randn(10, 784)
logits = x@w.t()
print(logits.shape)
pred = F.softmax(logits, dim=1)
print(pred)
pred_log = torch.log(pred)
loss1 = F.nll_loss(pred_log, torch.tensor([3]))
print(loss1)
loss2 = F.cross_entropy(logits, torch.tensor([3])) # 这里使用logits, 因为cross_entropy = softmax + log + nll_loss (这三个操作一起)
print(loss2)
全连接层
import torch
import torch.nn as nn
x = torch.randn(1, 784) # torch.Size([1, 784])
print(x.shape)
layer1 = nn.Linear(784, 200) # 在这里第一个参数使ch-in 第二个参数是ch-out
layer2 = nn.Linear(200, 200)
layer3 = nn.Linear(200, 10)
x = layer1(x)
print(x.shape) # torch.Size([1, 200])
x = layer2(x)
print(x.shape) # torch.Size([1, 200])
x = layer3(x)
print(x.shape) # torch.Size([1, 10])
print(x)
nn.Relu vs F.relu
import torch
import torch.nn as nn
from torch.nn import functional as F
x = torch.randn(1, 10)
print(x.shape)
# 调用方式:类方法
class ML(nn.module):
def __init__(self):
super(ML, self).__init__()
self.model = nn.Sequential( # 构建模型
nn.Linear(784, 200),
nn.ReLU(inplace=True),
nn.Linear(200, 200),
nn.ReLU(inplace=True),
nn.Linear(200, 10),
nn.ReLU(inplace=True),
)
# 函数方法
x = F.relu(x, inplace=True)
GPU加速
device = torch.device('cuda:0') # 使用设备， 可以选择将需要运算的搬到你需要的设备。
# 将需要加速的运算送进GPU
criteon = nn.CrossEntropyLoss().to(device) # 使用.to()方法会返回个inference,他的类型取决于原来的类型
计算准确的代码
"""
计算准确度的代码
"""
import torch
from torch.nn import functional as F
logits = torch.rand(4, 10)
pred = F.softmax(logits, dim=1)
print(pred)
pred_label = pred.argmax(dim=1) # 取最大值的下标
print(pred_label)
label = torch.tensor([9, 3, 2, 9])
correct = torch.eq(pred_label, label)
print(correct)
print(correct.sum().float().item()/4) # item()作用是得到里面的元素
Visdom可视化
"""
pytorch可视化需要:
方法一：
pip install tensorboardX
1. 需要开启一个监听的进程
方法二：Visdom
1. pip install visdom
2. python -m visdom.server (相当于开启了一个web服务器,web服务器会把数据渲染到网页上去)
可能会遇到的问题: ERROR:root:Error 404 while downloading https://unpkg.com/layout-bin-packer@1.4.0
解决方法: install form source（从github的facebookresearch/visdom下载）
步骤1: pip uninstall visdom
步骤2: 官网下载源代码，之后cd进去目录(进去visdom-master)，之后运行pip install -e .
步骤3： 退回用户目录后再python -m visdom.server
步骤4：打开浏览器，输入他给的地址
"""
# 测试:
from visdom import Visdom
viz = Visdom()
"""
{Y的值，X的值} win可以理解为ID(还有一个id叫做env(默认使用main)) opts是额外的配置信息
对于非image还是numpy数据，image数据是tensor
"""
# viz.line([0.], [0.], win='train_loss', opts=dict(title='train loss'))
# viz.line([loss.item()], [global_step], win='train_loss', update='append')
在训练中
global_step += 1
viz.line([loss.item()], [global_step], win='train_loss', update='append')
在test中
# viz进行可视化
viz.line([[test_loss, correct / len(test_loader.dataset)]],
[global_step], win='test', update='append')
viz.images(data.view(-1, 1, 28, 28), win='x')
viz.text(str(pred.detach().cpu().numpy()), win='pred',
opts=dict(title='pred'))
正则化
optimizer = optim.SGD(net.parameters(), lr=learning_rate, weight_decay=0.01) # 这里weight_decay=0.01是指进行正则化，这里是2范数
Dropout
import torch
net_droped = torch.nn.Sequential(
torch.nn.Linear(784, 200),
torch.nn.Dropout(0.5), # drop 50% of the neuron (在两层之间断掉一些层)
torch.nn.ReLU(),
torch.nn.Linear(200, 200),
torch.nn.Dropout(0.5), # drop 50% of the neuron
torch.nn.ReLU(),
torch.nn.Linear(200, 10),
)
"""
在训练是需要加上Dropout()
但是在test/val是不需要Dropout()
例如:
for epoch in range(epochs):
# train
net_dropped.train()
for batch_idx, (data, targt) in enumerate(train_loader):
...
net_dropped.eval() # 在测试是需要加上这句话去掉dropout
test_loss = 0
correct = 0
for data, target in test_loader:
"""
卷积神经网络
import torch.nn as nn
import torch
from torch.nn import functional as F
# 第一个参数为input的chanel,第二个参数为kernel的数量，kernel_size=3*3 [1, 3, 26, 26]
layer = nn.Conv2d(1, 3, kernel_size=3, stride=1, padding=0)
x = torch.rand(1, 1, 28, 28)
out = layer.forward(x)
print(out.shape) # torch.Size([1, 3, 26, 26]) # 26 = (28-3)/1 + 1
layer = nn.Conv2d(1, 3, kernel_size=3, stride=1, padding=1)
out = layer.forward(x)
print(out.shape) # torch.Size([1, 3, 28, 28])
layer = nn.Conv2d(1, 3, kernel_size=3, stride=2, padding=1)
out = layer.forward(x)
print(out.shape) # torch.Size([1, 3, 14, 14])
out = layer(x) # 会自动进行,运用了python的魔术方法 __call__
print(out.shape) # torch.Size([1, 3, 14, 14])
print(layer.weight) # 查看layer的权重
print(layer.weight.shape) # torch.Size([3, 1, 3, 3])
print(layer.bias.shape) # torch.Size([3])
# F.conv2D
# 上面x = torch.rand(1, 1, 28, 28)
w = torch.rand(16, 3, 5, 5)
b = torch.rand(16)
# out = F.conv2d(x, w, b, stride=1, padding=1)
# print(out) # 报错，一位x和w的chanels数对应不上
"""
RuntimeError: Given groups=1, weight of size 16 3 5 5, expected input[1, 1, 28, 28] to have 3 channels,
but got 1 channels instead
"""
x = torch.randn(1, 3, 28, 28)
out = F.conv2d(x, w, b, stride=1, padding=1)
print(out.shape) # torch.Size([1, 16, 26, 26])
out = F.conv2d(x, w, b, stride=2, padding=2)
print(out.shape) # torch.Size([1, 16, 14, 14])
池化层
"""
outline:
Pooling
upsample
Relu
"""
import torch
import torch.nn as nn
from torch.nn import functional as F
x = torch.randn(1, 16, 14, 14)
print(x.shape) # torch.Size([1, 16, 14, 14])
# 从nn中导入最大池化
layer = nn.MaxPool2d(2, stride=2)
out = layer(x)
print(out.shape) # torch.Size([1, 16, 7, 7]) (14-2)/2 + 1 = 7
# 使用F.的方式平均池化
out = F.avg_pool2d(x, 2, stride=2) # torch.Size([1, 16, 7, 7])
print(out.shape)
# ++++++++++++++++++++++++++++++++++++++++++++++++++++++#
# upsample
# 采用F.interpolate
# interpolate: 是插值的意思
# +++++++++++++++++++++++++++++++++++++++++++++++++++++#
x = out
out = F.interpolate(x, scale_factor=2, mode='nearest') # 采用最近邻采样
print(out.shape) # torch.Size([1, 16, 14, 14])
out = F.interpolate(x, scale_factor=3, mode='nearest')
print(out.shape) # torch.Size([1, 16, 21, 21])
#------------------------------------------------#
# Relu激活函数
#
# ------------------------------------------------#
x = torch.randn(1, 16, 7, 7)
print(x.shape) # torch.Size([1, 16, 7, 7])
# 方法1:采用nn.的方式
layer = nn.ReLU(inplace=True) # inplace=True x--->x'(x'使用x内存空间)
out = layer(x)
print(out.shape) # torch.Size([1, 16, 7, 7])
# 方法2：采用F.的方式
out = F.relu(x)
print(out.shape) # torch.Size([1, 16, 7, 7])
BatchNorm
import torch
import torch.nn as nn
# ----------------------------#
# BatchNorm1d
# ----------------------------#
x = torch.randn(100, 16) + 0.5
print(x.shape)
layer = torch.nn.BatchNorm1d(16) # 这个必须与前面的匹配起来否则会报错
print(layer.running_mean, layer.running_var)
"""
tensor([0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.])
tensor([1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.])
"""
out = layer(x)
print(layer.running_mean, layer.running_var)
"""
tensor([0.0452, 0.0446, 0.0516, 0.0671, 0.0644, 0.0622, 0.0514, 0.0449, 0.0520,
0.0546, 0.0461, 0.0620, 0.0332, 0.0450, 0.0384, 0.0580])
tensor([0.9868, 0.9935, 1.0214, 1.0137, 1.0009, 0.9895, 1.0065, 1.0319, 0.9841,
1.0051, 0.9967, 0.9968, 1.0045, 0.9877, 1.0011, 1.0031])
"""
#----------------------------------------#
# 这里的分布服从于 U(0.5, 1)
#
# ---------------------------------------#
x = torch.randn(100, 16) + 0.5
layer = torch.nn.BatchNorm1d(16)
for i in range(5): # 疑问？？？？？？？？，每循环一次经过一次batchnorm
out = layer(x)
print(layer.running_mean, layer.running_var)
# ---------------------------#
# nn.BatchNorm2d
# ---------------------------#
x = torch.rand(1, 16, 7, 7)
print(x.shape)
layer = nn.BatchNorm2d(16)
out = layer(x)
print(out.shape) # torch.Size([1, 16, 7, 7])
print(layer.weight)
"""
这里的weight,bias更权重的那个不太一样
"""
print(layer.weight.shape) # torch.Size([16])
print(layer.bias.shape) # torch.Size([16])
# -----------------------------------#
# class variables
# -----------------------------------#
print(vars(layer))
# ------------------------------------#
# Test
# ------------------------------------#
layer.eval() # 加这行表示现在是在test阶段
out = layer(x)
print(vars(layer))
nn.Module
import torch
from torch import nn
from torch import optim
# -----------------------------------#
# 使用nn.Module的好处
# 1. 所有的常用的方法都在里面,比如: Linear/Relu/Sigmoid等
# 2. 使用nn.Sequential()容器[sequential是串行的意思], 不管是nn.Module中的还是自己写的都可以在这里使用
# 3. nn.Module可以自动管理parameters
# 4. modules: all nodes / children: direct children
# 5. to(device) （第84行）
# 6. save and load(第90行)
# 7. train/test的方便的切换(第87行)
# 8. implement own layer 实现自己的类(第31 / 第41 行) 只有class才能写到nn.Sequential里面去[第48行]
# -----------------------------------#
class MyLinear(nn.Module):
def __init__(self, inp, outp):
super(MyLinear, self).__init__()
# requires_grad = True
self.w = nn.Parameter(torch.randn(outp, inp)) # nn.Parameter会自动地将torch.tensor通过nn.Parameter加到nn.parameter()里面去
self.b = nn.Parameter(torch.randn(outp))
def forward(self, x):
x = x @ self.w.t() + self.b
return x
class Flatten(nn.Module): # 将所有的打平
def __init__(self):
super(Flatten, self).__init__()
def forward(self, input):
return input.view(input.size(0), -1) # -1表示将其他所有的打平
class TestNet(nn.Module):
def __init__(self):
super(TestNet, self).__init__()
self.net = nn.Sequential(nn.Conv2d(1, 16, stride=1, padding=1),
nn.MaxPool2d(2, 2),
Flatten(), # 实现自己的类，里面只能写类
nn.Linear(1*14*14, 10))
def forward(self, x):
return self.net(x)
class BasicNet(nn.Module):
def __init__(self):
super(BasicNet, self).__init__()
self.net = nn.Linear(4, 3)
def forward(self, x):
return self.net(x)
class Net(nn.Module):
def __init__(self):
super(Net, self).__init__()
# 使用nn.Sequential()容器[sequential是串行的意思], 不管是nn.Module中的还是自己写的都可以在这里使用
self.net = nn.Sequential(BasicNet(),
nn.ReLU(),
nn.Linear(3, 2))
def forward(self, x):
return self.net(x)
def main():
device = torch.device('cuda')
net = Net()
net.to(device) # .to()会返回net引用(和原来的net引用一样) --->但是对于tensor操作来说不是这样的
# train
net.train()
# test
net.eval()
# net.load_state_dict(torch.load('ckpt.mdl')) # 在开始的时候要加载模型
#
#
# torch.save(net.state_dict(), 'ckpt.mdl') # 在模型断电或者中断保存模型的当前状态
for name, t in net.named_parameters():
print('parameters:', name, t.shape) # 打印里面地parameters:权重和bias
for name, m in net.named_children(): # 打印net Sequential的类
print('children:', name, m)
for name, m in net.named_modules():
print('modules:', name, m)
if __name__ == '__main__':
main()
数据增强
# Data argumentation
# ---------------------------------------#
# 这些操作在torchvision包里面
# 1. Flip：翻转
# 2. Rotate
# 3. Random Move & Crop
# 4. GAN : 生成更多的样本
# 5. Noise: N(0, 0.001)加高斯白噪声
# ---------------------------------------#
batch_size=200
learning_rate=0.01
epochs=10
train_loader = torch.utils.data.DataLoader(
datasets.MNIST('../data', train=True, download=True,
transform=transforms.Compose([ # Compose的操作类似于nn.Sequential里面
transforms.RandomHorizontalFlip(), # 水平角度的翻转 （随机翻转-可能翻转也有可能不翻转）
transforms.RandomVerticalFlip(), # 垂直方向
transforms.RandomRotation(15), # 旋转方向，参数为旋转的度数
transforms.RandomRotation([90, 180, 270]), # 随机的从90度180度270度中挑一个角度旋转
transforms.Resize([32, 32]), # 传入的参数为list
transforms.RandomCrop([28, 28]), # 裁剪
transforms.ToTensor(),
# transforms.Normalize((0.1307,), (0.3081,))
])), # x 转换成x'
batch_size=batch_size, shuffle=True)
Cifar-10与ResNet18实战
resnet.py
import torch
from torch import nn
from torch.nn import functional as F # 这里F和nn经常是交叉使用的
class ResBlk(nn.Module):
"""
resnet block：这里是resnet的一个基本模块
"""
def __init__(self, ch_in, ch_out, stride=1):
"""
:param ch_in:
:param ch_out:
"""
super(ResBlk, self).__init__()
# we add stride support for resbok, which is distinct from tutorials.
self.conv1 = nn.Conv2d(ch_in, ch_out, kernel_size=3, stride=stride, padding=1)
self.bn1 = nn.BatchNorm2d(ch_out)
self.conv2 = nn.Conv2d(ch_out, ch_out, kernel_size=3, stride=1, padding=1)
self.bn2 = nn.BatchNorm2d(ch_out)
self.extra = nn.Sequential() # nn.Sequential()本来是空的
if ch_out != ch_in: # 如果不相等就把他的ch_in变成ch_out, 也就是说:他这个是resnet的旁边短接线
# [b, ch_in, h, w] => [b, ch_out, h, w]
self.extra = nn.Sequential(
nn.Conv2d(ch_in, ch_out, kernel_size=1, stride=stride),
nn.BatchNorm2d(ch_out)
)
# -------------------------------#
# 疑问: python在实例化的时候为啥不用调用forward函数？
# 因为pytorch在nn.modules中使用了__call__,里面实现了forward方法
# 只要实例化对象就会自动调用__call__,当自己又没有__call__方法，所以调用父类方法，由于子类重写了forward方法
# 所以优先调用子类的forward方法
# -------------------------------#
def forward(self, x):
"""
:param x: [b, ch, h, w]
:return:
"""
out = F.relu(self.bn1(self.conv1(x))) # 这里经过卷积层，BN层， 然后经过relu层
out = self.bn2(self.conv2(out)) # 这里经过卷积层，BN层
# short cut. # 这里是短接
# extra module: [b, ch_in, h, w] => [b, ch_out, h, w]
out = self.extra(x) + out # element-wise add:
out = F.relu(out) # 最后再经过relu层输出
print('这里打印下out看看', out.shape)
return out
class ResNet18(nn.Module):
def __init__(self):
super(ResNet18, self).__init__()
self.conv1 = nn.Sequential(
nn.Conv2d(3, 64, kernel_size=3, stride=3, padding=0),
nn.BatchNorm2d(64)
)
# followed 4 blocks
# [b, 64, h, w] => [b, 128, h ,w] # 注意这里h,w是变化的
self.blk1 = ResBlk(64, 128, stride=2)
# [b, 128, h, w] => [b, 256, h, w]
self.blk2 = ResBlk(128, 256, stride=2)
# # [b, 256, h, w] => [b, 512, h, w]
self.blk3 = ResBlk(256, 512, stride=2)
# # [b, 512, h, w] => [b, 1024, h, w]
self.blk4 = ResBlk(512, 512, stride=2) # 这里视频是self.blk4 = ResBlk(512, 1024)
self.outlayer = nn.Linear(512*1*1, 10) # 最后再跟一个全连接层
def forward(self, x):
"""
:param x:
:return:
"""
x = F.relu(self.conv1(x)) # 先经过一个卷积层，后面再跟一个relu函数, 经过后x.shape = [128, 64, 10, 10]
# [b, 64, h, w] => [b, 1024, h, w]
x = self.blk1(x) # 经过这层后x.shape = torch.Size([128, 128, 5, 5])
x = self.blk2(x) # 经过这层后x.shape = torch.Size([128, 256, 3, 3])
x = self.blk3(x) # 经过这层后x.shape = torch.Size([128, 512, 2, 2])
x = self.blk4(x) # 经过这层后x.shape = torch.Size([128, 512, 2, 2])
# print('after conv:', x.shape) #[b, 512, 2, 2]
# [b, 512, h, w] => [b, 512, 1, 1]
x = F.adaptive_avg_pool2d(x, [1, 1])
# print('after pool:', x.shape)
x = x.view(x.size(0), -1) # 经过这层后x.shape = torch.Size([128, 512]) x.size(0) = 128
x = self.outlayer(x) # 经过一个全连接层 经过这层后x.shape = torch.Size([128, 10])
return x
def main():
# ResBlk
blk = ResBlk(64, 128, stride=4)
tmp = torch.randn(2, 64, 32, 32)
out = blk(tmp)
print('block:', out.shape) # block: torch.Size([2, 128, 8, 8])
# ResNet18
x = torch.randn(2, 3, 32, 32)
model = ResNet18()
out = model(x)
print('resnet:', out.shape) # resnet: torch.Size([2, 10])
if __name__ == '__main__':
main()
# ---------------ResNet18模型----------------------------#
"""
ResNet18(
(conv1): Sequential(
(0): Conv2d(3, 64, kernel_size=(3, 3), stride=(3, 3))
(1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
)
(blk1): ResBlk(
(conv1): Conv2d(64, 128, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1))
(bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(extra): Sequential(
(0): Conv2d(64, 128, kernel_size=(1, 1), stride=(2, 2))
(1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
)
)
(blk2): ResBlk(
(conv1): Conv2d(128, 256, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1))
(bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(extra): Sequential(
(0): Conv2d(128, 256, kernel_size=(1, 1), stride=(2, 2))
(1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
)
)
(blk3): ResBlk(
(conv1): Conv2d(256, 512, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1))
(bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(extra): Sequential(
(0): Conv2d(256, 512, kernel_size=(1, 1), stride=(2, 2))
(1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
)
)
(blk4): ResBlk(
(conv1): Conv2d(512, 512, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1))
(bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(extra): Sequential()
)
(outlayer): Linear(in_features=512, out_features=10, bias=True)
)
"""
# ----------------------------------------------------------------------#
main.py
import torch
from torch.utils.data import DataLoader # DataLoader是为了能够批量加载数据
from torchvision import datasets # 从torchvision中导入数据集
from torchvision import transforms
from torch import nn, optim
from lenet5 import Lenet5
from resnet import ResNet18
def main():
batchsz = 128 # 这里是batch-size
# torchvision中提供一些已有的数据集 # 第一个参数:自定目录，第二个参数:Train=True, transform：对数据做些变换
cifar_train = datasets.CIFAR10('cifar', True, transform=transforms.Compose([
transforms.Resize((32, 32)),
transforms.ToTensor(),
transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
]), download=False) # download=True:可以自动的download
cifar_train = DataLoader(cifar_train, batch_size=batchsz, shuffle=True) # Dataloader：方便一次加载多个. shuffle:加载的时候随机换一下
cifar_test = datasets.CIFAR10('cifar', False, transform=transforms.Compose([
transforms.Resize((32, 32)),
transforms.ToTensor(),
transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
]), download=False)
cifar_test = DataLoader(cifar_test, batch_size=batchsz, shuffle=True)
x, label = iter(cifar_train).next()
print('x:', x.shape, 'label:', label.shape) # x: torch.Size([128, 3, 32, 32]) label: torch.Size([128])
device = torch.device('cuda') # 后面可以使用GPU计算
# model = Lenet5().to(device)
model = ResNet18().to(device)
criteon = nn.CrossEntropyLoss().to(device) # loss函数他包含softmax, 因为是分类任务所以采用crossentropy
optimizer = optim.Adam(model.parameters(), lr=1e-3) # 优化器把网络里的参数传给他
print(model)
for epoch in range(1000):
model.train() # 模型为train模式
for batchidx, (x, label) in enumerate(cifar_train): # 从每个epoch里的batch_size
# [b, 3, 32, 32]
# [b]
x, label = x.to(device), label.to(device) # 转换到cuda上面来
logits = model(x) # 他与predict的区别是是否经过softmax操作
# logits: [b, 10]
# label: [b] # label不需要probality
# loss: tensor scalar # 长度为0的标量
loss = criteon(logits, label) # 这个label是y
# backprop
optimizer.zero_grad() # 如果不清0就是累加的效果
loss.backward()
optimizer.step() # 更新weight,更新的weight写进optimizer里面
print(epoch, 'loss:', loss.item()) # 对于标量,使用item()把他转换成Numpy
# test
model.eval() # 模型为test模式
with torch.no_grad(): # 这一步是告诉不需要构建梯度(不需要构建图)
# test
total_correct = 0 # 正确的数量
total_num = 0 # 总的数量
for x, label in cifar_test:
# [b, 3, 32, 32]
# [b]
x, label = x.to(device), label.to(device)
# [b, 10]
logits = model(x)
# [b]
pred = logits.argmax(dim=1)
# [b] vs [b] => scalar tensor
correct = torch.eq(pred, label).float().sum().item()
total_correct += correct
total_num += x.size(0)
# print(correct)
acc = total_correct / total_num
print(epoch, 'test acc:', acc)
if __name__ == '__main__':
main()
数据集格式
-- cifar
--cifar-10-batches-py
--batches.meta
--data_batch_1
--data_batch_2
--data_batch_3
--data_batch_4
--data_batch_5
--readme.html
--test_batch