准备数据集:从CIFAR-10抽离鸟与飞机的图片

from torchvision import datasets
from torchvision import transforms
data_path = './data'

# 加载训练集
cifar10 = datasets.CIFAR10(root = data_path, train=True, download=False)
# 加载验证集
cifar10_val = datasets.CIFAR10(root=data_path, train=False, download=False)

# 使用To_Tensor 将 32*32*3 的图片格式转为 3*32*32 的张量格式
to_tensor = transforms.ToTensor()

# 进行标签转换,否则下面开始训练时会报错:IndexError: Target 2 is out of bounds
label_map={0:0, 2:1}

# 分别从训练集和验证集中抽取鸟与飞机图片
cifar2 = [(to_tensor(img), label_map[label]) for img, label in cifar10 if label in [0, 2]]
cifar2_val = [(to_tensor(img), label_map[label]) for img, label in cifar10_val if label in [0, 2]]

验证下,是否获取成功

import matplotlib.pyplot as plt
img, _ = cifar2[100]
plt.imshow(img.permute(1, 2, 0))
<matplotlib.image.AxesImage at 0x29bdaed6aa0>

使用
DataLoader
封装数据集

from torch.utils.data import DataLoader

# 训练集数据加载器
train_loader = DataLoader(cifar2, batch_size=64, pin_memory=True, shuffle=True, num_workers=4, drop_last=True) # type: ignore
# 验证集数据加载器
val_loader = DataLoader(cifar2_val, batch_size=64, pin_memory=True, num_workers=4, drop_last=True)

子类化nn.Module

我们打算放弃
nn.Sequential
带来的灵活性。使用更自由的子类化
nn.Module


为了子类化
nn.Module
,我们至少需要定义一个
forward()
函数,该函数用于接收模块的输入并返回输出,这便是模块计算的之处。


Pytorch
中,如果使用标准的
torch
操作,自动求导将自动处理反向传播,也就是不需要定义
backward()
函数。

重新定义我们的模型:

import torch
from torch import nn
import torch.nn.functional as F

class Net(nn.Module):
    def __init__(self):
        super().__init__()
        self.conv1 = nn.Conv2d(in_channels=3, out_channels=16, kernel_size=3, padding=1)    # 卷积层
        self.conv2 = nn.Conv2d(in_channels=16, out_channels=8, kernel_size=3, padding=1)
        self.fc1 = nn.Linear(8*8*8, 32) # 全连接层,8个8x8的特征图,每个特征图有8个通道
        self.fc2 = nn.Linear(32, 2)

    def forward(self, x):
        out = F.max_pool2d(torch.tanh(self.conv1(x)), 2)    # 图片初始大小为32x32,经过第一次池化,特征图大小为16x16
        out = F.max_pool2d(torch.tanh(self.conv2(out)), 2)  # 经过池化,特征图大小为8x8
        out = out.view(-1, 8*8*8)
        out = torch.tanh(self.fc1(out))
        out = self.fc2(out)
        return out

假设卷积层输入特征图大小为
\(W_{in}\times H_{in}\)
,卷积核大小为
\(K\)
,padding大小为
\(P\)
,stride为
\(S\)
,卷积层输出特征图大小为
\(W_{out}\times H_{out}\)
,那么有如下公式:

\(W_{out} = \lfloor \frac{W_{in}+2P-K}{S} \rfloor +1\)

\(H_{out} = \lfloor \frac{H_{in}+2P-K}{S} \rfloor +1\)
其中,
\(\lfloor x \rfloor\)
表示将
\(x\)
向下取整的结果。

在这个代码中,第一个卷积层的输入特征图大小为32x32,卷积核大小为3,padding大小为1,stride为1,因此将上述公式代入计算,得到:

\(W_{out} = \lfloor \frac{32+2\times1-3}{1} \rfloor +1 = 32\)

\(H_{out} = \lfloor \frac{32+2\times1-3}{1} \rfloor +1 = 32\)

因此,第一个卷积层的输出特征图大小为32x32。

简单测试下模型是否运行

model = Net()
model(img.unsqueeze(0))
tensor([[-0.0153, -0.1532]], grad_fn=<AddmmBackward0>)

训练卷积神经网络

训练过程有两个迭代组成:

  • 第一层迭代:代表迭代周期(epoch)
  • 第二层迭代:对
    DataLoader
    传来的每批次数据集进行训练

在每一次循环中:

  • 向模型提供输入(正向传播)
  • 计算损失(正向传播)
  • 将老梯度归零
  • 调用
    loss.backward()
    来计算损失相对所有参数的梯度(反向传播)
  • 让优化器朝着更低的损失迈进

定义训练的函数,并尝试在GPU上进行训练:

device =torch.device('cuda' if torch.cuda.is_available() else 'cpu')
print(f"Training on {device}.")
Training on cuda.
import datetime

def train_loop(n_epochs, optimizer, model, loss_fn, train_loader):
    for epoch in range(1, n_epochs+1):
        loss_train = 0.0
        for imgs, labels in train_loader:   # 在数据加载器中获取批处理循环数据集

            imgs = imgs.to(device=device)   # 这两行代码将imgs labels移动到device指定的设备
            labels = labels.to(device=device)

            outputs = model(imgs)           # 通过模型计算一个批次的结果
            loss = loss_fn(outputs, labels) # 计算最小化损失
            optimizer.zero_grad()           # 去掉最后一轮的梯度
            loss.backward()                 # 执行反向传播
            optimizer.step()                # 更新模型
            loss_train += loss.item()       # 对每层循环得到的损失求和,避免梯度变化

        if epoch ==1 or epoch%10 == 0:
            print("{} Epoch {}, Train loss {}".             # 总损失/训练数据加载器的长度,得到每批平均损失
                  format(datetime.datetime.now(), epoch, loss_train / len(train_loader)))

上面已经准备好了
model

train_loader
,还需准备
optimizere

loss_fn

import torch.optim as optim

# 模型也需要搬到GPU,否则会报错:
model = Net().to(device=device)    # RuntimeError: Input type (torch.cuda.FloatTensor) and weight type (torch.FloatTensor) should be the same

optimizer = optim.SGD(model.parameters(), lr=1e-2)  # 使用随机梯度下降优化器
loss_fn = nn.CrossEntropyLoss() # 交叉熵损失

# 调用训练循环
train_loop(n_epochs=100,
            optimizer=optimizer,
            model=model,
            loss_fn=loss_fn,
            train_loader=train_loader)
2023-04-08 16:49:02.897419 Epoch 1, Train loss 0.6789790311684976
2023-04-08 16:50:12.260929 Epoch 10, Train loss 0.45727716023341203
2023-04-08 16:51:29.474510 Epoch 20, Train loss 0.3460641039105562
2023-04-08 16:52:45.412158 Epoch 30, Train loss 0.3255017975775095
2023-04-08 16:53:59.949844 Epoch 40, Train loss 0.3127688937462293
2023-04-08 16:55:14.758279 Epoch 50, Train loss 0.3003842735137695
2023-04-08 16:56:29.352129 Epoch 60, Train loss 0.2895182979603608
2023-04-08 16:57:44.294486 Epoch 70, Train loss 0.2761662933879938
2023-04-08 16:58:58.890680 Epoch 80, Train loss 0.2641859925710238
2023-04-08 17:00:13.058129 Epoch 90, Train loss 0.25313296078298336
2023-04-08 17:01:27.434814 Epoch 100, Train loss 0.2413799591266956
# 再创建一个没有被打乱的训练数据加载器,用于验证
train_loader_ = DataLoader(cifar2, batch_size=64, shuffle=False, num_workers=4, drop_last=True)

def validate(model, train_loader, val_loader):
    for name, loader in [('trian', train_loader), ('val', val_loader)]:
        correct = 0
        total = 0
        with torch.no_grad():   # 在这里,我们希望不更新参数
            for imgs, labels in loader:

                imgs = imgs.to(device=device)
                labels = labels.to(device=device)

                outputs = model(imgs)
                _, predicted = torch.max(outputs, dim=1)    # 将最大值的索引作为输出

                total += labels.shape[0]
                correct += int((predicted == labels).sum())
        print("Accuracy: {}: {}".format(name, correct/total))

validate(model, train_loader_, val_loader)
Accuracy: trian: 0.9037459935897436
Accuracy: val: 0.8765120967741935

准确率确实还可以,但模型结构还是过于简单,继续顺着书本调整下!

改进神经网络

一般来说,模型训练结果的优劣主要有三方面决定:1、模型结构;2、训练过程;3、数据集。

在这里,暂不考虑第三种带来的变化,事实上,很多情况下,数据集的质量很能影响模型的泛化性,但是由于我们使用的是专门用于教学的数据集,因此只考虑前两种变化对模型预测精确度带来的变化。

增加内存容量:宽度

宽度,即神经网络的宽度:每层神经元数,或每个卷积的通道数。

我们只需要在第1个卷积层中指定更多的输出通道,并相应地增加后续层数,便可得到更长的向量。

此外,将模型训练过程中的中间通道数作为参数而不是硬编码数字传递给
__init__()

现在重写
Net
类:

class NetWidth(nn.Module):
    def __init__(self, n_channel=32):
        super().__init__()
        self.n_channel = n_channel
        self.conv1 = nn.Conv2d(in_channels=3, out_channels=n_channel, kernel_size=3, padding=1)
        self.conv2 = nn.Conv2d(in_channels=n_channel, out_channels=n_channel//2,        # 增加了神经网络的宽度
                               kernel_size=3, padding=1)  
        self.fc1 = nn.Linear((n_channel//2)*8*8, 32)    
        self.fc2 = nn.Linear(32, 2)

    def forward(self, x):
        out = F.max_pool2d(torch.tanh(self.conv1(x)), 2)
        out = F.max_pool2d(torch.tanh(out), 2)
        out = out.view(-1, (self.n_channel//2)*8*8)
        out = torch.tanh(self.fc1(out))
        out = self.fc2(out)
        return out

现在看看改变了宽度后,模型的参数数量:

n1 = sum(p.numel() for p in model.parameters())  # 增加宽度前的模型参数数量
model2 = NetWidth().to(device=device)            
n2 = sum(p.numel() for p in model2.parameters())    # 增加宽度后的模型参数数量
print(n1)
print(n2)
18090
38386

容量越大,模型所能管理的输入的可变性就越大。但是相应的,模型出现过拟合的可能性也会增加。

处理增加数据集来避免过拟合之外,还可以调整训练过程。

模型收敛和泛化:正则化

  1. 权重惩罚
    稳定泛化第一种方法添加正则化项。在这里我们添加
    L2
    正则化,它是所有权重的平方和(
    L1
    正则化是模型中所有权重的绝对值之和)。

    L2
    正则化也成为权重衰减,对参数的负梯度为:
    \(w_i=-2\times lambda\times w_i\)
    ,其中
    lambda
    为超参数,在Pytorch中称为权重衰减。

    因此,在损失函数中加入L2正则化,相当于在优化步骤中将每个权重按其当前值的比例递减。权重参数适用于网络的所有参数,例如偏置。
def training_loop_l2reg(n_epochs, optimizer, model, loss_fn, train_loader):
    for epoch in range(1, n_epochs+1):
        loss_train = 0.0
        for imgs, labels in train_loader:
            imgs = imgs.to(device=device)
            labels = labels.to(device=device)
            outputs = model(imgs)
            loss = loss_fn(outputs, labels)

            l2_lambda = 0.001       # 加入L2正则化
            l2_norm = sum(p.pow(2.0).sum() for p in model.parameters())

            loss = loss+l2_lambda*l2_norm
            optimizer.zero_grad()
            loss.backward()
            optimizer.step()
            loss_train += loss.item()

        if epoch==1 or epoch%10 == 0:
            print("{} Epoch {}, Training loss {}".format(
                datetime.datetime.now(), epoch, loss_train/len(train_loader)
            ))
  1. Dropout

Dropout将网络每轮训练迭代中神经元随即清零。Dropout在每次迭代中有效地生成具有不同神经元拓扑结构的模型,使得模型中的神经元在过拟合过程中协调记忆的机会更少。另一中观点是,Dropout在整个网络中干扰了模型生成的特征,产生了一种接近于增强的效果。

class NetDropout(nn.Module):
    def __init__(self, n_channel=32):
        super().__init__()
        self.n_channel = n_channel
        self.conv1 = nn.Conv2d(in_channels=3, out_channels=n_channel, kernel_size=3, padding=1)
        self.conv1_dropout = nn.Dropout2d(p=0.4)                                        # 使用dropout,p为一个元素归零的概率
        self.conv2 = nn.Conv2d(in_channels=n_channel, out_channels=n_channel//2,        # 增加了神经网络的宽度
                               kernel_size=3, padding=1)  
        self.conv2_dropout = nn.Dropout2d(p=0.4)
        self.fc1 = nn.Linear((n_channel//2)*8*8, 32)    
        self.fc2 = nn.Linear(32, 2)

    def forward(self, x):
        out = F.max_pool2d(torch.tanh(self.conv1(x)), 2)
        out = self.conv2_dropout(out)
        out = F.max_pool2d(torch.tanh(out), 2)
        out = self.conv2_dropout(out)
        out = out.view(-1, (self.n_channel//2)*8*8)
        out = torch.tanh(self.fc1(out))
        out = self.fc2(out)
        return out
  1. 批量化归一

批量归一化背后的主要思想是将输入重新调整到网络的激活状态,从而使小批量具有一定的理想分布,这有助于避免激活函数的输入过多地进入函数的包和部分,从而消除梯度并减慢训练速度。

class NetBatchNorm(nn.Module):
    def __init__(self, n_channel=32):
        super().__init__()
        self.n_channel = n_channel
        self.conv1 = nn.Conv2d(in_channels=3, out_channels=n_channel, kernel_size=3, padding=1)
        self.conv1_batchnorm = nn.BatchNorm2d(num_features=n_channel)                   # 使用批量归一化
        self.conv2 = nn.Conv2d(in_channels=n_channel, out_channels=n_channel//2,        # 增加了神经网络的宽度
                               kernel_size=3, padding=1)  
        self.conv2_batchnorm = nn.BatchNorm2d(num_features=n_channel//2)
        self.fc1 = nn.Linear((n_channel//2)*8*8, 32)    
        self.fc2 = nn.Linear(32, 2)

    def forward(self, x):
        out = self.conv1_batchnorm(self.conv1(x))
        out = F.max_pool2d(torch.tanh(out), 2)
        out = self.conv2_batchnorm(self.conv2(out))
        out = F.max_pool2d(torch.tanh(out), 2)
        out = out.view(-1, (self.n_channel//2)*8*8)
        out = torch.tanh(self.fc1(out))
        out = self.fc2(out)
        return out

现在使用
NetBatchNorm

training_loop_l2reg
重新训练并评估我们的模型,希望较之前能有提升!

model = NetBatchNorm().to(device=device)
optimizer = optim.SGD(model.parameters(), lr=1e-2)  # 使用随机梯度下降优化器
loss_fn = nn.CrossEntropyLoss() # 交叉熵损失

training_loop_l2reg(
    n_epochs=100,
    optimizer=optimizer,
    model=model,
    loss_fn=loss_fn,
    train_loader=train_loader
)
2023-04-08 17:22:51.919275 Epoch 1, Training loss 0.5400954796335636
2023-04-08 17:24:01.077684 Epoch 10, Training loss 0.3433214044914796
2023-04-08 17:25:18.132063 Epoch 20, Training loss 0.2857391257316638
2023-04-08 17:26:34.441769 Epoch 30, Training loss 0.24476417631675035
2023-04-08 17:27:50.975030 Epoch 40, Training loss 0.21916839241599426
2023-04-08 17:29:09.751893 Epoch 50, Training loss 0.193350423557254
2023-04-08 17:30:26.556550 Epoch 60, Training loss 0.17405275838115278
2023-04-08 17:31:46.126329 Epoch 70, Training loss 0.15676446583790657
2023-04-08 17:33:06.333187 Epoch 80, Training loss 0.14270161565106648
2023-04-08 17:34:25.760439 Epoch 90, Training loss 0.13285309878679422
2023-04-08 17:35:45.502106 Epoch 100, Training loss 0.12409532667161563

再次测量模型精度:

model.eval()
validate(model=model, train_loader=train_loader_, val_loader=val_loader)
Accuracy: trian: 0.9859775641025641
Accuracy: val: 0.8805443548387096

可以看到在训练集上,准确率高达0.98,而验证集却只有0.88,还是存在着过拟合的风险。

最后将模型参数保存:

torch.save(model.state_dict(), "./models/birdsVsPlane.pt")  # 只保存了模型参数

由于我们使用的模型和数据都是在GPU上进行训练的,因此加载模型还需要确定设备位置:

load_model = NetBatchNorm().to(device=device)
load_model.load_state_dict(torch.load("./models/birdsVsPlane.pt", map_location=device))
<All keys matched successfully>

加载完毕,简单测试下:

img, label = cifar2[5]
img = img.to(device=device)
load_model(img.unsqueeze(0)), label
(tensor([[ 4.4285, -4.5254]], device='cuda:0', grad_fn=<AddmmBackward0>), 0)
img_ = img.to('cpu')    # 使用plt绘图,要先将图片转到cpu上
plt.imshow(img_.permute(1,2,0))
<matplotlib.image.AxesImage at 0x29d35a4b850>

参考文献

[1] Eli Stevens. Deep Learning with Pytorch[M]. 1. 人民邮电出版社, 2022.02 :144-163.

标签: none

添加新评论