丢弃法 正则化

丢弃法

目的

在训练中加入偏差，使得训练更稳定

方法

训练时在隐藏层后加一层扰动，在加入噪音的基础上不改变原先的期望。在推理时不使用。

内容讲解

代码

import torch 
from torch import nn
from d2l import torch as d2l
import matplotlib.pyplot as plt

def droppot_layer(X,dropput):
    assert 0 <= dropput <= 1   #限制dropput的范围
    if dropput == 0:  #如果dropput为0,则
        return X
    if dropput == 1:  #如果dropput为1,则
        return torch.zeros_like(X)  #返回与X同形状的全0张
    mask = (torch.randn(X.shape) > dropput).float()  #生成与X同形状的mask张量（0或1 ）

    return mask * X / (1.0 - dropput)  #返回mask




class Net(nn.Module):
    def __init__(self,num_inputs, num_outputs, num_hiddens1, num_hidden2, dropout1, dropout2):
        super(Net, self).__init__()
        self.flatten = nn.Flatten()
        self.linear1 = nn.Linear(num_inputs, num_hiddens1)       #隐藏层1
        self.linear2 = nn.Linear(num_hiddens1, num_hidden2)      #隐藏层2
        self.linear3 = nn.Linear(num_hidden2, num_outputs)       #输出层
        self.relu = nn.ReLU()  # 定义ReLU激活函数
        self.dropout1 = dropout1  #隐藏层1的dropout率
        self.dropout2 = dropout2

    def forward(self, X):
        H1 = torch.relu(self.linear1(self.flatten(X.reshape((-1, num_inputs)))))  #隐藏层1
        if self.training:  #如果模型处于训练模式
            H1 = droppot_layer(H1, dropout1)  #对隐藏层1应用dropout
        H2 = self.relu(self.linear2(H1))  #隐藏层2
        if self.training:  #如果模型处于训练模式
            H2 = droppot_layer(H2, dropout2)  #对隐藏层2应用dropout
        out = self.linear3(H2)  #输出层
        return out

if __name__ == "__main__":
    X = torch.arange(784, dtype=torch.float32).reshape((28, 28))
    num_inputs, num_outputs, num_hiddens1, num_hidden2 = 784,10,256,256
    dropout1, dropout2 = 0.2, 0.5

    net = Net(num_inputs, num_outputs, num_hiddens1, num_hidden2, dropout1, dropout2)
    num_epochs, lr, batch_size = 10, 0.1, 256
    loss = nn.CrossEntropyLoss()
    train_iter, test_iter = d2l.load_data_fashion_mnist(batch_size)
    trainer = torch.optim.SGD(net.parameters(), lr=lr)
    d2l.train_ch3(
        net, train_iter, test_iter, loss, num_epochs, trainer
    )
    plt.show()  # 确保脚本运行时弹出图像窗口
    print(net(X))  # 测试网络输出

个人理解

丢弃法是在训练时，随机舍弃几个输入值，并对其他输入做处理，即加入噪音又维持偏差为0