Pytoch (VI) -- model tuning tricks

1. Regularization Regularization

1.1 L1 Regularization

import torch
import torch.nn.functional as F
from torch import nn

device=torch.device("cuda:0")
MLP = nn.Sequential(nn.Linear(128,64),
                    nn.ReLU(inplace=True),
                    nn.Linear(64,32),
                    nn.ReLU(inplace=True),
                    nn.Linear(32,10)
)
MLP.to(device) 
loss_classify = nn.CrossEntropyLoss().to(device)
# L1 norm 
l1_loss = 0
for param in MLP.parameters():
    l1_loss += torch.sum(torch.abs(param))
loss = loss_classify+l1_loss

1.2 L2 Regularization

import torch
import torch.nn.functional as F
from torch import nn

device=torch.device("cuda:0")
MLP = nn.Sequential(nn.Linear(128,64),
                    nn.ReLU(inplace=True),
                    nn.Linear(64,32),
                    nn.ReLU(inplace=True),
                    nn.Linear(32,10)
)
MLP.to(device) 


# L2 norm 
opt = torch.optim.SGD(MLP.parameters(),lr=0.001,weight_decay=0.1) #  adopt weight_decay Realization L2
loss = nn.CrossEntropyLoss().to(device)

2 Momentum and learning rate decay

2.1 momentum

opt = torch.optim.SGD(model.parameters(),lr=0.001,momentum=0.78,weight_decay=0.1)

2.2 learning rate tunning

• torch.optim.lr_scheduler.ReduceLROnPlateau() Use when the value of the loss function does not decrease
• torch.optim.lr_scheduler.StepLR() Reduce the learning rate according to a certain number of steps

opt = torch.optim.SGD(net.parameters(),lr=1)
lr_scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer=opt,mode="min",factor=0.1,patience=10)
for epoch in torch.arange(1000):
    loss_val = train(...)
    lr_scheduler.step(loss_val) #  monitor loss

opt = torch.optim.SGD(net.parameters(),lr=1)
lr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer=opt,step_size=30,gamma=0.1)
for epoch in torch.arange(1000):
    lr_scheduler.step() #  monitor loss
    train(...)

3. Early Stopping

Click here

4. Dropout

model = nn.Sequential(
nn.Linear(256,128),
nn.Dropout(p=0.5),
nn.ReLu(),
)

by CyrusMay 2022 07 03