Implementation of jump connection of RESNET (pytorch)

https://www.cnblogs.com/jiyou/p/11272650.html

In the original paper , The residual path can be roughly divided into 2 Kind of , One kind has bottleneck structure , It is shown on the right of the figure below , Two of them 1×1  Convolution layer , Used to reduce dimension first and then increase dimension , Mainly out of Practical considerations to reduce computational complexity , be called “bottleneck block”, The other is not bottleneck structure , As shown on the left of the figure below , be called “basic block”.basic block from 2 individual 3×3 The convolution layer constitutes

 shortcut The path can also be roughly divided into 2 Kind of , Depends on whether the residual path has changed feature map Quantity and size , One is to input x Output... Intact , The other needs to go through 1×1 Convolution to increase dimension or reduce sampling , The main functions are Compare the output to 𝐹(𝑥) The output of the path remains shape Agreement , The improvement of network performance is not obvious .

ResNet V1 Implementation code ：

explain ：shortcut  Judgment condition

if stride != 1 or in_channel != self.expansion * out_channel

First ,shortcut The judgment is Of the branch of identity mapping featuremap Whether it is output with another branch featuremap Is the dimension the same （ Size and depth ）. If different , Change the branch of identity mapping featuremap dimension , Make it the same .
in_channel != self.expansion * out_channel The channels are different in depth , It's obvious to use 1x1 Convolution of changes the channel .
stride != 1, The default used in the code pad = 1, stripe = 1, Through convolution formula ,

(W + 2*P - k)/s +1 You know , The input and output dimensions remain unchanged . When stripe ！= 1 when , Two branches featuremap Different sizes , Can't add directly . Use here 1*1 Convolution of , Equivalent to the featturemap Sampling on , Discarded a lot of information . This is rarely the case , At most, it is the difference in the channel .

nn.BatchNorm2d(out_channel)  # out_channel For doing BatchNorm2d Number of convoluted channels

The code is mainly divided into two functions ：

BasicBlock ： This is the basic module , By two superimposed 3*3 Convolution composition . BasicBlock Generally, there is no need to change the number of channels , So general expansion = 1, The final number of output channels is out_channel.
Bottleneck： Bottleneck module , There are three convolutions 1x1,3x3,1x1, They are used to reduce dimensions , Convolution processing , Raise the dimension , Bottleneck It is generally used to raise the channel dimension , therefore  out_channel Is in the branch 1x1 Dimension reduction （ Reduce the amount of computation ） and 3x3 Number of convoluted channels , The final number of output channels is  expansion x out_channel.

import torch
import torch.nn as nn

#  be used for ResNet18 and 34 The remnant of , It's using 2 individual 3x3 Convolution of 
#  notes ： The final number of output channels is  expansion * out_channel

class BasicBlock_V1(nn.Module):

   #  Channel magnification 
    expansion = 1

    def __init__(self, in_channel, out_channel, stride=1):
        super(BasicBlock_V1, self).__init__()

        #  first  3x3  Convolution normal incoming  stride
        self.conv1 = nn.Conv2d(in_channel, out_channel, kernel_size=3,
                               stride=stride, padding=1, bias=False)
        self.bn1 = nn.BatchNorm2d(out_channel)
        self.relu = nn.ReLU(inplace=True)

        # the second 3x3 Convolution  stride=1,  Do not change the previous step featuremap Size 
        self.conv2 = nn.Conv2d(out_channel, self.expansion * out_channel, kernel_size=3,
                               stride=1, padding=1, bias=False)
        self.bn2 = nn.BatchNorm2d(self.expansion * out_channel)

        self.shortcut = None
        #  Treated x To work with x The dimensions of are the same ( Size and depth )
        #  If it's not the same , Convolution needs to be added +BN To transform into the same dimension 
        if stride != 1 or in_channel != self.expansion * out_channel:
            self.shortcut = nn.Sequential(
                nn.Conv2d(in_channel, self.expansion * out_channel,
                          kernel_size=1, stride=stride, bias=False),
                nn.BatchNorm2d(self.expansion * out_channel)
            )

    def forward(self, x):
        identity = x

        # 3x3 cov + BN + relu
        out = self.conv1(x)
        out = self.bn1(out)
        out = self.relu(out)

        # 3x3 cov + BN
        out = self.conv2(out)
        out = self.bn2(out)

        #  In case of dimension reduction or dimension increase, make sure to add 
        #  What needs to be transformed is the branch of identity mapping featuremap
        if self.shortcut is not None:
            identity = self.shortcut(x)

        # add + relu
        out += identity
        out = self.relu(out)
        return out


#  be used for ResNet50,101 and 152 The remnant of , It's using 1x1+3x3+1x1 Convolution of 
class Bottleneck_V1(nn.Module):

    #  Channel magnification 
    #  front 1x1 and 3x3 Convolution filter The number is equal , Last 1x1 Convolution is its expansion times 
    expansion = 4

    def __init__(self, in_channel, out_channel, stride=1):
        super(Bottleneck_V1, self).__init__()
        self.conv1 = nn.Conv2d(in_channel, out_channel, kernel_size=1, bias=False)
        self.bn1 = nn.BatchNorm2d(out_channel)
        self.relu = nn.ReLU(inplace=True)  # relu  share 

        self.conv2 = nn.Conv2d(out_channel, out_channel, kernel_size=3,
                               stride=stride, padding=1, bias=False)
        self.bn2 = nn.BatchNorm2d(out_channel)

        self.conv3 = nn.Conv2d(out_channel, self.expansion * out_channel,
                               kernel_size=1, bias=False)
        self.bn3 = nn.BatchNorm2d(self.expansion * out_channel)

        self.shortcut = None

        #  Treated x To work with x The dimensions of are the same ( Size and depth )
        #  If it's not the same , Convolution needs to be added +BN To transform into the same dimension 
        if stride != 1 or in_channel != self.expansion * out_channel:
            self.shortcut = nn.Sequential(
                nn.Conv2d(in_channel, self.expansion * out_channel,
                          kernel_size=1, stride=stride, bias=False),
                nn.BatchNorm2d(self.expansion * out_channel)
            )

    #  Same as basicblock
    def forward(self, x):

        #  Identity mapping branch 
        identity = x

        # 1x1 cov + BN + relu
        out = self.conv1(x)
        out = self.bn1(out)
        out = self.relu(out)

        # 3x3 cov + BN + relu
        out = self.conv2(out)
        out = self.bn2(out)
        out = self.relu(out)

        # 1x1 cov + BN
        out = self.conv3(out)
        out = self.bn3(out)

        #  In case of dimension reduction or dimension increase, make sure to add 
        #  What needs to be transformed is the branch of identity mapping featuremap
        if self.shortcut is not None:
            identity = self.shortcut(x)
        # add + relu
        out += identity
        out = self.relu(out)
        return out

ResNet V2  Implementation code ：

  above resnetV1 Code implementation of , resnetV2 Structure and V1 Different , As shown in the figure below

Resnet V2 shotcut and Resnet V1 It's different ,Resnet V2 Of shotcut No, BatchNorm

import torch
import torch.nn as nn

class BasicBlock_V2(nn.Module):

    #  Channel magnification , In less than 
    expansion = 1

    def __init__(self, in_channel, out_channel, stride=1):
        super(BasicBlock_V2, self).__init__()
        self.bn1 = nn.BatchNorm2d(in_channel)
        self.conv1 = nn.Conv2d(in_channel, out_channel, kernel_size=3,
                               stride=stride, padding=1, bias=False)
        self.relu = nn.ReLU(inplace=True)

        self.bn2 = nn.BatchNorm2d(out_channel)

        self.conv2 = nn.Conv2d(out_channel, out_channel, kernel_size=3,
                               stride=1, padding=1, bias=False)

        self.shortcut = None
        if stride != 1 or in_channel != out_channel:
            self.shortcut = nn.Sequential(
                nn.Conv2d(in_channel, out_channel,
                          kernel_size=1, stride=stride, bias=False)
            )

    def forward(self, x):

        #  Identity mapping branch 
        identity = x

        # BN + relu
        out = self.bn1(x)
        out = self.relu(out)

        # 3x3 conv + BN + relu
        out = self.conv1(out)
        out = self.bn2(out)
        out = self.relu(out)

        # 3x3 conv
        out = self.conv2(out)

        #  Judge   Identity 
        if self.shortcut is not None:
            identity = self.shortcut(out)

        # add
        out += identity

        return out


class Bottleneck_V2(nn.Module):

    #  Channel magnification 
    #  front 1x1 and 3x3 Convolution filter The number is equal , Last 1x1 Convolution is its expansion times 
    expansion = 4

    def __init__(self, in_channel, out_channel, stride=1):
        super(Bottleneck_V2, self).__init__()

        self.bn0 = nn.BatchNorm2d(in_channel)
        self.relu = nn.ReLU(inplace=True)

        self.conv1 = nn.Conv2d(in_channel, out_channel, kernel_size=1, bias=False)
        self.bn1 = nn.BatchNorm2d(out_channel)
        self.conv2 = nn.Conv2d(out_channel, out_channel, kernel_size=3,
                               stride=stride, padding=1, bias=False)
        self.bn2 = nn.BatchNorm2d(out_channel)

        #  The final output channel of convolution branch is  self.expansion*out_channel
        self.conv3 = nn.Conv2d(out_channel, self.expansion * out_channel,
                               kernel_size=1, bias=False)
        self.shortcut = None
        if stride != 1 or in_channel != self.expansion * out_channel:
            self.shortcut = nn.Sequential(
                nn.Conv2d(in_channel, self.expansion * out_channel,
                          kernel_size=1, stride=stride, bias=False)
            )

    def forward(self, x):

        #  Identity mapping branch 
        identity = x

        # BN + relu
        out = self.bn0(x)
        out = self.relu(out)

        # 1x1 conv + BN + relu
        out = self.conv1(out)
        out = self.bn1(out)
        out = self.relu(out)

        # 3x3 conv + BN + relu
        out = self.conv2(out)
        out = self.bn2(out)
        out = self.relu(out)

        # 1x1 conv
        out = self.conv3(out)

        #  Treated x To work with x The dimensions of are the same ( Size and depth )
        #  If it's not the same , Convolution needs to be added +BN To transform into the same dimension 
        if self.shortcut is not None:
            identity = self.shortcut(out)

        # add
        out += identity

        return out

Source code ：

The source code is relatively simple , There are mainly two things that cannot be

1、 Is in the source code  relu It uses F.relu()

#  The first one is 
import torch.functional as F
out = F.ReLU(input)

#  The second kind 
import torch.nn as nn
nn.ReLU(inplace=True)
nn.RuLU(input)

Both methods use relu Activate , It's just that the scenarios used are different ,F.ReLU() It's a function call , Generally used in foreward In the function . and nn.ReLU() Is a module call , It is generally used when defining the network layer .

When used print(net) When the output , There will be nn.ReLU() layer , and F.ReLU() There is no output .

nn.ReLU(inplace=True) in inplace The role of : https://www.cnblogs.com/wanghui-garcia/p/10642665.html

2、shortcut() Branch judgment processing , better , You can learn

resnet V1 Source code ：

import torch
import torch.nn as nn
import torch.nn.functional as F

#  be used for ResNet18 and 34 The remnant of , It's using 2 individual 3x3 Convolution of 
class BasicBlock(nn.Module):
    expansion = 1

    def __init__(self, in_planes, planes, stride=1):
        super(BasicBlock, self).__init__()
        self.conv1 = nn.Conv2d(in_planes, planes, kernel_size=3,
                               stride=stride, padding=1, bias=False)
        self.bn1 = nn.BatchNorm2d(planes)
        self.conv2 = nn.Conv2d(planes, planes, kernel_size=3,
                               stride=1, padding=1, bias=False)
        self.bn2 = nn.BatchNorm2d(planes)
        self.shortcut = nn.Sequential()
        #  Treated x To work with x The dimensions of are the same ( Size and depth )
        #  If it's not the same , Convolution needs to be added +BN To transform into the same dimension 
        if stride != 1 or in_planes != self.expansion*planes:
            self.shortcut = nn.Sequential(
                nn.Conv2d(in_planes, self.expansion*planes,
                          kernel_size=1, stride=stride, bias=False),
                nn.BatchNorm2d(self.expansion*planes)
            )

    def forward(self, x):
        out = F.relu(self.bn1(self.conv1(x)))
        out = self.bn2(self.conv2(out))
        out += self.shortcut(x)
        out = F.relu(out)
        return out


#  be used for ResNet50,101 and 152 The remnant of , It's using 1x1+3x3+1x1 Convolution of 
class Bottleneck(nn.Module):
    #  front 1x1 and 3x3 Convolution filter The number is equal , Last 1x1 Convolution is its expansion times 
    expansion = 4

    def __init__(self, in_planes, planes, stride=1):
        super(Bottleneck, self).__init__()
        self.conv1 = nn.Conv2d(in_planes, planes, kernel_size=1, bias=False)
        self.bn1 = nn.BatchNorm2d(planes)
        self.conv2 = nn.Conv2d(planes, planes, kernel_size=3,
                               stride=stride, padding=1, bias=False)
        self.bn2 = nn.BatchNorm2d(planes)
        self.conv3 = nn.Conv2d(planes, self.expansion*planes,
                               kernel_size=1, bias=False)
        self.bn3 = nn.BatchNorm2d(self.expansion*planes)

        self.shortcut = nn.Sequential()
        if stride != 1 or in_planes != self.expansion*planes:
            self.shortcut = nn.Sequential(
                nn.Conv2d(in_planes, self.expansion*planes,
                          kernel_size=1, stride=stride, bias=False),
                nn.BatchNorm2d(self.expansion*planes)
            )

    def forward(self, x):
        out = F.relu(self.bn1(self.conv1(x)))
        out = F.relu(self.bn2(self.conv2(out)))
        out = self.bn3(self.conv3(out))
        out += self.shortcut(x)
        out = F.relu(out)
        return out


class ResNet(nn.Module):
    def __init__(self, block, num_blocks, num_classes=10):
        super(ResNet, self).__init__()
        self.in_planes = 64

        self.conv1 = nn.Conv2d(3, 64, kernel_size=3,
                               stride=1, padding=1, bias=False)
        self.bn1 = nn.BatchNorm2d(64)
        
        self.layer1 = self._make_layer(block, 64, num_blocks[0], stride=1)
        self.layer2 = self._make_layer(block, 128, num_blocks[1], stride=2)
        self.layer3 = self._make_layer(block, 256, num_blocks[2], stride=2)
        self.layer4 = self._make_layer(block, 512, num_blocks[3], stride=2)
        self.linear = nn.Linear(512*block.expansion, num_classes)

    def _make_layer(self, block, planes, num_blocks, stride):
        strides = [stride] + [1]*(num_blocks-1)
        layers = []
        for stride in strides:
            layers.append(block(self.in_planes, planes, stride))
            self.in_planes = planes * block.expansion
        return nn.Sequential(*layers)

    def forward(self, x):
        out = F.relu(self.bn1(self.conv1(x)))
        out = self.layer1(out)
        out = self.layer2(out)
        out = self.layer3(out)
        out = self.layer4(out)
        out = F.avg_pool2d(out, 4)
        out = out.view(out.size(0), -1)
        out = self.linear(out)
        return out


def ResNet18():
    return ResNet(BasicBlock, [2,2,2,2])

def ResNet34():
    return ResNet(BasicBlock, [3,4,6,3])

def ResNet50():
    return ResNet(Bottleneck, [3,4,6,3])

def ResNet101():
    return ResNet(Bottleneck, [3,4,23,3])

def ResNet152():
    return ResNet(Bottleneck, [3,8,36,3])


def test():
    net = ResNet18()
    y = net(torch.randn(1,3,32,32))
    print(y.size())

# test()

 resnet V2 Source code

'''Pre-activation ResNet in PyTorch.

Reference:
[1] Kaiming He, Xiangyu Zhang, Shaoqing Ren, Jian Sun
    Identity Mappings in Deep Residual Networks. arXiv:1603.05027
'''
import torch
import torch.nn as nn
import torch.nn.functional as F


class PreActBlock(nn.Module):
    '''Pre-activation version of the BasicBlock.'''
    expansion = 1

    def __init__(self, in_planes, planes, stride=1):
        super(PreActBlock, self).__init__()
        self.bn1 = nn.BatchNorm2d(in_planes)
        self.conv1 = nn.Conv2d(in_planes, planes, kernel_size=3,
                               stride=stride, padding=1, bias=False)
        self.bn2 = nn.BatchNorm2d(planes)
        self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, 
                               stride=1, padding=1, bias=False)

        if stride != 1 or in_planes != self.expansion*planes:
            self.shortcut = nn.Sequential(
                nn.Conv2d(in_planes, self.expansion*planes,
                          kernel_size=1, stride=stride, bias=False)
            )

    def forward(self, x):
        out = F.relu(self.bn1(x))
        shortcut = self.shortcut(out) if hasattr(self, 'shortcut') else x
        out = self.conv1(out)
        out = self.conv2(F.relu(self.bn2(out)))
        out += shortcut
        return out


class PreActBottleneck(nn.Module):
    '''Pre-activation version of the original Bottleneck module.'''
    expansion = 4

    def __init__(self, in_planes, planes, stride=1):
        super(PreActBottleneck, self).__init__()
        self.bn1 = nn.BatchNorm2d(in_planes)
        self.conv1 = nn.Conv2d(in_planes, planes, kernel_size=1, bias=False)
        self.bn2 = nn.BatchNorm2d(planes)
        self.conv2 = nn.Conv2d(planes, planes, kernel_size=3,
                               stride=stride, padding=1, bias=False)
        self.bn3 = nn.BatchNorm2d(planes)
        self.conv3 = nn.Conv2d(planes, self.expansion*planes,
                               kernel_size=1, bias=False)

        if stride != 1 or in_planes != self.expansion*planes:
            self.shortcut = nn.Sequential(
                nn.Conv2d(in_planes, self.expansion*planes,
                          kernel_size=1, stride=stride, bias=False)
            )

    def forward(self, x):
        out = F.relu(self.bn1(x))
        shortcut = self.shortcut(out) if hasattr(self, 'shortcut') else x
        out = self.conv1(out)
        out = self.conv2(F.relu(self.bn2(out)))
        out = self.conv3(F.relu(self.bn3(out)))
        out += shortcut
        return out


class PreActResNet(nn.Module):
    def __init__(self, block, num_blocks, num_classes=10):
        super(PreActResNet, self).__init__()
        self.in_planes = 64

        self.conv1 = nn.Conv2d(3, 64, kernel_size=3, stride=1,
                               padding=1, bias=False)
        self.layer1 = self._make_layer(block, 64, num_blocks[0], stride=1)
        self.layer2 = self._make_layer(block, 128, num_blocks[1], stride=2)
        self.layer3 = self._make_layer(block, 256, num_blocks[2], stride=2)
        self.layer4 = self._make_layer(block, 512, num_blocks[3], stride=2)
        self.linear = nn.Linear(512*block.expansion, num_classes)

    def _make_layer(self, block, planes, num_blocks, stride):
        strides = [stride] + [1]*(num_blocks-1)
        layers = []
        for stride in strides:
            layers.append(block(self.in_planes, planes, stride))
            self.in_planes = planes * block.expansion
        return nn.Sequential(*layers)

    def forward(self, x):
        out = self.conv1(x)
        out = self.layer1(out)
        out = self.layer2(out)
        out = self.layer3(out)
        out = self.layer4(out)
        out = F.avg_pool2d(out, 4)
        out = out.view(out.size(0), -1)
        out = self.linear(out)
        return out


def PreActResNet18():
    return PreActResNet(PreActBlock, [2,2,2,2])

def PreActResNet34():
    return PreActResNet(PreActBlock, [3,4,6,3])

def PreActResNet50():
    return PreActResNet(PreActBottleneck, [3,4,6,3])

def PreActResNet101():
    return PreActResNet(PreActBottleneck, [3,4,23,3])

def PreActResNet152():
    return PreActResNet(PreActBottleneck, [3,8,36,3])


def test():
    net = PreActResNet18()
    y = net((torch.randn(1,3,32,32)))
    print(y.size())

# test()